Debugging on SIL level.

This change follows up on an idea from Michael (thanks!).
It enables debugging and profiling on SIL level, which is useful for compiler debugging.

There is a new frontend option -gsil which lets the compiler write a SIL file and generated debug info for it.
For details see docs/DebuggingTheCompiler.rst and the comments in SILDebugInfoGenerator.cpp.

docs/DebuggingTheCompiler.rst

@@ -113,6 +113,19 @@ debugging press <CTRL>-C on the LLDB prompt.
 Note that this only works in Xcode if the PATH variable in the scheme's
 environment setting contains the path to the dot tool.
 
+Debugging and Profiling on SIL level
+````````````````````````````````````
+
+The compiler provides a way to debug and profile on SIL level. To enable SIL
+debugging add the front-end option -gsil together with -g. Example::
+
+    swiftc -g -Xfrontend -gsil -O test.swift -o a.out
+
+This writes the SIL after optimizations into a file and generates debug info
+for it. In the debugger and profiler you can then see the SIL code instead of
+the swift source code.
+For details see the SILDebugInfoGenerator pass.
+
 Other Utilities
 ```````````````
 

include/swift/AST/SILOptions.h

@@ -18,6 +18,7 @@
 #ifndef SWIFT_AST_SILOPTIONS_H
 #define SWIFT_AST_SILOPTIONS_H
 
+#include "llvm/ADT/StringRef.h"
 #include <string>
 #include <climits>
 
@@ -96,11 +97,14 @@ public:
   bool EmitProfileCoverageMapping = false;
 
   /// Should we use a pass pipeline passed in via a json file? Null by default.
-  StringRef ExternalPassPipelineFilename;
+  llvm::StringRef ExternalPassPipelineFilename;
   
   /// Emit captures and function contexts using +0 caller-guaranteed ARC
   /// conventions.
   bool EnableGuaranteedClosureContexts = false;
+
+  /// The name of the SIL outputfile if compiled with SIL debugging (-gsil).
+  std::string SILOutputFileNameForDebugging;
 };
 
 } // end namespace swift

include/swift/Option/FrontendOptions.td

@@ -293,6 +293,10 @@ def print_inst_counts : Flag<["-"], "print-inst-counts">,
   HelpText<"Before IRGen, count all the various SIL instructions. Must be used "
            "in conjunction with -print-stats.">;
 
+def debug_on_sil : Flag<["-"], "gsil">,
+  HelpText<"Write the SIL into a file and generate debug-info to debug on SIL "
+           " level.">;
+
 def print_llvm_inline_tree : Flag<["-"], "print-llvm-inline-tree">,
   HelpText<"Print the LLVM inline tree.">;
 

include/swift/SIL/SILInstruction.h

@@ -132,6 +132,11 @@ public:
   const SILDebugScope *getDebugScope() const;
   SILDebugLocation getDebugLocation() const { return Location; }
 
+  /// Sets the debug location.
+  /// Note: Usually it should not be needed to use this function as the location
+  /// is already set in when creating an instruction.
+  void setDebugLocation(SILDebugLocation Loc) { Location = Loc; }
+
   /// removeFromParent - This method unlinks 'self' from the containing basic
   /// block, but does not delete it.
   ///

include/swift/SILOptimizer/PassManager/Passes.def

@@ -187,6 +187,8 @@ PASS(SILCleanup, "cleanup",
      "Cleanup SIL in preparation for IRGen")
 PASS(SILCombine, "sil-combine",
      "Perform small peepholes and combine operations")
+PASS(SILDebugInfoGenerator, "sil-debuginfo-gen",
+     "Write a SIL file for debugging")
 PASS(SILLinker, "linker",
      "Link in all of the serialized SIL referenced in the module")
 PASS(SROA, "sroa",

lib/Frontend/CompilerInvocation.cpp

@@ -1071,6 +1071,16 @@ static bool ParseSILArgs(SILOptions &Opts, ArgList &Args,
   Opts.EnableGuaranteedClosureContexts |=
     Args.hasArg(OPT_enable_guaranteed_closure_contexts);
 
+  if (Args.hasArg(OPT_debug_on_sil)) {
+    // Derive the name of the SIL file for debugging from
+    // the regular outputfile.
+    StringRef BaseName = FEOpts.getSingleOutputFilename();
+    // If there are no or multiple outputfiles, derive the name
+    // from the module name.
+    if (BaseName.empty())
+      BaseName = FEOpts.ModuleName;
+    Opts.SILOutputFileNameForDebugging = BaseName.str();
+  }
   return false;
 }
 
@@ -1107,12 +1117,15 @@ void CompilerInvocation::buildDWARFDebugFlags(std::string &Output,
 static bool ParseIRGenArgs(IRGenOptions &Opts, ArgList &Args,
                            DiagnosticEngine &Diags,
                            const FrontendOptions &FrontendOpts,
+                           const SILOptions &SILOpts,
                            StringRef SDKPath,
                            StringRef ResourceDir,
                            const llvm::Triple &Triple) {
   using namespace options;
 
-  if (const Arg *A = Args.getLastArg(OPT_g_Group)) {
+  if (!SILOpts.SILOutputFileNameForDebugging.empty()) {
+      Opts.DebugInfoKind = IRGenDebugInfoKind::LineTables;
+  } else if (const Arg *A = Args.getLastArg(OPT_g_Group)) {
     if (A->getOption().matches(OPT_g))
       Opts.DebugInfoKind = IRGenDebugInfoKind::Normal;
     else if (A->getOption().matches(options::OPT_gline_tables_only))
@@ -1178,7 +1191,9 @@ static bool ParseIRGenArgs(IRGenOptions &Opts, ArgList &Args,
 
   // TODO: investigate whether these should be removed, in favor of definitions
   // in other classes.
-  if (FrontendOpts.PrimaryInput && FrontendOpts.PrimaryInput->isFilename()) {
+  if (!SILOpts.SILOutputFileNameForDebugging.empty()) {
+    Opts.MainInputFilename = SILOpts.SILOutputFileNameForDebugging;
+  } else if (FrontendOpts.PrimaryInput && FrontendOpts.PrimaryInput->isFilename()) {
     unsigned Index = FrontendOpts.PrimaryInput->Index;
     Opts.MainInputFilename = FrontendOpts.InputFilenames[Index];
   } else if (FrontendOpts.InputFilenames.size() == 1) {
@@ -1310,7 +1325,7 @@ bool CompilerInvocation::parseArgs(ArrayRef<const char *> Args,
     return true;
   }
 
-  if (ParseIRGenArgs(IRGenOpts, ParsedArgs, Diags, FrontendOpts,
+  if (ParseIRGenArgs(IRGenOpts, ParsedArgs, Diags, FrontendOpts, SILOpts,
                      getSDKPath(), SearchPathOpts.RuntimeResourcePath,
                      LangOpts.Target)) {
     return true;

lib/SILOptimizer/PassManager/Passes.cpp

@@ -367,12 +367,16 @@ void swift::runSILOptimizationPasses(SILModule &Module) {
   PM.addSimplifyCFG();
   PM.runOneIteration();
 
+  PM.resetAndRemoveTransformations();
+  
+  // Has only an effect if the -gsil option is specified.
+  PM.addSILDebugInfoGenerator();
+
   // Call the CFG viewer.
   if (SILViewCFG) {
-    PM.resetAndRemoveTransformations();
     PM.addCFGPrinter();
-    PM.runOneIteration();
   }
+  PM.runOneIteration();
 
   // Verify the module, if required.
   if (Module.getOptions().VerifyAll)
@@ -401,6 +405,9 @@ void swift::runSILPassesForOnone(SILModule &Module) {
   // eventually remove unused declarations.
   PM.addExternalDefsToDecls();
 
+  // Has only an effect if the -gsil option is specified.
+  PM.addSILDebugInfoGenerator();
+
   PM.runOneIteration();
 
   // Verify the module, if required.

lib/SILOptimizer/UtilityPasses/CMakeLists.txt

@@ -19,5 +19,6 @@ set(UTILITYPASSES_SOURCES
   UtilityPasses/MemBehaviorDumper.cpp
   UtilityPasses/RCIdentityDumper.cpp
   UtilityPasses/SideEffectsDumper.cpp
+  UtilityPasses/SILDebugInfoGenerator.cpp
   UtilityPasses/StripDebugInfo.cpp
   PARENT_SCOPE)

lib/SILOptimizer/UtilityPasses/SILDebugInfoGenerator.cpp

@@ -0,0 +1,169 @@
+//===--- SILDebugInfoGenerator.cc - Writes a SIL file for debugging -------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "gsil-gen"
+#include "swift/AST/SILOptions.h"
+#include "swift/SIL/SILPrintContext.h"
+#include "swift/SIL/SILModule.h"
+#include "swift/SILOptimizer/PassManager/Transforms.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+using namespace swift;
+
+namespace {
+
+/// A pass for generating debug info on SIL level.
+///
+/// This pass is only enabled if SILOptions::SILOutputFileNameForDebugging is
+/// set (i.e. if the -gsil command line option is specified).
+/// The pass writes all SIL functions into one or multiple output files,
+/// depending on the size of the SIL. The names of the output files are derived
+/// from the main output file.
+///
+///     output file name = <main-output-filename>.gsil_<n>.sil
+///
+/// Where <n> is a consecutive number. The files are stored in the same
+/// same directory as the main output file.
+/// The debug locations and scopes of all functions and instructions are changed
+/// to point to the generated SIL output files.
+/// This enables debugging and profiling on SIL level.
+class SILDebugInfoGenerator : public SILModuleTransform {
+
+  enum {
+    /// To prevent extra large output files, e.g. when compiling the stdlib.
+    LineLimitPerFile = 10000
+  };
+
+  /// A stream for counting line numbers.
+  struct LineCountStream : public llvm::raw_ostream {
+    llvm::raw_ostream &Underlying;
+    int LineNum = 1;
+    uint64_t Pos = 0;
+
+    void write_impl(const char *Ptr, size_t Size) override {
+      for (size_t Idx = 0; Idx < Size; Idx++) {
+        char c = Ptr[Idx];
+        if (c == '\n')
+        ++LineNum;
+      }
+      Underlying.write(Ptr, Size);
+      Pos += Size;
+    }
+
+    uint64_t current_pos() const override { return Pos; }
+
+    LineCountStream(llvm::raw_ostream &Underlying) :
+      llvm::raw_ostream(/* unbuffered = */ true),
+      Underlying(Underlying) { }
+    
+    ~LineCountStream() {
+      flush();
+    }
+  };
+
+  /// A print context which records the line numbers where instructions are
+  /// printed.
+  struct PrintContext : public SILPrintContext {
+
+    LineCountStream LCS;
+
+    llvm::DenseMap<const SILInstruction *, int> LineNums;
+
+    void printInstructionCallBack(const SILInstruction *I) override {
+      // Record the current line number of the instruction.
+      LineNums[I] = LCS.LineNum;
+    }
+
+    PrintContext(llvm::raw_ostream &OS) : SILPrintContext(LCS), LCS(OS) { }
+
+    virtual ~PrintContext() { }
+  };
+
+  void run() override {
+    SILModule *M = getModule();
+    StringRef FileBaseName = M->getOptions().SILOutputFileNameForDebugging;
+    if (FileBaseName.empty())
+      return;
+
+    DEBUG(llvm::dbgs() << "** SILDebugInfoGenerator **\n");
+
+    std::vector<SILFunction *> PrintedFuncs;
+    int FileIdx = 0;
+    auto FIter = M->begin();
+    while (FIter != M->end()) {
+
+      std::string FileName;
+      llvm::raw_string_ostream NameOS(FileName);
+      NameOS << FileBaseName << ".gsil_" << FileIdx++ << ".sil";
+      NameOS.flush();
+
+      char *FileNameBuf = (char *)M->allocate(FileName.size() + 1, 1);
+      strcpy(FileNameBuf, FileName.c_str());
+
+      DEBUG(llvm::dbgs() << "Write debug SIL file " << FileName << '\n');
+
+      std::error_code EC;
+      llvm::raw_fd_ostream OutFile(FileName, EC,
+                                   llvm::sys::fs::OpenFlags::F_None);
+      assert(!OutFile.has_error() && !EC && "Can't write SIL debug file");
+
+      PrintContext Ctx(OutFile);
+
+      // Write functions until we reach the LineLimitPerFile.
+      do {
+        SILFunction *F = &*FIter++;
+        PrintedFuncs.push_back(F);
+
+        // Set the debug scope for the function.
+        SILLocation::DebugLoc DL(Ctx.LCS.LineNum, 1, FileNameBuf);
+        RegularLocation Loc(DL);
+        SILDebugScope *Scope = new (*M) SILDebugScope(Loc, F);
+        F->setDebugScope(Scope);
+
+        // Ensure that the function is visible for debugging.
+        F->setBare(IsNotBare);
+
+        // Print it to the output file.
+        F->print(Ctx);
+      } while (FIter != M->end() && Ctx.LCS.LineNum < LineLimitPerFile);
+
+      // Set the debug locations of all instructions.
+      for (SILFunction *F : PrintedFuncs) {
+        const SILDebugScope *Scope = F->getDebugScope();
+        for (SILBasicBlock &BB : *F) {
+          for (SILInstruction &I : BB) {
+            SILLocation Loc = I.getLoc();
+            SILLocation::DebugLoc DL(Ctx.LineNums[&I], 1, FileNameBuf);
+            assert(DL.Line && "no line set for instruction");
+            if (Loc.is<ReturnLocation>() || Loc.is<ImplicitReturnLocation>()) {
+              Loc.setDebugInfoLoc(DL);
+              I.setDebugLocation(SILDebugLocation(Loc, Scope));
+            } else {
+              RegularLocation RLoc(DL);
+              I.setDebugLocation(SILDebugLocation(RLoc, Scope));
+            }
+          }
+        }
+      }
+      PrintedFuncs.clear();
+    }
+  }
+
+  StringRef getName() override { return "SILDebugInfoGenerator"; }
+};
+
+} // end anonymous namespace
+
+SILTransform *swift::createSILDebugInfoGenerator() {
+  return new SILDebugInfoGenerator();
+}

test/DebugInfo/gsil.swift

@@ -0,0 +1,13 @@
+// RUN: rm -rf %t && mkdir %t
+// RUN: %target-swift-frontend %s -O -gsil -emit-ir -o %t/out.ir
+// RUN: FileCheck %s < %t/out.ir
+// RUN: FileCheck %s --check-prefix=CHECK_OUT_SIL < %t/out.ir.gsil_0.sil
+
+// CHECK: [[F:![0-9]+]] = !DIFile(filename: "out.ir.gsil_0.sil", directory: "{{.+}}")
+// CHECK: !DISubprogram(linkageName: "_TF3out6testitFT_T_", scope: !{{[0-9]+}}, file: [[F]], line: {{[1-9][0-9]+}},
+
+// CHECK_OUT_SIL: sil @_TF3out6testitFT_T_ : $@convention(thin) () -> () {
+public func testit() {
+  print("Hello")
+}
+