[Driver] Write out a record of which files were compiled in a build.

This is important because we might get part-way through the full
compilation, overwriting swiftdeps files as we go, and then encounter an
error. We don't want to lose information about any decls that have been
removed since the previous compile, so we propagate forward the information
we already have by saving it to a "build record" file.

More simply, this is necessary to track when a file is removed from a target.

The next commit will handle reading in this file at the start of a build.

Swift SVN r23968

include/swift/Driver/Compilation.h

@@ -80,6 +80,11 @@ class Compilation {
   /// These apply whether the compilation succeeds or fails.
   std::vector<std::string> TempFilePaths;
 
+  /// Write information about this compilation to this file.
+  ///
+  /// This is used for incremental builds.
+  std::string CompilationRecordPath;
+
   /// The number of commands which this compilation should attempt to run in
   /// parallel.
   unsigned NumberOfParallelCommands;
@@ -130,6 +135,11 @@ public:
     EnableIncrementalBuild = false;
   }
 
+  void setCompilationRecordPath(StringRef path) {
+    assert(CompilationRecordPath.empty() && "already set");
+    CompilationRecordPath = path;
+  }
+
   /// Asks the Compilation to perform the Jobs which it knows about.
   /// \returns result code for the Compilation's Jobs; 0 indicates success and
   /// -2 indicates that one of the Compilation's Jobs crashed during execution

lib/Driver/Compilation.cpp

@@ -30,6 +30,7 @@
 #include "llvm/Option/ArgList.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/YAMLParser.h"
 
 using namespace swift;
 using namespace swift::sys;
@@ -67,6 +68,12 @@ struct Compilation::PerformJobsState {
   ///
   /// The blocked jobs should be scheduled as soon as possible.
   llvm::DenseMap<const Job *, TinyPtrVector<const Job *>> BlockingCommands;
+
+  /// A map from commands that didn't get to run to whether or not they affect
+  /// downstream commands.
+  ///
+  /// Only intended for source files.
+  llvm::DenseMap<const Job *, bool> UnfinishedCommands;
 };
 
 Compilation::~Compilation() = default;
@@ -324,6 +331,26 @@ int Compilation::performJobsInList(const JobList &JL, PerformJobsState &State) {
   if (Result == 0) {
     assert(State.BlockingCommands.empty() &&
            "some blocking commands never finished properly");
+  } else {
+    // Make sure we record any files that haven't been touched but nonetheless
+    // need to be rebuilt.
+    for (const Job *Cmd : JL) {
+      switch (Cmd->getCondition()) {
+      case Job::Condition::Always:
+        // Don't worry about files that would be rebuilt anyway.
+        continue;
+      case Job::Condition::CheckDependencies:
+        break;
+      }
+
+      // Don't worry about commands that finished or weren't going to run.
+      if (State.FinishedCommands.count(Cmd))
+        continue;
+      if (!State.ScheduledCommands.count(Cmd))
+        continue;
+
+      State.UnfinishedCommands.insert({Cmd, DepGraph.isMarked(Cmd)});
+    }
   }
 
   return Result;
@@ -364,6 +391,44 @@ int Compilation::performSingleCommand(const Job *Cmd) {
   return ExecuteInPlace(ExecPath, argv);
 }
 
+static void writeCompilationRecord(
+    StringRef path,
+    const llvm::DenseMap<const Job *, bool> &unfinishedCommands,
+    const llvm::opt::ArgList &args) {
+  llvm::DenseMap<const llvm::opt::Arg *, bool> unfinishedInputs;
+  for (auto &entry : unfinishedCommands) {
+    ArrayRef<Action *> actionInputs = entry.first->getSource().getInputs();
+    assert(actionInputs.size() == 1);
+    auto inputFile = cast<InputAction>(actionInputs.front());
+    unfinishedInputs[&inputFile->getInputArg()] = entry.second;
+  }
+
+  std::error_code error;
+  llvm::raw_fd_ostream out(path, error, llvm::sys::fs::F_None);
+  if (out.has_error()) {
+    // FIXME: How should we report this error?
+    out.clear_error();
+    return;
+  }
+
+  for (auto &inputArg : args) {
+    if (inputArg->getOption().getKind() != llvm::opt::Option::InputClass)
+      continue;
+    out << "- ";
+
+    auto unfinishedIter = unfinishedInputs.find(inputArg);
+    if (unfinishedIter == unfinishedInputs.end()) {
+      /* do nothing */
+    } else if (unfinishedIter->second) {
+      out << "!dirty ";
+    } else {
+      out << "!private ";
+    }
+
+    out << "\"" << llvm::yaml::escape(inputArg->getValue()) << "\"\n";
+  }
+}
+
 int Compilation::performJobs() {
   // We require buffered output if Parseable output was requested.
   bool RequiresBufferedOutput = (Level == OutputLevel::Parseable);
@@ -379,6 +444,11 @@ int Compilation::performJobs() {
   PerformJobsState State;
   int result = performJobsInList(*Jobs, State);
 
+  if (!CompilationRecordPath.empty()) {
+    writeCompilationRecord(CompilationRecordPath, State.UnfinishedCommands,
+                           getArgs());
+  }
+
   // FIXME: Do we want to be deleting temporaries even when a child process
   // crashes?
   for (auto &path : TempFilePaths) {

lib/Driver/Driver.cpp

@@ -234,6 +234,10 @@ std::unique_ptr<Compilation> Driver::buildCompilation(
 
   buildJobs(Actions, OI, OFM.get(), *C);
 
+  if (OFM)
+    if (auto *masterOutputMap = OFM->getOutputMapForSingleOutput())
+      C->setCompilationRecordPath(masterOutputMap->lookup(types::TY_SwiftDeps));
+
   if (Diags.hadAnyError())
     return nullptr;
 

test/Driver/Dependencies/Inputs/fail-chained/a.swift

@@ -0,0 +1,2 @@
+# Dependencies after compilation:
+provides: [a]

test/Driver/Dependencies/Inputs/fail-chained/b.swift

@@ -0,0 +1,2 @@
+# Dependencies after compilation:
+provides: [b]

test/Driver/Dependencies/Inputs/fail-chained/bad.swift

@@ -0,0 +1,3 @@
+# Dependencies after compilation:
+provides: [bad]
+top-level: [a, !private b]

test/Driver/Dependencies/Inputs/fail-chained/c.swift

@@ -0,0 +1,3 @@
+# Dependencies after compilation:
+provides: [c]
+top-level: [bad]

test/Driver/Dependencies/Inputs/fail-chained/d.swift

@@ -0,0 +1,3 @@
+# Dependencies after compilation:
+provides: [d]
+top-level: [c]

test/Driver/Dependencies/Inputs/fail-chained/e.swift

@@ -0,0 +1,3 @@
+# Dependencies after compilation:
+provides: [e]
+top-level: [!private bad]

test/Driver/Dependencies/Inputs/fail-chained/f.swift

@@ -0,0 +1,3 @@
+# Dependencies after compilation:
+provides: [f]
+top-level: [e]

test/Driver/Dependencies/Inputs/fail-chained/output.json

@@ -0,0 +1,33 @@
+{
+  "./a.swift": {
+    "object": "./a.o",
+    "swift-dependencies": "./a.swiftdeps"
+  },
+  "./b.swift": {
+    "object": "./b.o",
+    "swift-dependencies": "./b.swiftdeps"
+  },
+  "./c.swift": {
+    "object": "./c.o",
+    "swift-dependencies": "./c.swiftdeps"
+  },
+  "./d.swift": {
+    "object": "./d.o",
+    "swift-dependencies": "./d.swiftdeps"
+  },
+  "./e.swift": {
+    "object": "./e.o",
+    "swift-dependencies": "./e.swiftdeps"
+  },
+  "./f.swift": {
+    "object": "./f.o",
+    "swift-dependencies": "./f.swiftdeps"
+  },
+  "./bad.swift": {
+    "object": "./bad.o",
+    "swift-dependencies": "./bad.swiftdeps"
+  },
+  "": {
+    "swift-dependencies": "./main~buildrecord.swiftdeps"
+  }  
+}

test/Driver/Dependencies/Inputs/fail-simple/bad.swift

@@ -0,0 +1,2 @@
+# Dependencies after compilation:
+provides: [a]

test/Driver/Dependencies/Inputs/fail-simple/main.swift

@@ -0,0 +1,2 @@
+# Dependencies after compilation:
+top-level: [a]

test/Driver/Dependencies/Inputs/fail-simple/other.swift

@@ -0,0 +1,2 @@
+# Dependencies after compilation:
+top-level: [!private a]

test/Driver/Dependencies/Inputs/fail-simple/output.json

@@ -0,0 +1,17 @@
+{
+  "./main.swift": {
+    "object": "./main.o",
+    "swift-dependencies": "./main.swiftdeps"
+  },
+  "./bad.swift": {
+    "object": "./bad.o",
+    "swift-dependencies": "./bad.swiftdeps"
+  },
+  "./other.swift": {
+    "object": "./other.o",
+    "swift-dependencies": "./other.swiftdeps"
+  },
+  "": {
+    "swift-dependencies": "./main~buildrecord.swiftdeps"
+  }  
+}

test/Driver/Dependencies/Inputs/update-dependencies-bad.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+# Fails if the input file is named "bad.swift"; otherwise dispatches to
+# update-dependencies.py.
+
+import os
+import sys
+
+assert sys.argv[1] == '-frontend'
+
+primaryFile = sys.argv[sys.argv.index('-primary-file') + 1]
+
+if os.path.basename(primaryFile) == 'bad.swift':
+    print "Handled", os.path.basename(primaryFile)
+    exit(1)
+
+dir = os.path.dirname(os.path.abspath(__file__))
+execfile(os.path.join(dir, "update-dependencies.py"))

test/Driver/Dependencies/Inputs/update-dependencies.py

@@ -12,9 +12,9 @@
 # actually stick *the new dependencies* in the input file, and copy that over
 # the old dependencies (if present).
 
-import sys
 import os
 import shutil
+import sys
 
 assert sys.argv[1] == '-frontend'
 

test/Driver/Dependencies/fail-chained.swift

@@ -0,0 +1,92 @@
+/// a ==> bad ==> c ==> d | b --> bad --> e ==> f
+
+// RUN: rm -rf %t && cp -r %S/Inputs/fail-chained/ %t
+// RUN: touch -t 201401240005 %t/*
+
+// RUN: cd %t && %swiftc_driver -c -driver-use-frontend-path %S/Inputs/update-dependencies.py -output-file-map %t/output.json -incremental ./a.swift ./b.swift ./c.swift ./d.swift ./e.swift ./f.swift ./bad.swift -module-name main -j1 -v 2>&1 | FileCheck -check-prefix=CHECK-FIRST %s
+
+// CHECK-FIRST: Handled a.swift
+// CHECK-FIRST: Handled b.swift
+// CHECK-FIRST: Handled c.swift
+// CHECK-FIRST: Handled d.swift
+// CHECK-FIRST: Handled e.swift
+// CHECK-FIRST: Handled f.swift
+// CHECK-FIRST: Handled bad.swift
+
+// RUN: rm %t/a.o
+// RUN: cd %t && not %swiftc_driver -c -driver-use-frontend-path %S/Inputs/update-dependencies-bad.py -output-file-map %t/output.json -incremental ./a.swift ./b.swift ./c.swift ./d.swift ./e.swift ./f.swift ./bad.swift -module-name main -j1 -v > %t/a.txt 2>&1
+// RUN: FileCheck -check-prefix=CHECK-A %s < %t/a.txt
+// RUN: FileCheck -check-prefix=NEGATIVE-A %s < %t/a.txt
+// RUN: FileCheck -check-prefix=CHECK-RECORD-A %s < %t/main~buildrecord.swiftdeps
+
+// CHECK-A: Handled a.swift
+// CHECK-A: Handled bad.swift
+// NEGATIVE-A-NOT: Handled b.swift
+// NEGATIVE-A-NOT: Handled c.swift
+// NEGATIVE-A-NOT: Handled d.swift
+// NEGATIVE-A-NOT: Handled e.swift
+// NEGATIVE-A-NOT: Handled f.swift
+
+// CHECK-RECORD-A-DAG: - "./a.swift"
+// CHECK-RECORD-A-DAG: - "./b.swift"
+// CHECK-RECORD-A-DAG: - !dirty "./c.swift"
+// CHECK-RECORD-A-DAG: - !dirty "./d.swift"
+// CHECK-RECORD-A-DAG: - !private "./e.swift"
+// CHECK-RECORD-A-DAG: - "./f.swift"
+// CHECK-RECORD-A-DAG: - !dirty "./bad.swift"
+
+
+// RUN: rm -rf %t && cp -r %S/Inputs/fail-chained/ %t
+// RUN: touch -t 201401240005 %t/*
+
+// RUN: cd %t && %swiftc_driver -c -driver-use-frontend-path %S/Inputs/update-dependencies.py -output-file-map %t/output.json -incremental ./a.swift ./b.swift ./c.swift ./d.swift ./e.swift ./f.swift ./bad.swift -module-name main -j1 -v 2>&1 | FileCheck -check-prefix=CHECK-FIRST %s
+
+// RUN: rm %t/b.o
+// RUN: cd %t && not %swiftc_driver -c -driver-use-frontend-path %S/Inputs/update-dependencies-bad.py -output-file-map %t/output.json -incremental ./a.swift ./b.swift ./c.swift ./d.swift ./e.swift ./f.swift ./bad.swift -module-name main -j1 -v > %t/b.txt 2>&1
+// RUN: FileCheck -check-prefix=CHECK-B %s < %t/b.txt
+// RUN: FileCheck -check-prefix=NEGATIVE-B %s < %t/b.txt
+// RUN: FileCheck -check-prefix=CHECK-RECORD-B %s < %t/main~buildrecord.swiftdeps
+
+// CHECK-B: Handled b.swift
+// CHECK-B: Handled bad.swift
+// NEGATIVE-B-NOT: Handled a.swift
+// NEGATIVE-B-NOT: Handled c.swift
+// NEGATIVE-B-NOT: Handled d.swift
+// NEGATIVE-B-NOT: Handled e.swift
+// NEGATIVE-B-NOT: Handled f.swift
+
+// CHECK-RECORD-B-DAG: - "./a.swift"
+// CHECK-RECORD-B-DAG: - "./b.swift"
+// CHECK-RECORD-B-DAG: - "./c.swift"
+// CHECK-RECORD-B-DAG: - "./d.swift"
+// CHECK-RECORD-B-DAG: - "./e.swift"
+// CHECK-RECORD-B-DAG: - "./f.swift"
+// CHECK-RECORD-B-DAG: - !private "./bad.swift"
+
+
+// RUN: rm -rf %t && cp -r %S/Inputs/fail-chained/ %t
+// RUN: touch -t 201401240005 %t/*
+
+// RUN: cd %t && %swiftc_driver -c -driver-use-frontend-path %S/Inputs/update-dependencies.py -output-file-map %t/output.json -incremental ./a.swift ./b.swift ./c.swift ./d.swift ./e.swift ./f.swift ./bad.swift -module-name main -j1 -v 2>&1 | FileCheck -check-prefix=CHECK-FIRST %s
+
+// RUN: rm %t/bad.o
+// RUN: cd %t && not %swiftc_driver -c -driver-use-frontend-path %S/Inputs/update-dependencies-bad.py -output-file-map %t/output.json -incremental ./a.swift ./b.swift ./c.swift ./d.swift ./e.swift ./f.swift ./bad.swift -module-name main -j1 -v > %t/b.txt 2>&1
+// RUN: FileCheck -check-prefix=CHECK-BAD %s < %t/b.txt
+// RUN: FileCheck -check-prefix=NEGATIVE-BAD %s < %t/b.txt
+// RUN: FileCheck -check-prefix=CHECK-RECORD-BAD %s < %t/main~buildrecord.swiftdeps
+
+// CHECK-BAD: Handled bad.swift
+// NEGATIVE-BAD-NOT: Handled a.swift
+// NEGATIVE-BAD-NOT: Handled b.swift
+// NEGATIVE-BAD-NOT: Handled c.swift
+// NEGATIVE-BAD-NOT: Handled d.swift
+// NEGATIVE-BAD-NOT: Handled e.swift
+// NEGATIVE-BAD-NOT: Handled f.swift
+
+// CHECK-RECORD-BAD-DAG: - "./a.swift"
+// CHECK-RECORD-BAD-DAG: - "./b.swift"
+// CHECK-RECORD-BAD-DAG: - !dirty "./c.swift"
+// CHECK-RECORD-BAD-DAG: - !dirty "./d.swift"
+// CHECK-RECORD-BAD-DAG: - !private "./e.swift"
+// CHECK-RECORD-BAD-DAG: - "./f.swift"
+// CHECK-RECORD-BAD-DAG: - "./bad.swift"

test/Driver/Dependencies/fail-simple.swift

@@ -0,0 +1,22 @@
+/// bad ==> main | bad --> other
+
+// RUN: rm -rf %t && cp -r %S/Inputs/fail-simple/ %t
+// RUN: touch -t 201401240005 %t/*
+
+// RUN: cd %t && %swiftc_driver -c -driver-use-frontend-path %S/Inputs/update-dependencies.py -output-file-map %t/output.json -incremental ./main.swift ./bad.swift ./other.swift -module-name main -j1 -v 2>&1 | FileCheck -check-prefix=CHECK-FIRST %s
+
+// CHECK-FIRST: Handled main.swift
+// CHECK-FIRST: Handled bad.swift
+// CHECK-FIRST: Handled other.swift
+
+// RUN: rm %t/bad.o
+// RUN: cd %t && not %swiftc_driver -c -driver-use-frontend-path %S/Inputs/update-dependencies-bad.py -output-file-map %t/output.json -incremental ./main.swift ./bad.swift ./other.swift -module-name main -j1 -v 2>&1 | FileCheck -check-prefix=CHECK-SECOND %s
+// RUN: FileCheck -check-prefix=CHECK-RECORD %s < %t/main~buildrecord.swiftdeps
+
+// CHECK-SECOND: Handled bad.swift
+// CHECK-SECOND-NOT: Handled main.swift
+// CHECK-SECOND-NOT: Handled other.swift
+
+// CHECK-RECORD-DAG: - "./bad.swift"
+// CHECK-RECORD-DAG: - !dirty "./main.swift"
+// CHECK-RECORD-DAG: - !private "./other.swift"