diff --git a/include/swift/Basic/FileTypes.def b/include/swift/Basic/FileTypes.def index 895fac710fa..395e63173ac 100644 --- a/include/swift/Basic/FileTypes.def +++ b/include/swift/Basic/FileTypes.def @@ -61,6 +61,7 @@ TYPE("llvm-bc", LLVM_BC, "bc", "") TYPE("diagnostics", SerializedDiagnostics, "dia", "") TYPE("objc-header", ObjCHeader, "h", "") TYPE("swift-dependencies", SwiftDeps, "swiftdeps", "") +TYPE("external-swift-dependencies", ExternalSwiftDeps, "swiftdeps.external", "") TYPE("swift-ranges", SwiftRanges, "swiftranges", "") TYPE("compiled-source", CompiledSource, "compiledsource", "") TYPE("remap", Remapping, "remap", "") diff --git a/include/swift/Driver/Compilation.h b/include/swift/Driver/Compilation.h index c07537bcfe8..a68f0c2f0b3 100644 --- a/include/swift/Driver/Compilation.h +++ b/include/swift/Driver/Compilation.h @@ -508,6 +508,17 @@ public: /// \sa types::isPartOfSwiftCompilation const char *getAllSourcesPath() const; + /// Retrieve the path to the external swift deps file. + /// + /// For cross-module incremental builds, this file contains the dependencies + /// from all the modules integrated over the prior build. + /// + /// Currently this patch is relative to the build record, but we may want + /// to allow the output file map to customize this at some point. + std::string getExternalSwiftDepsFilePath() const { + return CompilationRecordPath + ".external"; + } + /// Asks the Compilation to perform the Jobs which it knows about. /// /// \param TQ The TaskQueue used to schedule jobs for execution. diff --git a/lib/Basic/FileTypes.cpp b/lib/Basic/FileTypes.cpp index 2c39c852d03..7320ae94d1b 100644 --- a/lib/Basic/FileTypes.cpp +++ b/lib/Basic/FileTypes.cpp @@ -100,6 +100,7 @@ bool file_types::isTextual(ID Id) { case file_types::TY_SerializedDiagnostics: case file_types::TY_ClangModuleFile: case file_types::TY_SwiftDeps: + case file_types::TY_ExternalSwiftDeps: case file_types::TY_SwiftRanges: case file_types::TY_CompiledSource: case file_types::TY_Nothing: @@ -145,6 +146,7 @@ bool file_types::isAfterLLVM(ID Id) { case file_types::TY_SerializedDiagnostics: case file_types::TY_ClangModuleFile: case file_types::TY_SwiftDeps: + case file_types::TY_ExternalSwiftDeps: case file_types::TY_SwiftRanges: case file_types::TY_CompiledSource: case file_types::TY_Nothing: @@ -197,6 +199,7 @@ bool file_types::isPartOfSwiftCompilation(ID Id) { case file_types::TY_SerializedDiagnostics: case file_types::TY_ClangModuleFile: case file_types::TY_SwiftDeps: + case file_types::TY_ExternalSwiftDeps: case file_types::TY_SwiftRanges: case file_types::TY_CompiledSource: case file_types::TY_Nothing: diff --git a/lib/Driver/Compilation.cpp b/lib/Driver/Compilation.cpp index 79d7e9584eb..a99a244f29d 100644 --- a/lib/Driver/Compilation.cpp +++ b/lib/Driver/Compilation.cpp @@ -15,6 +15,7 @@ #include "swift/AST/DiagnosticEngine.h" #include "swift/AST/DiagnosticsDriver.h" #include "swift/AST/FineGrainedDependencies.h" +#include "swift/AST/FineGrainedDependencyFormat.h" #include "swift/Basic/OutputFileMap.h" #include "swift/Basic/Program.h" #include "swift/Basic/STLExtras.h" @@ -1185,6 +1186,30 @@ namespace driver { return ExternallyDependentJobs; } + using ChangeSet = fine_grained_dependencies::ModuleDepGraph::Changes::value_type; + static void + pruneChangeSetFromExternalDependency(ChangeSet &changes) { + // The changeset includes detritus from the graph that gets consed up + // in \c writePriorDependencyGraph. We need to ignore the fake + // source file provides nodes and the fake incremental external + // dependencies linked to them. + swift::erase_if( + changes, [&](fine_grained_dependencies::ModuleDepGraphNode *node) { + if (node->getKey().getKind() == + fine_grained_dependencies::NodeKind::sourceFileProvide || + node->getKey().getKind() == + fine_grained_dependencies::NodeKind:: + incrementalExternalDepend) { + return true; + } + if (node->getKey().getAspect() == + fine_grained_dependencies::DeclAspect::implementation) { + return true; + } + return !node->getIsProvides(); + }); + } + SmallVector collectIncrementalExternallyDependentJobsFromDependencyGraph( const bool forRanges) { @@ -1196,6 +1221,17 @@ namespace driver { } }; + // Load our priors, which are always adjacent to the build record. We + // don't care if this load succeeds or not. If it fails, and we succeed at + // integrating one of the external files below, the changeset will be the + // entire module! + const auto *externalPriorJob = Comp.addExternalJob( + std::make_unique(Comp.getDerivedOutputFileMap(), + Comp.getExternalSwiftDepsFilePath())); + getFineGrainedDepGraph(forRanges).loadFromPath( + externalPriorJob, Comp.getExternalSwiftDepsFilePath(), + Comp.getDiags()); + for (auto external : getFineGrainedDepGraph(forRanges) .getIncrementalExternalDependencies()) { llvm::sys::fs::file_status depStatus; @@ -1231,20 +1267,23 @@ namespace driver { // code's internal invariants. const auto *externalJob = Comp.addExternalJob( std::make_unique(Comp.getDerivedOutputFileMap(), external)); - auto subChanges = + auto maybeChanges = getFineGrainedDepGraph(forRanges).loadFromSwiftModuleBuffer( externalJob, *buffer.get(), Comp.getDiags()); // If the incremental dependency graph failed to load, fall back to // treating this as plain external job. - if (!subChanges.hasValue()) { + if (!maybeChanges.hasValue()) { fallbackToExternalBehavior(external); continue; } - for (auto *CMD : - getFineGrainedDepGraph(forRanges) - .findJobsToRecompileWhenNodesChange(subChanges.getValue())) { + // Prune away the detritus from the build record. + auto &changes = maybeChanges.getValue(); + pruneChangeSetFromExternalDependency(changes); + + for (auto *CMD : getFineGrainedDepGraph(forRanges) + .findJobsToRecompileWhenNodesChange(changes)) { if (CMD == externalJob) { continue; } @@ -1861,6 +1900,76 @@ static void writeCompilationRecord(StringRef path, StringRef argsHash, } } +using SourceFileDepGraph = swift::fine_grained_dependencies::SourceFileDepGraph; + +/// Render out the unified module dependency graph to the given \p path, which +/// is expected to be a path relative to the build record. +static void withPriorDependencyGraph(StringRef path, + const Compilation::Result &result, + llvm::function_ref cont) { + // Building a source file dependency graph from the module dependency graph + // is a strange task on its face because a source file dependency graph is + // usually built for exactly one file. However, the driver is going to use + // some encoding tricks to get the dependencies for each incremental external + // dependency into one big file. Note that these tricks + // are undone in \c pruneChangeSetFromExternalDependency, so if you modify + // this you need to go fix that algorithm up as well. This is a diagrammatic + // view of the structure of the dependencies this function builds: + // + // SourceFile => interface .external + // | - Incremetal External Dependency => interface .swiftmodule + // | | - ... + // | | - ... + // | | - ... + // | - Incremetal External Dependency => interface .swiftmodule + // | | - ... + // | | - ... + // | - Incremetal External Dependency => interface .swiftmodule + // | - ... + // + // Where each node has an arc back to its parent swiftmodule. + // That swiftmodule, in turn, takes the form of as an incremental external + // dependency. This formulation allows us to easily discern the original + // swiftmodule that a came from just by examining that arc. This + // is used in integrate to "move" the from the build record to + // the swiftmodule by swapping the key it uses. + using namespace swift::fine_grained_dependencies; + SourceFileDepGraph g; + const auto &resultModuleGraph = result.depGraph; + // Create the key for the entire external build record. + auto fileKey = + DependencyKey::createKeyForWholeSourceFile(DeclAspect::interface, path); + auto fileNodePair = g.findExistingNodePairOrCreateAndAddIfNew(fileKey, None); + for (StringRef incrExternalDep : + resultModuleGraph.getIncrementalExternalDependencies()) { + // Now make a node for each incremental external dependency. + auto interfaceKey = + DependencyKey(NodeKind::incrementalExternalDepend, + DeclAspect::interface, "", incrExternalDep.str()); + auto ifaceNode = g.findExistingNodeOrCreateIfNew(interfaceKey, None, + false /* = !isProvides */); + resultModuleGraph.forEachNodeInJob(incrExternalDep, [&](const auto *node) { + // Reject + // 1) Implementation nodes: We don't care about the interface nodes + // for cross-module dependencies because the client cannot observe it + // by definition. + // 2) Source file nodes: we're about to define our own. + if (!node->getKey().isInterface() || + node->getKey().getKind() == NodeKind::sourceFileProvide) { + return; + } + assert(node->getIsProvides() && + "Found a node in module depdendencies that is not a provides!"); + auto *newNode = new SourceFileDepGraphNode( + node->getKey(), node->getFingerprint(), /*isProvides*/ true); + g.addNode(newNode); + g.addArc(ifaceNode, newNode); + }); + g.addArc(fileNodePair.getInterface(), ifaceNode); + } + return cont(std::move(g)); +} + static void writeInputJobsToFilelist(llvm::raw_fd_ostream &out, const Job *job, const file_types::ID infoType) { // FIXME: Duplicated from ToolChains.cpp. @@ -1960,6 +2069,15 @@ Compilation::performJobsImpl(std::unique_ptr &&TQ) { auto result = std::move(State).takeResult(); writeCompilationRecord(CompilationRecordPath, ArgsHash, BuildStartTime, InputInfo); + if (EnableCrossModuleIncrementalBuild) { + // Write out our priors adjacent to the build record so we can pick + // the up in a subsequent build. + withPriorDependencyGraph(getExternalSwiftDepsFilePath(), result, + [&](SourceFileDepGraph &&g) { + writeFineGrainedDependencyGraphToPath( + Diags, getExternalSwiftDepsFilePath(), g); + }); + } return result; } else { return std::move(State).takeResult(); diff --git a/lib/Driver/Driver.cpp b/lib/Driver/Driver.cpp index aeb01bfc8eb..c6d8d94a63c 100644 --- a/lib/Driver/Driver.cpp +++ b/lib/Driver/Driver.cpp @@ -2039,6 +2039,7 @@ void Driver::buildActions(SmallVectorImpl &TopLevelActions, case file_types::TY_ObjCHeader: case file_types::TY_ClangModuleFile: case file_types::TY_SwiftDeps: + case file_types::TY_ExternalSwiftDeps: case file_types::TY_SwiftRanges: case file_types::TY_CompiledSource: case file_types::TY_Remapping: diff --git a/lib/Driver/FineGrainedDependencyDriverGraph.cpp b/lib/Driver/FineGrainedDependencyDriverGraph.cpp index 7d1b2d9eea3..5f356aa56a6 100644 --- a/lib/Driver/FineGrainedDependencyDriverGraph.cpp +++ b/lib/Driver/FineGrainedDependencyDriverGraph.cpp @@ -287,15 +287,29 @@ ModuleDepGraph::Changes ModuleDepGraph::integrate(const SourceFileDepGraph &g, g.forEachNode([&](const SourceFileDepGraphNode *integrand) { const auto &key = integrand->getKey(); - - auto preexistingMatch = findPreexistingMatch(swiftDepsOfJob, integrand); + StringRef realSwiftDepsPath = swiftDepsOfJob; + // If we're doing a cross-module incremental build, we'll see these + // `.external` "swiftdeps" files. See \c writePriorDependencyGraph for + // the structure of the graph we're traversing here. Essentially, follow + // the arc laid down there to discover the file path for the swiftmodule + // where this dependency node originally came from. + if (swiftDepsOfJob.endswith(file_types::getExtension(file_types::TY_ExternalSwiftDeps)) && + integrand->getKey().getKind() != NodeKind::sourceFileProvide) { + integrand->forEachDefIDependUpon([&](size_t seqNum) { + auto &external = g.getNode(seqNum)->getKey(); + if (external.getKind() == NodeKind::incrementalExternalDepend) { + realSwiftDepsPath = external.getName(); + } + }); + } + auto preexistingMatch = findPreexistingMatch(realSwiftDepsPath, integrand); if (preexistingMatch.hasValue() && preexistingMatch.getValue().first == LocationOfPreexistingNode::here) disappearedNodes.erase(key); // Node was and still is. Do not erase it. Optional> newNodeOrChangedNode = integrateSourceFileDepGraphNode(g, integrand, preexistingMatch, - swiftDepsOfJob); + realSwiftDepsPath); if (!newNodeOrChangedNode) changedNodes = None; diff --git a/lib/Driver/ToolChains.cpp b/lib/Driver/ToolChains.cpp index 376b039e766..22e87dbcec1 100644 --- a/lib/Driver/ToolChains.cpp +++ b/lib/Driver/ToolChains.cpp @@ -619,6 +619,7 @@ const char *ToolChain::JobContext::computeFrontendModeForCompile() const { case file_types::TY_ObjCHeader: case file_types::TY_Image: case file_types::TY_SwiftDeps: + case file_types::TY_ExternalSwiftDeps: case file_types::TY_SwiftRanges: case file_types::TY_CompiledSource: case file_types::TY_ModuleTrace: @@ -881,6 +882,7 @@ ToolChain::constructInvocation(const BackendJobAction &job, case file_types::TY_ObjCHeader: case file_types::TY_Image: case file_types::TY_SwiftDeps: + case file_types::TY_ExternalSwiftDeps: case file_types::TY_SwiftRanges: case file_types::TY_CompiledSource: case file_types::TY_Remapping: