[Dependency Scanning] Add Link Libraries to the dependency scanner cache serialization format

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -73,6 +73,7 @@ using FileIDArrayIDField = IdentifierIDField;
 using ContextHashIDField = IdentifierIDField;
 using ModuleCacheKeyIDField = IdentifierIDField;
 using ImportArrayIDField = IdentifierIDField;
+using LinkLibrariesArrayIDField = IdentifierIDField;
 using FlagIDArrayIDField = IdentifierIDField;
 using DependencyIDArrayIDField = IdentifierIDField;
 using AuxiliaryFilesArrayIDField = IdentifierIDField;
@@ -85,12 +86,15 @@ const unsigned GRAPH_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID;
 /// zero or more IDENTIFIER records that contain various strings seen in the graph
 /// (e.g. file names or compiler flags), followed by zero or more IDENTIFIER_ARRAY records
 /// which are arrays of identifiers seen in the graph (e.g. list of source files or list of compile flags),
+/// followed by zero or more LINK_LIBRARY_NODE records along with associated
+///
 /// followed by zero or more MODULE_NODE, *_DETAILS_NODE pairs of records.
 namespace graph_block {
 enum {
   METADATA = 1,
   MODULE_NODE,
   LINK_LIBRARY_NODE,
+  LINK_LIBRARY_ARRAY_NODE,
   SOURCE_LOCATION_NODE,
   IMPORT_STATEMENT_NODE,
   SWIFT_INTERFACE_MODULE_DETAILS_NODE,
@@ -129,6 +133,9 @@ using IdentifierNodeLayout = BCRecordLayout<IDENTIFIER_NODE, BCBlob>;
 using IdentifierArrayLayout =
     BCRecordLayout<IDENTIFIER_ARRAY_NODE, IdentifierIDArryField>;
 
+using LinkLibraryArrayLayout =
+    BCRecordLayout<LINK_LIBRARY_ARRAY_NODE, IdentifierIDArryField>;
+
 using LinkLibraryLayout =
     BCRecordLayout<LINK_LIBRARY_NODE,            // ID
                    IdentifierIDField,            // libraryName
@@ -159,10 +166,9 @@ using ImportStatementLayout =
 using ModuleInfoLayout =
     BCRecordLayout<MODULE_NODE,                    // ID
                    IdentifierIDField,              // moduleName
-                   ContextHashIDField,           // contextHash
                    ImportArrayIDField,             // moduleImports
                    ImportArrayIDField,             // optionalModuleImports
-                   // ACTODO: LinkLibrariesArrayIDField,           // linkLibraries
+                   LinkLibrariesArrayIDField,      // linkLibraries
                    DependencyIDArrayIDField,       // importedSwiftModules
                    DependencyIDArrayIDField,       // importedClangModules
                    DependencyIDArrayIDField,       // crossImportOverlayModules
@@ -211,6 +217,7 @@ using SwiftBinaryModuleDetailsLayout =
                    FileIDField,                      // moduleDocPath
                    FileIDField,                      // moduleSourceInfoPath
                    FileIDField,                      // headerImport
+                   FileIDField,                      // definingInterfacePath
                    IdentifierIDField,                // headerModuleDependencies
                    FileIDArrayIDField,               // headerSourceFiles
                    IsFrameworkField,                 // isFramework

lib/AST/ModuleDependencies.cpp

@@ -44,7 +44,8 @@ bool ModuleDependencyInfo::isTextualSwiftModule() const {
   return isSwiftInterfaceModule() || isSwiftSourceModule();
 }
 
-ModuleDependencyKind &operator++(ModuleDependencyKind &e) {
+namespace {
+  ModuleDependencyKind &operator++(ModuleDependencyKind &e) {
     if (e == ModuleDependencyKind::LastKind) {
       llvm_unreachable(
                        "Attempting to increment last enum value on ModuleDependencyKind");
@@ -52,8 +53,8 @@ ModuleDependencyKind &operator++(ModuleDependencyKind &e) {
     e = ModuleDependencyKind(
                              static_cast<std::underlying_type<ModuleDependencyKind>::type>(e) + 1);
     return e;
+  }
 }
-
 bool ModuleDependencyInfo::isSwiftInterfaceModule() const {
   return isa<SwiftInterfaceModuleDependenciesStorage>(storage.get());
 }

lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

@@ -24,12 +24,26 @@ using namespace swift;
 using namespace dependencies;
 using namespace module_dependency_cache_serialization;
 
+namespace {
+  ModuleDependencyKind &operator++(ModuleDependencyKind &e) {
+    if (e == ModuleDependencyKind::LastKind) {
+      llvm_unreachable(
+          "Attempting to increment last enum value on ModuleDependencyKind");
+    }
+    e = ModuleDependencyKind(
+        static_cast<std::underlying_type<ModuleDependencyKind>::type>(e) + 1);
+    return e;
+  }
+}
+
 // MARK: Deserialization
 namespace swift {
 
 class ModuleDependenciesCacheDeserializer {
   std::vector<std::string> Identifiers;
   std::vector<std::vector<uint64_t>> ArraysOfIdentifierIDs;
+  std::vector<LinkLibrary> LinkLibraries;
+  std::vector<std::vector<uint64_t>> ArraysOfLinkLibraryIDs;
   llvm::BitstreamCursor Cursor;
   SmallVector<uint64_t, 64> Scratch;
   StringRef BlobData;
@@ -38,10 +52,12 @@ class ModuleDependenciesCacheDeserializer {
   bool readSignature();
   bool enterGraphBlock();
   bool readMetadata();
+  bool readLinkLibraries();
   bool readGraph(ModuleDependenciesCache &cache);
 
   std::optional<std::string> getIdentifier(unsigned n);
   std::optional<std::vector<std::string>> getStringArray(unsigned n);
+  std::optional<std::vector<LinkLibrary>> getLinkLibraryArray(unsigned n);
   std::optional<std::vector<ModuleDependencyID>>
   getModuleDependencyIDArray(unsigned n);
 
@@ -155,7 +171,6 @@ bool ModuleDependenciesCacheDeserializer::readGraph(ModuleDependenciesCache &cac
 
   bool hasCurrentModule = false;
   std::string currentModuleName;
-  unsigned currentContextHashID;
   std::vector<ScannerImportStatementInfo> currentModuleImports;
   std::vector<ScannerImportStatementInfo> currentOptionalModuleImports;
 
@@ -163,6 +178,7 @@ bool ModuleDependenciesCacheDeserializer::readGraph(ModuleDependenciesCache &cac
   std::vector<ModuleDependencyID> importedClangDependenciesIDs;
   std::vector<ModuleDependencyID> crossImportOverlayDependenciesIDs;
   std::vector<ModuleDependencyID> swiftOverlayDependenciesIDs;
+  std::vector<LinkLibrary> linkLibraries;
   std::vector<std::string> auxiliaryFiles;
 
   while (!Cursor.AtEndOfStream()) {
@@ -208,18 +224,44 @@ bool ModuleDependenciesCacheDeserializer::readGraph(ModuleDependenciesCache &cac
       break;
     }
 
+    case LINK_LIBRARY_NODE: {
+      unsigned libraryIdentifierID;
+      bool isFramework, shouldForceLoad;
+      LinkLibraryLayout::readRecord(Scratch,
+                                    libraryIdentifierID,
+                                    isFramework, shouldForceLoad);
+      auto libraryIdentifier = getIdentifier(libraryIdentifierID);
+      if (!libraryIdentifier)
+        llvm::report_fatal_error("Bad link library identifier");
+
+      LinkLibraries.push_back(LinkLibrary(libraryIdentifier.value(),
+                                          isFramework ? LibraryKind::Framework : LibraryKind::Library,
+                                          shouldForceLoad));
+      break;
+    }
+
+    case LINK_LIBRARY_ARRAY_NODE: {
+      ArrayRef<uint64_t> identifierIDs;
+      LinkLibraryArrayLayout::readRecord(Scratch, identifierIDs);
+      ArraysOfLinkLibraryIDs.push_back(identifierIDs.vec());
+      break;
+    }
+
     case MODULE_NODE: {
       hasCurrentModule = true;
-      unsigned moduleNameID, contextHashID,
+      unsigned moduleNameID,
                moduleImportsArrayID, optionalModuleImportsArrayID,
+               linkLibraryArrayID,
                importedSwiftDependenciesIDsArrayID,
                importedClangDependenciesIDsArrayID,
                crossImportOverlayDependenciesIDsArrayID,
                swiftOverlayDependenciesIDsArrayID,
                moduleCacheKeyID, AuxiliaryFilesArrayID;
-      ModuleInfoLayout::readRecord(Scratch, moduleNameID, contextHashID,
+
+      ModuleInfoLayout::readRecord(Scratch, moduleNameID,
                                    moduleImportsArrayID,
                                    optionalModuleImportsArrayID,
+                                   linkLibraryArrayID,
                                    importedSwiftDependenciesIDsArrayID,
                                    importedClangDependenciesIDsArrayID,
                                    crossImportOverlayDependenciesIDsArrayID,
@@ -229,10 +271,11 @@ bool ModuleDependenciesCacheDeserializer::readGraph(ModuleDependenciesCache &cac
       if (!moduleName)
         llvm::report_fatal_error("Bad module name");
       currentModuleName = *moduleName;
-      currentContextHashID = contextHashID;
-
       auto importStrings = getStringArray(moduleImportsArrayID);
       auto optionalImportStrings = getStringArray(optionalModuleImportsArrayID);
+      // ACTODO: Proper import infos.
+      currentModuleImports.clear();
+      currentOptionalModuleImports.clear();
       if (importStrings.has_value())
         for (const auto &is : importStrings.value())
           currentModuleImports.push_back(is);
@@ -264,6 +307,11 @@ bool ModuleDependenciesCacheDeserializer::readGraph(ModuleDependenciesCache &cac
       if (!optionalSwiftOverlayDependenciesIDs)
         llvm::report_fatal_error("Bad Swift Overlay dependencies: no qualified dependencies");
       swiftOverlayDependenciesIDs = optionalSwiftOverlayDependenciesIDs.value();
+
+      auto optionalLinkLibraries = getLinkLibraryArray(linkLibraryArrayID);
+      if (!optionalLinkLibraries)
+        llvm::report_fatal_error("Bad Link Libraries info");
+      linkLibraries = *optionalLinkLibraries;
       break;
     }
 
@@ -330,11 +378,11 @@ bool ModuleDependenciesCacheDeserializer::readGraph(ModuleDependenciesCache &cac
       if (!userModuleVersion)
         llvm::report_fatal_error("Bad userModuleVersion");
 
-      // TODO: LinkLibraries, MacroDependencies
+      // TODO: MacroDependencies
       // Form the dependencies storage object
       auto moduleDep = ModuleDependencyInfo::forSwiftInterfaceModule(
           outputModulePath.value(), optionalSwiftInterfaceFile.value(),
-          compiledCandidatesRefs, buildCommandRefs, {}, extraPCMRefs,
+          compiledCandidatesRefs, buildCommandRefs, linkLibraries, extraPCMRefs,
           *contextHash, isFramework, isStatic, *rootFileSystemID, *moduleCacheKey,
           *userModuleVersion);
 
@@ -403,11 +451,6 @@ bool ModuleDependenciesCacheDeserializer::readGraph(ModuleDependenciesCache &cac
       if (!hasCurrentModule)
         llvm::report_fatal_error(
             "Unexpected SWIFT_SOURCE_MODULE_DETAILS_NODE record");
-      // Expected context hash ID is 0
-      if (currentContextHashID)
-        llvm::report_fatal_error(
-            "Unexpected context hash on MODULE_NODE corresponding to a "
-            "SWIFT_SOURCE_MODULE_DETAILS_NODE record");
       unsigned extraPCMArgsArrayID, bridgingHeaderFileID, sourceFilesArrayID,
           bridgingSourceFilesArrayID, bridgingModuleDependenciesArrayID,
           CASFileSystemRootID,
@@ -514,16 +557,15 @@ bool ModuleDependenciesCacheDeserializer::readGraph(ModuleDependenciesCache &cac
         llvm::report_fatal_error(
             "Unexpected SWIFT_BINARY_MODULE_DETAILS_NODE record");
       unsigned compiledModulePathID, moduleDocPathID, moduleSourceInfoPathID,
-               headerImportID,
+               headerImportID, definingInterfacePathID,
                headerModuleDependenciesArrayID,
                headerImportsSourceFilesArrayID, isFramework, isStatic,
                moduleCacheKeyID, userModuleVersionID;
       SwiftBinaryModuleDetailsLayout::readRecord(
           Scratch, compiledModulePathID, moduleDocPathID,
-          moduleSourceInfoPathID,
+          moduleSourceInfoPathID, headerImportID, definingInterfacePathID,
-          headerImportID, headerModuleDependenciesArrayID,
+          headerModuleDependenciesArrayID, headerImportsSourceFilesArrayID,
-          headerImportsSourceFilesArrayID, isFramework, isStatic,
+          isFramework, isStatic, moduleCacheKeyID, userModuleVersionID);
-          moduleCacheKeyID, userModuleVersionID);
 
       auto compiledModulePath = getIdentifier(compiledModulePathID);
       if (!compiledModulePath)
@@ -543,13 +585,16 @@ bool ModuleDependenciesCacheDeserializer::readGraph(ModuleDependenciesCache &cac
       auto headerImport = getIdentifier(headerImportID);
       if (!headerImport)
         llvm::report_fatal_error("Bad binary direct dependencies: no header import");
+      auto definingInterfacePath = getIdentifier(definingInterfacePathID);
+      if (!definingInterfacePath)
+        llvm::report_fatal_error("Bad binary direct dependencies: no defining interface path");
 
-      // TODO: LinkLibraries, DefiningModulePath
       // Form the dependencies storage object
       auto moduleDep = ModuleDependencyInfo::forSwiftBinaryModule(
            *compiledModulePath, *moduleDocPath, *moduleSourceInfoPath,
-           currentModuleImports, currentOptionalModuleImports, {},
+           currentModuleImports, currentOptionalModuleImports, linkLibraries,
-           *headerImport, "", isFramework, isStatic, *moduleCacheKey, *userModuleVersion);
+           *headerImport, *definingInterfacePath, isFramework, isStatic,
+           *moduleCacheKey, *userModuleVersion);
 
       // Add imports of this module
       for (const auto &moduleName : currentModuleImports)
@@ -665,11 +710,10 @@ bool ModuleDependenciesCacheDeserializer::readGraph(ModuleDependenciesCache &cac
       if (!moduleCacheKey)
         llvm::report_fatal_error("Bad moduleCacheKey");
 
-      // TODO: LinkLibraries
       // Form the dependencies storage object
       auto moduleDep = ModuleDependencyInfo::forClangModule(
           *pcmOutputPath, *mappedPCMPath, *moduleMapPath, *contextHash,
-          *commandLineArgs, *fileDependencies, *capturedPCMArgs, {},
+          *commandLineArgs, *fileDependencies, *capturedPCMArgs, linkLibraries,
           *rootFileSystemID, *clangIncludeTreeRoot, *moduleCacheKey, isSystem);
 
       // Add imports of this module
@@ -747,6 +791,27 @@ ModuleDependenciesCacheDeserializer::getStringArray(unsigned n) {
   return result;
 }
 
+std::optional<std::vector<LinkLibrary>>
+ModuleDependenciesCacheDeserializer::getLinkLibraryArray(unsigned n) {
+  if (n == 0)
+    return std::vector<LinkLibrary>();
+
+  --n;
+  if (n >= ArraysOfLinkLibraryIDs.size())
+    return std::nullopt;
+
+  auto &llIDs = ArraysOfLinkLibraryIDs[n];
+
+  auto IDtoLLMap = [this](unsigned index) {
+    return LinkLibraries[index];
+  };
+  std::vector<LinkLibrary> result;
+  result.reserve(llIDs.size());
+  std::transform(llIDs.begin(), llIDs.end(),
+                 std::back_inserter(result), IDtoLLMap);
+  return result;
+}
+
 std::optional<std::vector<ModuleDependencyID>>
 ModuleDependenciesCacheDeserializer::getModuleDependencyIDArray(unsigned n) {
   auto encodedIdentifierStringArray = getStringArray(n);
@@ -806,7 +871,6 @@ enum ModuleIdentifierArrayKind : uint8_t {
   Empty = 0,
   DependencyImports,
   OptionalDependencyImports,
-  DependencyHeaders,
   ImportedSwiftDependenciesIDs,
   ImportedClangDependenciesIDs,
   CrossImportOverlayDependenciesIDs,
@@ -858,12 +922,16 @@ class ModuleDependenciesCacheSerializer {
   llvm::StringMap<unsigned, llvm::BumpPtrAllocator> IdentifierIDs;
   std::unordered_map<ModuleDependencyID,
                      llvm::DenseMap<ModuleIdentifierArrayKind, unsigned>>
-      ArrayIDs;
+      IdentifierArrayIDsMap;
   unsigned LastIdentifierID = 0;
-  unsigned LastArrayID = 0;
+  unsigned LastIdentifierArrayID = 0;
   std::vector<StringRef> Identifiers;
   std::vector<std::vector<unsigned>> ArraysOfIdentifiers;
 
+  std::unordered_map<ModuleDependencyID,
+                     unsigned>
+      LinkLibraryArrayIDsMap;
+
   llvm::BitstreamWriter &Out;
 
   /// A reusable buffer for emitting records.
@@ -882,8 +950,9 @@ class ModuleDependenciesCacheSerializer {
   void addDependencyIDArray(ModuleDependencyID moduleID,
                             ModuleIdentifierArrayKind arrayKind,
                             const ArrayRef<ModuleDependencyID> vec);
-  unsigned getArrayID(ModuleDependencyID moduleID,
+  unsigned getIdentifierArrayID(ModuleDependencyID moduleID,
                                 ModuleIdentifierArrayKind arrayKind) const;
+  unsigned getLinkLibrariesArrayID(ModuleDependencyID moduleID) const;
 
   template <typename Layout>
   void registerRecordAbbr() {
@@ -908,6 +977,11 @@ class ModuleDependenciesCacheSerializer {
   void writeIdentifiers();
   void writeArraysOfIdentifiers();
 
+  void writeLinkLibraries(const ModuleDependenciesCache &cache);
+  unsigned writeLinkLibraryInfos(ModuleDependencyID moduleID,
+                                 const ModuleDependencyInfo &dependencyInfo);
+  void writeLinkLibraryInfoArray(unsigned startIndex, unsigned count);
+
   void writeModuleInfo(ModuleDependencyID moduleID,
                        const ModuleDependencyInfo &dependencyInfo);
 
@@ -958,6 +1032,8 @@ void ModuleDependenciesCacheSerializer::writeBlockInfoBlock() {
   BLOCK_RECORD(graph_block, IDENTIFIER_NODE);
   BLOCK_RECORD(graph_block, IDENTIFIER_ARRAY_NODE);
 
+  BLOCK_RECORD(graph_block, LINK_LIBRARY_NODE);
+  BLOCK_RECORD(graph_block, LINK_LIBRARY_ARRAY_NODE);
   BLOCK_RECORD(graph_block, MODULE_NODE);
   BLOCK_RECORD(graph_block, SWIFT_INTERFACE_MODULE_DETAILS_NODE);
   BLOCK_RECORD(graph_block, SWIFT_SOURCE_MODULE_DETAILS_NODE);
@@ -997,24 +1073,77 @@ void ModuleDependenciesCacheSerializer::writeArraysOfIdentifiers() {
   }
 }
 
+void ModuleDependenciesCacheSerializer::writeLinkLibraries(const ModuleDependenciesCache &cache) {
+  unsigned lastLLIndex = 0;
+  std::map<ModuleDependencyID, std::pair<unsigned, unsigned>> moduleLLArrayMap;
+  for (auto kind = ModuleDependencyKind::FirstKind;
+       kind != ModuleDependencyKind::LastKind; ++kind) {
+    auto modMap = cache.getDependenciesMap(kind);
+    for (const auto &entry : modMap) {
+      ModuleDependencyID moduleID = {entry.getKey().str(), kind};
+      auto optionalDependencyInfo = cache.findDependency(moduleID);
+      assert(optionalDependencyInfo.has_value() && "Expected dependency info.");
+      auto dependencyInfo = optionalDependencyInfo.value();
+      unsigned numLLs = writeLinkLibraryInfos(moduleID, *dependencyInfo);
+      moduleLLArrayMap.insert({moduleID, std::make_pair(lastLLIndex, numLLs)});
+      lastLLIndex += numLLs;
+    }
+  }
+
+  unsigned lastLLArrayIndex = 1;
+  for (auto kind = ModuleDependencyKind::FirstKind;
+       kind != ModuleDependencyKind::LastKind; ++kind) {
+    auto modMap = cache.getDependenciesMap(kind);
+    for (const auto &entry : modMap) {
+      ModuleDependencyID moduleID = {entry.getKey().str(), kind};
+      auto entries = moduleLLArrayMap.at(moduleID);
+      if (entries.second == 0)
+        continue;
+      writeLinkLibraryInfoArray(entries.first, entries.second);
+      LinkLibraryArrayIDsMap.insert({moduleID, lastLLArrayIndex++});
+    }
+  }
+}
+
+unsigned ModuleDependenciesCacheSerializer::writeLinkLibraryInfos(
+    ModuleDependencyID moduleID,
+    const ModuleDependencyInfo &dependencyInfo) {
+  using namespace graph_block;
+  for (auto &linkLibrary : dependencyInfo.getLinkLibraries()) {
+    LinkLibraryLayout::emitRecord(Out, ScratchRecord, AbbrCodes[LinkLibraryLayout::Code],
+                                  getIdentifier(linkLibrary.getName().str()),
+                                  linkLibrary.getKind() == LibraryKind::Framework,
+                                  linkLibrary.shouldForceLoad());
+  }
+  return dependencyInfo.getLinkLibraries().size();
+}
+
+void ModuleDependenciesCacheSerializer::writeLinkLibraryInfoArray(
+    unsigned startIndex, unsigned count) {
+  using namespace graph_block;
+  std::vector<unsigned> vec(count);
+  std::iota(vec.begin(), vec.end(), startIndex);
+  LinkLibraryArrayLayout::emitRecord(
+      Out, ScratchRecord, AbbrCodes[LinkLibraryArrayLayout::Code], vec);
+}
+
 void ModuleDependenciesCacheSerializer::writeModuleInfo(
     ModuleDependencyID moduleID,
     const ModuleDependencyInfo &dependencyInfo) {
   using namespace graph_block;
-  // TODO: Eliminate per-module context hash
-  auto contextHashStrID = 0;
 
   ModuleInfoLayout::emitRecord(
       Out, ScratchRecord, AbbrCodes[ModuleInfoLayout::Code],
-      getIdentifier(moduleID.ModuleName), contextHashStrID,
+      getIdentifier(moduleID.ModuleName),
-      getArrayID(moduleID, ModuleIdentifierArrayKind::DependencyImports),
+      getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::DependencyImports),
-      getArrayID(moduleID, ModuleIdentifierArrayKind::OptionalDependencyImports),
+      getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::OptionalDependencyImports),
-      getArrayID(moduleID, ModuleIdentifierArrayKind::ImportedSwiftDependenciesIDs),
+      getLinkLibrariesArrayID(moduleID),
-      getArrayID(moduleID, ModuleIdentifierArrayKind::ImportedClangDependenciesIDs),
+      getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::ImportedSwiftDependenciesIDs),
-      getArrayID(moduleID, ModuleIdentifierArrayKind::CrossImportOverlayDependenciesIDs),
+      getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::ImportedClangDependenciesIDs),
-      getArrayID(moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependenciesIDs),
+      getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::CrossImportOverlayDependenciesIDs),
+      getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependenciesIDs),
       getIdentifier(dependencyInfo.getModuleCacheKey()),
-      getArrayID(moduleID, ModuleIdentifierArrayKind::AuxiliaryFileIDs));
+      getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::AuxiliaryFileIDs));
 
   switch (dependencyInfo.getKind()) {
   case swift::ModuleDependencyKind::SwiftInterface: {
@@ -1032,16 +1161,16 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(
     SwiftInterfaceModuleDetailsLayout::emitRecord(
         Out, ScratchRecord, AbbrCodes[SwiftInterfaceModuleDetailsLayout::Code],
         outputModulePathFileId, swiftInterfaceFileId,
-        getArrayID(moduleID,
+        getIdentifierArrayID(moduleID,
                    ModuleIdentifierArrayKind::CompiledModuleCandidates),
-        getArrayID(moduleID, ModuleIdentifierArrayKind::BuildCommandLine),
+        getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::BuildCommandLine),
-        getArrayID(moduleID, ModuleIdentifierArrayKind::ExtraPCMArgs),
+        getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::ExtraPCMArgs),
         getIdentifier(swiftTextDeps->contextHash), 
         swiftTextDeps->isFramework, swiftTextDeps->isStatic,
         bridgingHeaderFileId,
-        getArrayID(moduleID, ModuleIdentifierArrayKind::SourceFiles),
+        getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::SourceFiles),
-        getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles),
+        getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles),
-        getArrayID(moduleID,
+        getIdentifierArrayID(moduleID,
                    ModuleIdentifierArrayKind::BridgingModuleDependencies),
         getIdentifier(swiftTextDeps->textualModuleDetails.CASFileSystemRootID),
         getIdentifier(swiftTextDeps->textualModuleDetails
@@ -1060,18 +1189,18 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(
             : 0;
     SwiftSourceModuleDetailsLayout::emitRecord(
         Out, ScratchRecord, AbbrCodes[SwiftSourceModuleDetailsLayout::Code],
-        getArrayID(moduleID, ModuleIdentifierArrayKind::ExtraPCMArgs),
+        getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::ExtraPCMArgs),
         bridgingHeaderFileId,
-        getArrayID(moduleID, ModuleIdentifierArrayKind::SourceFiles),
+        getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::SourceFiles),
-        getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles),
+        getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles),
-        getArrayID(moduleID,
+        getIdentifierArrayID(moduleID,
                    ModuleIdentifierArrayKind::BridgingModuleDependencies),
         getIdentifier(
             swiftSourceDeps->textualModuleDetails.CASFileSystemRootID),
         getIdentifier(swiftSourceDeps->textualModuleDetails
                           .CASBridgingHeaderIncludeTreeRootID),
-        getArrayID(moduleID, ModuleIdentifierArrayKind::BuildCommandLine),
+        getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::BuildCommandLine),
-        getArrayID(moduleID,
+        getIdentifierArrayID(moduleID,
                    ModuleIdentifierArrayKind::BridgingHeaderBuildCommandLine));
     break;
   }
@@ -1083,9 +1212,10 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(
         getIdentifier(swiftBinDeps->compiledModulePath),
         getIdentifier(swiftBinDeps->moduleDocPath),
         getIdentifier(swiftBinDeps->sourceInfoPath),
-        getArrayID(moduleID, ModuleIdentifierArrayKind::DependencyHeaders),
+        getIdentifier(swiftBinDeps->headerImport),
-        getArrayID(moduleID, ModuleIdentifierArrayKind::HeaderInputModuleDependencies),
+        getIdentifier(swiftBinDeps->definingModuleInterfacePath),
-        getArrayID(moduleID, ModuleIdentifierArrayKind::HeaderInputDependencySourceFiles),
+        getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::HeaderInputModuleDependencies),
+        getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::HeaderInputDependencySourceFiles),
         swiftBinDeps->isFramework, swiftBinDeps->isStatic,
         getIdentifier(swiftBinDeps->moduleCacheKey),
         getIdentifier(swiftBinDeps->userModuleVersion));
@@ -1112,9 +1242,9 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(
         getIdentifier(clangDeps->mappedPCMPath),
         getIdentifier(clangDeps->moduleMapFile),
         getIdentifier(clangDeps->contextHash),
-        getArrayID(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine),
+        getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine),
-        getArrayID(moduleID, ModuleIdentifierArrayKind::FileDependencies),
+        getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::FileDependencies),
-        getArrayID(moduleID, ModuleIdentifierArrayKind::CapturedPCMArgs),
+        getIdentifierArrayID(moduleID, ModuleIdentifierArrayKind::CapturedPCMArgs),
         getIdentifier(clangDeps->CASFileSystemRootID),
         getIdentifier(clangDeps->CASClangIncludeTreeRootID),
         getIdentifier(clangDeps->moduleCacheKey), clangDeps->IsSystem);
@@ -1166,21 +1296,21 @@ void ModuleDependenciesCacheSerializer::addDependencyIDArray(ModuleDependencyID
 void ModuleDependenciesCacheSerializer::addStringArray(ModuleDependencyID moduleID,
                                                        ModuleIdentifierArrayKind arrayKind,
                                                        const std::vector<std::string> &vec) {
-  if (ArrayIDs.find(moduleID) != ArrayIDs.end()) {
+  if (IdentifierArrayIDsMap.find(moduleID) != IdentifierArrayIDsMap.end()) {
     // Already have arrays for this module
     llvm::DenseMap<ModuleIdentifierArrayKind, unsigned>::iterator iter;
     bool isNew;
     std::tie(iter, isNew) =
-        ArrayIDs[moduleID].insert({arrayKind, LastArrayID + 1});
+        IdentifierArrayIDsMap[moduleID].insert({arrayKind, LastIdentifierArrayID + 1});
     if (!isNew)
       return;
   } else {
     // Do not yet have any arrays for this module
-    ArrayIDs[moduleID] = llvm::DenseMap<ModuleIdentifierArrayKind, unsigned>();
+    IdentifierArrayIDsMap[moduleID] = llvm::DenseMap<ModuleIdentifierArrayKind, unsigned>();
-    ArrayIDs[moduleID].insert({arrayKind, LastArrayID + 1});
+    IdentifierArrayIDsMap[moduleID].insert({arrayKind, LastIdentifierArrayID + 1});
   }
 
-  ++LastArrayID;
+  ++LastIdentifierArrayID;
 
   // Add in the individual identifiers in the array
   std::vector<unsigned> identifierIDs;
@@ -1193,24 +1323,33 @@ void ModuleDependenciesCacheSerializer::addStringArray(ModuleDependencyID module
   return;
 }
 
-unsigned ModuleDependenciesCacheSerializer::getArrayID(ModuleDependencyID moduleID,
+unsigned ModuleDependenciesCacheSerializer::getIdentifierArrayID(ModuleDependencyID moduleID,
                                                        ModuleIdentifierArrayKind arrayKind) const {
-  auto iter = ArrayIDs.find(moduleID);
+  auto iter = IdentifierArrayIDsMap.find(moduleID);
-  assert(iter != ArrayIDs.end());
+  assert(iter != IdentifierArrayIDsMap.end());
   auto &innerMap = iter->second;
   auto arrayIter = innerMap.find(arrayKind);
   assert(arrayIter != innerMap.end());
   return arrayIter->second;
 }
 
+unsigned ModuleDependenciesCacheSerializer::getLinkLibrariesArrayID(ModuleDependencyID moduleID) const {
+  auto iter = LinkLibraryArrayIDsMap.find(moduleID);
+  if (iter == LinkLibraryArrayIDsMap.end())
+    return 0;
+
+  return iter->second;
+}
+
 void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
     const ModuleDependenciesCache &cache) {
   addIdentifier(cache.scannerContextHash);
-  // TODO: Serialize *all* modules
+  for (auto kind = ModuleDependencyKind::FirstKind;
-  for (auto &moduleID : cache.getAllDependencies({cache.mainScanModuleName,
+       kind != ModuleDependencyKind::LastKind; ++kind) {
-                                                  ModuleDependencyKind::SwiftSource})) {
+    auto modMap = cache.getDependenciesMap(kind);
-    auto optionalDependencyInfo =
+    for (const auto &entry : modMap) {
-        cache.findDependency(moduleID.ModuleName, moduleID.Kind);
+      ModuleDependencyID moduleID = {entry.getKey().str(), kind};
+      auto optionalDependencyInfo = cache.findDependency(moduleID);
       assert(optionalDependencyInfo.has_value() && "Expected dependency info.");
       auto dependencyInfo = optionalDependencyInfo.value();
       // Add the module's name
@@ -1222,7 +1361,10 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
           return importInfo.importIdentifier;
         };
 
-    // Add the module's dependencies
+      for (const auto &ll : dependencyInfo->getLinkLibraries())
+        addIdentifier(ll.getName().str());
+
+      // Add the module's imports
       std::vector<std::string> importIdentifiers;
       llvm::transform(dependencyInfo->getModuleImports(),
                       std::back_inserter(importIdentifiers),
@@ -1237,28 +1379,21 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
       addStringArray(moduleID, ModuleIdentifierArrayKind::OptionalDependencyImports,
                      optionalImportIdentifiers);
 
-    ModuleDependencyIDSetVector allDependencies;
+      addDependencyIDArray(moduleID,
-    if (dependencyInfo->isSwiftModule()) {
+                           ModuleIdentifierArrayKind::ImportedSwiftDependenciesIDs,
-      auto swiftImportedDepsRef = dependencyInfo->getImportedSwiftDependencies();
+                           dependencyInfo->getImportedSwiftDependencies());
-      auto headerClangDepsRef = dependencyInfo->getHeaderClangDependencies();
+      addDependencyIDArray(moduleID,
-      auto overlayDependenciesRef = dependencyInfo->getSwiftOverlayDependencies();
+                           ModuleIdentifierArrayKind::ImportedClangDependenciesIDs,
-      allDependencies.insert(swiftImportedDepsRef.begin(),
+                           dependencyInfo->getImportedClangDependencies());
-                    swiftImportedDepsRef.end());
+      addDependencyIDArray(moduleID,
-      allDependencies.insert(headerClangDepsRef.begin(),
+                           ModuleIdentifierArrayKind::CrossImportOverlayDependenciesIDs,
-                    headerClangDepsRef.end());
+                           dependencyInfo->getCrossImportOverlayDependencies());
-      allDependencies.insert(overlayDependenciesRef.begin(),
+      addDependencyIDArray(moduleID,
-                    overlayDependenciesRef.end());
+                           ModuleIdentifierArrayKind::SwiftOverlayDependenciesIDs,
-    }
+                           dependencyInfo->getSwiftOverlayDependencies());
 
-    if (dependencyInfo->isSwiftSourceModule()) {
+      addStringArray(moduleID, ModuleIdentifierArrayKind::AuxiliaryFileIDs,
-      auto crossImportOverlayDepsRef = dependencyInfo->getCrossImportOverlayDependencies();
+                     dependencyInfo->getAuxiliaryFiles());
-      allDependencies.insert(crossImportOverlayDepsRef.begin(),
-                    crossImportOverlayDepsRef.end());
-    }
-
-    auto clangImportedDepsRef = dependencyInfo->getImportedClangDependencies();
-    allDependencies.insert(clangImportedDepsRef.begin(),
-                  clangImportedDepsRef.end());
 
       std::vector<std::string> clangHeaderDependencyNames;
       for (const auto &headerDepID :
@@ -1304,6 +1439,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
         addIdentifier(swiftBinDeps->sourceInfoPath);
         addIdentifier(swiftBinDeps->moduleCacheKey);
         addIdentifier(swiftBinDeps->headerImport);
+        addIdentifier(swiftBinDeps->userModuleVersion);
         addStringArray(moduleID, ModuleIdentifierArrayKind::HeaderInputModuleDependencies,
                        clangHeaderDependencyNames);
         addStringArray(moduleID, ModuleIdentifierArrayKind::HeaderInputDependencySourceFiles,
@@ -1367,6 +1503,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
         llvm_unreachable("Unhandled dependency kind.");
       }
     }
+  }
 }
 
 void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache(
@@ -1385,8 +1522,9 @@ void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache(
   registerRecordAbbr<IdentifierNodeLayout>();
   registerRecordAbbr<IdentifierArrayLayout>();
   registerRecordAbbr<LinkLibraryLayout>();
-  registerRecordAbbr<SourceLocationLayout>();
+  registerRecordAbbr<LinkLibraryArrayLayout>();
-  registerRecordAbbr<ImportStatementLayout>();
+//  registerRecordAbbr<SourceLocationLayout>();
+//  registerRecordAbbr<ImportStatementLayout>();
   registerRecordAbbr<ModuleInfoLayout>();
   registerRecordAbbr<SwiftSourceModuleDetailsLayout>();
   registerRecordAbbr<SwiftInterfaceModuleDetailsLayout>();
@@ -1407,13 +1545,16 @@ void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache(
   // Write the arrays
   writeArraysOfIdentifiers();
 
+  // Write all the arrays of link library infos for this graph
+  writeLinkLibraries(cache);
+
   // Write the core graph
-  // TODO: Serialize *all* modules
+  for (auto kind = ModuleDependencyKind::FirstKind;
-  for (auto &moduleID : cache.getAllDependencies({cache.mainScanModuleName,
+       kind != ModuleDependencyKind::LastKind; ++kind) {
-                                                  ModuleDependencyKind::SwiftSource})) {
+    auto modMap = cache.getDependenciesMap(kind);
-    auto dependencyInfo = cache.findDependency(moduleID.ModuleName, moduleID.Kind);
+    for (const auto &modInfo : modMap) {
-    assert(dependencyInfo.has_value() && "Expected dependency info.");
+      writeModuleInfo({modInfo.getKey().str(), kind}, modInfo.second);
-    writeModuleInfo(moduleID, **dependencyInfo);
+    }
   }
   return;
 }

lib/DependencyScan/ModuleDependencyScanner.cpp

@@ -713,9 +713,6 @@ ModuleDependencyScanner::resolveAllClangModuleDependencies(
     } else {
       // We need to query the Clang dependency scanner for this module's
       // unresolved imports
-      auto moduleDependencyInfo = cache.findKnownDependency(moduleID);
-
-      // Figure out which imports have already been resolved to module dependencies
       llvm::StringSet<> resolvedImportIdentifiers;
       for (const auto &resolvedDep : moduleDependencyInfo.getImportedSwiftDependencies())
         resolvedImportIdentifiers.insert(resolvedDep.ModuleName);

lib/DependencyScan/ScanDependencies.cpp

@@ -1287,7 +1287,14 @@ bool swift::dependencies::scanDependencies(CompilerInstance &CI) {
       CI.getInvocation().getFrontendOptions().ExplicitModulesOutputPath,
       CI.getInvocation().getModuleScanningHash());
 
-  // ACTODO: Deserialize cache
+  // Load the dependency cache if -reuse-dependency-scan-cache
+  // is specified
+  if (opts.ReuseDependencyScannerCache) {
+    auto cachePath = opts.SerializedDependencyScannerCachePath;
+    module_dependency_cache_serialization::readInterModuleDependenciesCache(cachePath, cache);
+    if (opts.EmitDependencyScannerCacheRemarks)
+      ctx.Diags.diagnose(SourceLoc(), diag::remark_reuse_cache, cachePath);
+  }
   
   if (service->setupCachingDependencyScanningService(CI))
     return true;
@@ -1298,7 +1305,13 @@ bool swift::dependencies::scanDependencies(CompilerInstance &CI) {
 
   // Serialize the dependency cache if -serialize-dependency-scan-cache
   // is specified
-  // ACTODO: Serialize cache
+  if (opts.SerializeDependencyScannerCache) {
+    auto savePath = opts.SerializedDependencyScannerCachePath;
+    module_dependency_cache_serialization::writeInterModuleDependenciesCache(
+          ctx.Diags, CI.getOutputBackend(), savePath, cache);
+    if (opts.EmitDependencyScannerCacheRemarks)
+      ctx.Diags.diagnose(SourceLoc(), diag::remark_save_cache, savePath);
+  }
 
   if (dependenciesOrErr.getError())
     return true;