[ParseableInterface] Don't serialize swiftmodule dependencies in the module cache or prebuilt module cache

*Their* dependencies are already being serialized out, so this shouldn't affect
up-to-date-checking except by alowing the regular and prebuilt module caches to
be relocated without invalidating their contents. In the case of the prebuilt
module cache, this gets us closer to supporting relocation across machines.

lib/Frontend/ParseableInterfaceModuleLoader.cpp

@@ -356,48 +356,39 @@ class swift::ParseableInterfaceBuilder {
     return false;
   }
 
-  /// Populate the provided \p Deps with \c FileDependency entries including:
+  /// Determines if the dependency with the provided path is a swiftmodule in
-  ///
+  /// either the module cache or prebuilt module cache
-  ///    - All the dependencies mentioned by \p SubInstance's DependencyTracker,
+  bool isCachedModule(StringRef DepName) const {
-  ///      that were read while compiling the module.
+    if (moduleCachePath.empty() && prebuiltCachePath.empty())
-  ///
+      return false;
-  ///    - For any file in the latter set that is itself a .swiftmodule
+
-  ///      living in \p cachePath, all of _its_ dependencies, copied
+    auto Ext = llvm::sys::path::extension(DepName);
-  ///      out to avoid having to do recursive scanning when rechecking this
+    auto Ty = file_types::lookupTypeForExtension(Ext);
-  ///      dependency in the future.
+
+    if (Ty != file_types::TY_SwiftModuleFile)
+      return false;
+    if (!moduleCachePath.empty() && DepName.startswith(moduleCachePath))
+      return true;
+    return !prebuiltCachePath.empty() && DepName.startswith(prebuiltCachePath);
+  }
+
+  /// Populate the provided \p Deps with \c FileDependency entries for all
+  /// dependencies \p SubInstance's DependencyTracker recorded while compiling
+  /// the module, excepting .swiftmodules in \p moduleCachePath or
+  /// \p prebuiltCachePath. Those have _their_ dependencies added instead, both
+  /// to avoid having to do recursive scanning when rechecking this dependency
+  /// in future and to make the module caches relocatable.
   bool collectDepsForSerialization(CompilerInstance &SubInstance,
                                    SmallVectorImpl<FileDependency> &Deps,
                                    bool IsHashBased) {
     StringRef SDKPath = SubInstance.getASTContext().SearchPathOpts.SDKPath;
-
     auto DTDeps = SubInstance.getDependencyTracker()->getDependencies();
     SmallVector<StringRef, 16> InitialDepNames(DTDeps.begin(), DTDeps.end());
     InitialDepNames.push_back(interfacePath);
     llvm::StringSet<> AllDepNames;
+
     for (auto const &DepName : InitialDepNames) {
-      if (AllDepNames.insert(DepName).second && dependencyTracker) {
+      // Adjust the paths of dependences in the SDK to be relative to it
-        dependencyTracker->addDependency(DepName, /*isSystem*/false);
-      }
-
-      /// Lazily load the dependency buffer if we need it. If we're not
-      /// dealing with a hash-based dependencies, and if the dependency is
-      /// not a .swiftmodule, we can avoid opening the buffer.
-      std::unique_ptr<llvm::MemoryBuffer> DepBuf = nullptr;
-      auto getDepBuf = [&]() -> llvm::MemoryBuffer * {
-        if (DepBuf) return DepBuf.get();
-        if (auto Buf = getBufferOfDependency(fs, DepName, interfacePath,
-                                             diags, diagnosticLoc)) {
-          DepBuf = std::move(Buf);
-          return DepBuf.get();
-        }
-        return nullptr;
-      };
-
-      auto Status = getStatusOfDependency(fs, DepName, interfacePath,
-                                          diags, diagnosticLoc);
-      if (!Status)
-        return true;
-
       bool IsSDKRelative = false;
       StringRef DepNameToStore = DepName;
       if (SDKPath.size() > 1 && DepName.startswith(SDKPath)) {
@@ -418,32 +409,29 @@ class swift::ParseableInterfaceBuilder {
         }
       }
 
-      if (IsHashBased) {
+      if (AllDepNames.insert(DepName).second && dependencyTracker) {
-        auto buf = getDepBuf();
+        dependencyTracker->addDependency(DepName, /*isSystem*/IsSDKRelative);
-        if (!buf) return true;
-        uint64_t hash = xxHash64(buf->getBuffer());
-        Deps.push_back(
-          FileDependency::hashBased(DepNameToStore, IsSDKRelative,
-                                    Status->getSize(), hash));
-      } else {
-        uint64_t mtime =
-          Status->getLastModificationTime().time_since_epoch().count();
-        Deps.push_back(
-          FileDependency::modTimeBased(DepNameToStore, IsSDKRelative,
-                                       Status->getSize(), mtime));
       }
 
-      if (moduleCachePath.empty())
+      /// Lazily load the dependency buffer if we need it. If we're not
-        continue;
+      /// dealing with a hash-based dependencies, and if the dependency is
+      /// not a .swiftmodule, we can avoid opening the buffer.
+      std::unique_ptr<llvm::MemoryBuffer> DepBuf = nullptr;
+      auto getDepBuf = [&]() -> llvm::MemoryBuffer * {
+        if (DepBuf) return DepBuf.get();
+        if (auto Buf = getBufferOfDependency(fs, DepName, interfacePath,
+                                             diags, diagnosticLoc)) {
+          DepBuf = std::move(Buf);
+          return DepBuf.get();
+        }
+        return nullptr;
+      };
 
-      // If Dep is itself a .swiftmodule in the cache dir, pull out its deps
+      // If Dep is itself a cached .swiftmodule, pull out its deps and include
-      // and include them in our own, so we have a single-file view of
+      // them in our own, so we have a single-file view of transitive deps:
-      // transitive deps: removes redundancies, and avoids opening and reading
+      // removes redundancies, makes the cache more relocatable, and avoids
-      // multiple swiftmodules during future loads.
+      // opening and reading multiple swiftmodules during future loads.
-      auto Ext = llvm::sys::path::extension(DepName);
+      if (isCachedModule(DepName)) {
-      auto Ty = file_types::lookupTypeForExtension(Ext);
-      if (Ty == file_types::TY_SwiftModuleFile &&
-          DepName.startswith(moduleCachePath)) {
         auto buf = getDepBuf();
         if (!buf) return true;
         SmallVector<FileDependency, 16> SubDeps;
@@ -459,10 +447,32 @@ class swift::ParseableInterfaceBuilder {
           if (AllDepNames.insert(SubDep.getPath()).second) {
             Deps.push_back(SubDep);
             if (dependencyTracker)
-              dependencyTracker->addDependency(SubDep.getPath(),
+              dependencyTracker->addDependency(
-                                               /*IsSystem=*/false);
+                  SubDep.getPath(), /*IsSystem=*/SubDep.isSDKRelative());
           }
         }
+        continue;
+      }
+
+      // Otherwise, include this dependency directly
+      auto Status = getStatusOfDependency(fs, DepName, interfacePath,
+                                          diags, diagnosticLoc);
+      if (!Status)
+        return true;
+
+      if (IsHashBased) {
+        auto buf = getDepBuf();
+        if (!buf) return true;
+        uint64_t hash = xxHash64(buf->getBuffer());
+        Deps.push_back(
+          FileDependency::hashBased(DepNameToStore, IsSDKRelative,
+                                    Status->getSize(), hash));
+      } else {
+        uint64_t mtime =
+          Status->getLastModificationTime().time_since_epoch().count();
+        Deps.push_back(
+          FileDependency::modTimeBased(DepNameToStore, IsSDKRelative,
+                                       Status->getSize(), mtime));
       }
     }
     return false;

test/ParseableInterface/ModuleCache/SDKDependencies.swift

@@ -1,6 +1,3 @@
-// swift-interface-format-version: 1.0
-// swift-module-flags: -module-name SDKDependencies
-
 // RUN: %empty-directory(%t)
 
 // 1) Build a prebuilt cache for our SDK
@@ -46,11 +43,10 @@
 // RUN: cat %t/MCP/ExportedLib-*.swiftmodule | %FileCheck %s -check-prefix=EXLIB
 // RUN: cat %t/MCP/SdkLib-*.swiftmodule | %FileCheck %s -check-prefixes=EXLIB,SDKLIB
 //
-// EXLIB-DAG: /prebuilt-cache/ExportedLib.swiftmodule
+// EXLIB: dependencies:
 // EXLIB-DAG: /my-sdk/usr/include/module.modulemap
 // EXLIB-DAG: /my-sdk/usr/include/SomeCModule.h
 // EXLIB-DAG: /my-sdk/ExportedLib.swiftinterface
-// SDKLIB-DAG: /prebuilt-cache/SdkLib.swiftmodule
 // SDKLIB-DAG: /my-sdk/SdkLib.swiftinterface
 //
 // RUN: %empty-directory(%t/MCP)
@@ -60,7 +56,8 @@
 // 4) Move the SDK without changing its contents
 //
 // RUN: mv %t/my-sdk %t/my-new-sdk
-// RUN: %target-swift-frontend -typecheck -I %t/my-new-sdk -sdk %t/my-new-sdk -prebuilt-module-cache-path %t/prebuilt-cache -module-cache-path %t/MCP -emit-dependencies-path %t/dummy.d -track-system-dependencies %s
+// RUN: mv %t/prebuilt-cache %t/new-prebuilt-cache
+// RUN: %target-swift-frontend -typecheck -I %t/my-new-sdk -sdk %t/my-new-sdk -prebuilt-module-cache-path %t/new-prebuilt-cache -module-cache-path %t/MCP -emit-dependencies-path %t/dummy.d -track-system-dependencies %s
 //
 // Check SdkLib and ExportedLib are in the module cache
 // RUN: test -f %t/MCP/SdkLib-*.swiftmodule
@@ -74,11 +71,9 @@
 // RUN: cat %t/MCP/ExportedLib-*.swiftmodule | %FileCheck %s -check-prefix=NEW-EXLIB
 // RUN: cat %t/MCP/SdkLib-*.swiftmodule | %FileCheck %s -check-prefixes=NEW-EXLIB,NEW-SDKLIB
 //
-// NEW-EXLIB-DAG: /prebuilt-cache/ExportedLib.swiftmodule
 // NEW-EXLIB-DAG: /my-new-sdk/usr/include/module.modulemap
 // NEW-EXLIB-DAG: /my-new-sdk/usr/include/SomeCModule.h
 // NEW-EXLIB-DAG: /my-new-sdk/ExportedLib.swiftinterface
-// NEW-SDKLIB-DAG: /prebuilt-cache/SdkLib.swiftmodule
 // NEW-SDKLIB-DAG: /my-new-sdk/SdkLib.swiftinterface
 //
 // RUN: %empty-directory(%t/MCP)
@@ -88,7 +83,7 @@
 // 5) Now change the SDK's content and check it no longer uses the prebuilt modules
 //
 // RUN: echo "// size change" >> %t/my-new-sdk/SdkLib.swiftinterface
-// RUN: %target-swift-frontend -typecheck -I %t/my-new-sdk -sdk %t/my-new-sdk -prebuilt-module-cache-path %t/prebuilt-cache -module-cache-path %t/MCP -emit-dependencies-path %t/dummy.d -track-system-dependencies %s
+// RUN: %target-swift-frontend -typecheck -I %t/my-new-sdk -sdk %t/my-new-sdk -prebuilt-module-cache-path %t/new-prebuilt-cache -module-cache-path %t/MCP -emit-dependencies-path %t/dummy.d -track-system-dependencies %s
 //
 // Check SDKLib and ExportedLib are in the module cache
 // RUN: test -f %t/MCP/SdkLib-*.swiftmodule
@@ -104,7 +99,7 @@
 // RUN: %empty-directory(%t/MCP)
 //
 // RUN: echo "// size change" >> %t/my-new-sdk/usr/include/SomeCModule.h
-// RUN: %target-swift-frontend -typecheck -I %t/my-new-sdk -sdk %t/my-new-sdk -prebuilt-module-cache-path %t/prebuilt-cache -module-cache-path %t/MCP -emit-dependencies-path %t/dummy.d -track-system-dependencies %s
+// RUN: %target-swift-frontend -typecheck -I %t/my-new-sdk -sdk %t/my-new-sdk -prebuilt-module-cache-path %t/new-prebuilt-cache -module-cache-path %t/MCP -emit-dependencies-path %t/dummy.d -track-system-dependencies %s
 //
 // Check SDKLib and ExportLib are in the module cache
 // RUN: test -f %t/MCP/SdkLib-*.swiftmodule

test/ParseableInterface/ModuleCache/module-cache-init.swift

@@ -39,7 +39,6 @@
 // CHECK-OTHERMODULE: {{MODULE_NAME.*blob data = 'OtherModule'}}
 // CHECK-OTHERMODULE: {{FILE_DEPENDENCY.*Swift.swiftmodule([/\\].+[.]swiftmodule)?'}}
 // CHECK-OTHERMODULE: {{FILE_DEPENDENCY.*LeafModule.swiftinterface'}}
-// CHECK-OTHERMODULE: {{FILE_DEPENDENCY.*LeafModule-.*.swiftmodule'}}
 // CHECK-OTHERMODULE: {{FILE_DEPENDENCY.*OtherModule.swiftinterface'}}
 // CHECK-OTHERMODULE: FUNC_DECL
 //