IRGen: Deserialize SIL witness tables and shared-linkage definitions by need.

Code may end up indirectly using a witness table for a Clang-imported type by inlining code that used the conformance from another module, in which case we need to ensure we have a local definition at hand in the inlining module so we can have something to link against independently. This needs to be fixed from both sides:

- During serialization, serialize not only witness tables from the current module, but from Clang-imported modules too, so that their definitions can be used by other modules that inline code from the current module
- During IRGen, when we emit a reference to a SILWitnessTable or SILFunction declaration with shared linkage, attempt to deserialize the definition on demand

Fixes rdar://problem/38687726.

include/swift/AST/SILOptions.h

@@ -46,6 +46,9 @@ public:
     /// Skip SIL linking.
     LinkNone,
     
+    /// Link only the given function without recursively visiting its uses.
+    LinkThisFunctionOnly,
+
     /// Perform normal SIL linking.
     LinkNormal,
 

include/swift/SIL/SILModule.h

@@ -715,6 +715,11 @@ public:
   bool isDefaultAtomic() const {
     return ! getOptions().AssumeSingleThreaded;
   }
+  
+  /// Returns true if SIL entities associated with declarations in the given
+  /// declaration context ought to be serialized as part of this module.
+  bool shouldSerializeEntitiesAssociatedWithDeclContext(const DeclContext *DC)
+    const;
 };
 
 inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const SILModule &M){

lib/IRGen/GenDecl.cpp

@@ -1054,7 +1054,8 @@ void IRGenerator::emitGlobalTopLevel() {
   unsigned nextOrderNumber = 0;
   for (auto &silFn : PrimaryIGM->getSILModule().getFunctions()) {
     // Don't bother adding external declarations to the function order.
-    if (!silFn.isDefinition()) continue;
+    if (!silFn.isDefinition())
+      continue;
     FunctionOrder.insert(std::make_pair(&silFn, nextOrderNumber++));
   }
 
@@ -1204,6 +1205,15 @@ void IRGenerator::emitLazyDefinitions() {
 }
 
 void IRGenerator::addLazyFunction(SILFunction *f) {
+  // If the function is a shared declaration, it may have a body to
+  // deserialize.
+  if (!f->isDefinition() && hasSharedVisibility(f->getLinkage())) {
+    SIL.linkFunction(f, SILOptions::LinkingMode::LinkThisFunctionOnly);
+    if (!f->isDefinition())
+      return;
+    FunctionOrder.insert({f, FunctionOrder.size()});
+  }
+
   // Add it to the queue if it hasn't already been put there.
   if (!LazilyEmittedFunctions.insert(f).second)
     return;
@@ -2241,8 +2251,8 @@ llvm::Function *IRGenModule::getAddrOfSILFunction(SILFunction *f,
     clangAddr = getAddrOfClangGlobalDecl(globalDecl, forDefinition);
   }
 
-  bool isDefinition = f->isDefinition();
-  bool hasOrderNumber = isDefinition;
+  bool isDefinition = f->isDefinition() || hasSharedVisibility(f->getLinkage());
+  bool hasOrderNumber = f->isDefinition();
   unsigned orderNumber = ~0U;
   llvm::Function *insertBefore = nullptr;
 
@@ -2273,8 +2283,9 @@ llvm::Function *IRGenModule::getAddrOfSILFunction(SILFunction *f,
     }
 
   // Otherwise, if we have a lazy definition for it, be sure to queue that up.
-  } else if (isDefinition && !forDefinition && !f->isPossiblyUsedExternally() &&
-             !hasCodeCoverageInstrumentation(*f, getSILModule())) {
+  } else if (isDefinition
+             && !forDefinition && !f->isPossiblyUsedExternally()
+             && !hasCodeCoverageInstrumentation(*f, getSILModule())) {
     IRGen.addLazyFunction(f);
   }
 
@@ -4116,7 +4127,7 @@ llvm::StructType *IRGenModule::getGenericWitnessTableCacheTy() {
 llvm::Function *
 IRGenModule::getAddrOfWitnessTableAccessFunction(
                                       const NormalProtocolConformance *conf,
-                                              ForDefinition_t forDefinition) {
+                                      ForDefinition_t forDefinition) {
   IRGen.addLazyWitnessTable(conf);
 
   LinkEntity entity = LinkEntity::forProtocolWitnessTableAccessFunction(conf);

lib/IRGen/IRGenModule.cpp

@@ -24,6 +24,7 @@
 #include "swift/IRGen/Linking.h"
 #include "swift/Runtime/RuntimeFnWrappersGen.h"
 #include "swift/Runtime/Config.h"
+#include "swift/SIL/FormalLinkage.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/Basic/CharInfo.h"
 #include "clang/Basic/TargetInfo.h"
@@ -658,6 +659,9 @@ llvm::Module *IRGenModule::releaseModule() {
 }
 
 bool IRGenerator::canEmitWitnessTableLazily(SILWitnessTable *wt) {
+  if (LazilyEmittedWitnessTables.count(wt))
+    return true;
+
   if (Opts.UseJIT)
     return false;
 
@@ -685,6 +689,21 @@ void IRGenerator::addLazyWitnessTable(const ProtocolConformance *Conf) {
         LazilyEmittedWitnessTables.insert(wt).second) {
       LazyWitnessTables.push_back(wt);
     }
+  // Shared-linkage protocol conformances may not have been deserialized yet
+  // if they were used by inlined code. See if we can deserialize it now.
+  } else {
+    auto linkage =
+      getLinkageForProtocolConformance(Conf->getRootNormalConformance(),
+                                       ForDefinition);
+    if (hasSharedVisibility(linkage)) {
+      SIL.createWitnessTableDeclaration(const_cast<ProtocolConformance*>(Conf),
+                                        linkage);
+      SILWitnessTable *wt = SIL.lookUpWitnessTable(Conf);
+      if (wt && wt->isDefinition()
+          && LazilyEmittedWitnessTables.insert(wt).second) {
+        LazyWitnessTables.push_back(wt);
+      }
+    }
   }
 }
 

lib/SIL/Linker.cpp

@@ -47,6 +47,9 @@ bool SILLinkerVisitor::processFunction(SILFunction *F) {
 
   ++NumFuncLinked;
 
+  if (Mode == LinkingMode::LinkThisFunctionOnly)
+    return true;
+  
   // Try to transitively deserialize everything referenced by this
   // function.
   Worklist.push_back(F);
@@ -69,6 +72,9 @@ bool SILLinkerVisitor::processFunction(StringRef Name) {
 
   ++NumFuncLinked;
   
+  if (Mode == LinkingMode::LinkThisFunctionOnly)
+    return true;
+
   // Try to transitively deserialize everything referenced by NewFn.
   Worklist.push_back(NewFn);
   process();

lib/SIL/SILModule.cpp

@@ -22,6 +22,7 @@
 #include "Linker.h"
 #include "swift/SIL/SILVisitor.h"
 #include "swift/SIL/SILValue.h"
+#include "swift/ClangImporter/ClangModule.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringSwitch.h"
@@ -781,6 +782,23 @@ bool SILModule::isNoReturnBuiltinOrIntrinsic(Identifier Name) {
   }
 }
 
+bool SILModule::
+shouldSerializeEntitiesAssociatedWithDeclContext(const DeclContext *DC) const {
+  // Serialize entities associated with this module's associated context.
+  if (DC->isChildContextOf(getAssociatedContext())) {
+    return true;
+  }
+  
+  // Serialize entities associated with clang modules, since other entities
+  // may depend on them, and someone who deserializes those entities may not
+  // have their own copy.
+  if (isa<ClangModuleUnit>(DC->getModuleScopeContext())) {
+    return true;
+  }
+  
+  return false;
+}
+
 /// Returns true if it is the OnoneSupport module.
 bool SILModule::isOnoneSupportModule() const {
   return getSwiftModule()->getName().str() == SWIFT_ONONE_SUPPORT;

lib/Serialization/SerializeSIL.cpp

@@ -2420,21 +2420,23 @@ void SILSerializer::writeSILBlock(const SILModule *SILMod) {
   assert(assocDC && "cannot serialize SIL without an associated DeclContext");
   for (const SILVTable &vt : SILMod->getVTables()) {
     if ((ShouldSerializeAll || vt.isSerialized()) &&
-        vt.getClass()->isChildContextOf(assocDC))
+        SILMod->shouldSerializeEntitiesAssociatedWithDeclContext(vt.getClass()))
       writeSILVTable(vt);
   }
   
   // Write out property descriptors.
   for (const SILProperty &prop : SILMod->getPropertyList()) {
     if ((ShouldSerializeAll || prop.isSerialized()) &&
-        prop.getDecl()->getInnermostDeclContext()->isChildContextOf(assocDC))
+        SILMod->shouldSerializeEntitiesAssociatedWithDeclContext(
+                                     prop.getDecl()->getInnermostDeclContext()))
       writeSILProperty(prop);
   }
 
   // Write out fragile WitnessTables.
   for (const SILWitnessTable &wt : SILMod->getWitnessTables()) {
     if ((ShouldSerializeAll || wt.isSerialized()) &&
-        wt.getConformance()->getDeclContext()->isChildContextOf(assocDC))
+        SILMod->shouldSerializeEntitiesAssociatedWithDeclContext(
+                                         wt.getConformance()->getDeclContext()))
       writeSILWitnessTable(wt);
   }
 
@@ -2442,7 +2444,8 @@ void SILSerializer::writeSILBlock(const SILModule *SILMod) {
   for (const SILDefaultWitnessTable &wt : SILMod->getDefaultWitnessTables()) {
     // FIXME: Don't need to serialize private and internal default witness
     // tables.
-    if (wt.getProtocol()->getDeclContext()->isChildContextOf(assocDC))
+    if (SILMod->shouldSerializeEntitiesAssociatedWithDeclContext(
+                                                              wt.getProtocol()))
       writeSILDefaultWitnessTable(wt);
   }
 

test/IRGen/Inputs/deserialize-clang-importer-witness-tables/regex.swift

@@ -0,0 +1,24 @@
+import Foundation
+
+public struct RegEx {
+    public let pattern: String
+    fileprivate let regex: NSRegularExpression
+    public typealias Options = NSRegularExpression.Options
+    
+    public init(pattern: String, options: Options = []) throws {
+        self.pattern = pattern
+        self.regex = try NSRegularExpression(pattern: pattern, options: options)
+    }
+    
+    /// Returns a match group for the first match, or nil if there was no match.
+    public func firstMatch(in string: String) -> [String]? {
+        let nsString = string as NSString
+        
+        return regex.firstMatch(in: string, range: NSMakeRange(0, nsString.length)).map { match -> [String] in
+            return (1 ..< match.numberOfRanges).map { idx -> String in
+                let range = match.range(at: idx)
+                return range.location == NSNotFound ? "" : nsString.substring(with: range)
+            }
+        }
+    }
+}

test/IRGen/deserialize-clang-importer-witness-tables.swift

@@ -0,0 +1,18 @@
+// RUN: %empty-directory(%t)
+// RUN: %target-swift-frontend -swift-version 4 -emit-module -o %t/regex.swiftmodule %S/Inputs/deserialize-clang-importer-witness-tables/regex.swift
+// RUN: %target-swift-frontend -swift-version 4 -emit-ir %s -I %t | %FileCheck %s
+
+// REQUIRES: objc_interop
+
+import regex
+
+public func foo(line: String) {
+  // The NSRegularExpressionOptions: SetAlgebra conformance is used indirectly
+  // from the default argument expression passed to `RegEx(pattern:options:)`
+  // below. Ensure that a local copy of the definition was deserialized
+  // and lowered to IR.
+  // CHECK-LABEL: define {{.*}} i8** @"$SSo26NSRegularExpressionOptionsVs10SetAlgebraSCWa"
+  // CHECK-LABEL: define {{.*}} void @"$SSo26NSRegularExpressionOptionsVs10SetAlgebraSCsACPxycfCTW"
+  let versionRegex = try! RegEx(pattern: "Apple")
+  _ = versionRegex.firstMatch(in: line)  
+}