[Serialization] Main deserialization safety logic on the writer side

When writing decls to a swiftmodule files, the serialization logic
evaluates whether the decl will be safe to deserialize. This is inferred
from the access level of the decl, whether it's local, if the module is
built for testing, etc. If the decl in unsafe to deserialize, a record
will be written down before the decl itself in the swiftmodule file.

On the reader side, attempting to deserialize a decl marked as unsafe
raises a deserialization error early. This error is handled by the existing
deserialization recovery logic.

In theory, we want to consider as safe only decls that are actually needed by
the client. Marking as many internal details as possible as unsafe will
prevent more errors. Getting the right scope may require more work in
the future.

lib/Serialization/Serialization.cpp

@@ -70,6 +70,8 @@
 
 #include <vector>
 
+#define DEBUG_TYPE "Serialization"
+
 using namespace swift;
 using namespace swift::serialization;
 using namespace llvm::support;
@@ -3088,6 +3090,171 @@ class Serializer::DeclSerializer : public DeclVisitor<DeclSerializer> {
     }
   }
 
+  /// Determine if \p decl is safe to deserialize when it's public
+  /// or otherwise needed by the client in normal builds, this should usually
+  /// correspond to logic in type-checking ensuring these safe decls don't
+  /// refer to implementation details. We have to be careful not to mark
+  /// anything needed by a client as unsafe as the client will reject reading
+  /// it, but at the same time keep the safety checks precise to avoid
+  /// XRef errors and such.
+  ///
+  /// \p decl should be either an \c ExtensionDecl or a \c ValueDecl.
+  static bool declIsDeserializationSafe(const Decl *decl) {
+    if (auto ext = dyn_cast<ExtensionDecl>(decl)) {
+      // Consider extensions as safe as their extended type.
+      auto nominalType = ext->getExtendedNominal();
+      if (!nominalType ||
+          !declIsDeserializationSafe(nominalType))
+        return false;
+
+      // We can mark the extension unsafe only if it has no public members.
+      auto members = ext->getMembers();
+      int membersCount = 0;
+      auto hasSafeMembers = std::any_of(members.begin(), members.end(),
+        [&membersCount](const Decl *D) -> bool {
+          membersCount ++;
+          if (auto VD = dyn_cast<ValueDecl>(D))
+            return declIsDeserializationSafe(VD);
+          return true;
+        });
+      if (hasSafeMembers)
+        return true;
+
+      // We can mark the extension unsafe only if it has no  public
+      // conformances.
+      auto protocols = ext->getLocalProtocols(
+                                        ConformanceLookupKind::OnlyExplicit);
+      bool hasSafeConformances = std::any_of(protocols.begin(),
+                                             protocols.end(),
+                                             declIsDeserializationSafe);
+      if (hasSafeConformances)
+        return true;
+
+      // Truly empty extensions are safe, it may happen in swiftinterfaces.
+      if (membersCount == 0 && protocols.size() == 0)
+        return true;
+
+      return false;
+    }
+
+    auto value = cast<ValueDecl>(decl);
+
+    // A decl is safe if formally accessible publicly.
+    auto accessScope = value->getFormalAccessScope(/*useDC=*/nullptr,
+                       /*treatUsableFromInlineAsPublic=*/true);
+    if (accessScope.isPublic())
+      return true;
+
+    // Testable allows access to internal details.
+    if (value->getDeclContext()->getParentModule()->isTestingEnabled() &&
+        accessScope.isInternal())
+      return true;
+
+    if (auto accessor = dyn_cast<AccessorDecl>(value))
+      // Accessors are as safe as their storage.
+      if (declIsDeserializationSafe(accessor->getStorage()))
+        return true;
+
+    // Frozen fields are always safe.
+    if (auto var = dyn_cast<VarDecl>(value)) {
+      if (var->isLayoutExposedToClients())
+        return true;
+
+      // Consider all lazy var storage as "safe".
+      // FIXME: We should keep track of what lazy var is associated to the
+      //        storage for them to preserve the same safeness.
+      if (var->isLazyStorageProperty())
+        return true;
+
+      // Property wrappers storage is as safe as the wrapped property.
+      if (VarDecl *wrapped = var->getOriginalWrappedProperty())
+        if (declIsDeserializationSafe(wrapped))
+          return true;
+    }
+
+    return false;
+  }
+
+  /// Write a \c DeserializationSafetyLayout record only when \p decl is unsafe
+  /// to deserialize.
+  ///
+  /// \sa declIsDeserializationSafe
+  void writeDeserializationSafety(const Decl *decl) {
+    using namespace decls_block;
+
+    auto DC = decl->getDeclContext();
+    if (!DC->getParentModule()->isResilient())
+      return;
+
+    // Everything should be safe in a swiftinterface. So, don't emit any safety
+    // record when building a swiftinterface in release builds. Debug builds
+    // instead print inconsistencies.
+    auto parentSF = DC->getParentSourceFile();
+    bool fromModuleInterface = parentSF &&
+                               parentSF->Kind == SourceFileKind::Interface;
+#if NDEBUG
+    if (fromModuleInterface)
+      return;
+#endif
+
+    // Private imports allow safe access to everything.
+    if (DC->getParentModule()->arePrivateImportsEnabled())
+      return;
+
+    // Ignore things with no access level.
+    // Note: There's likely room to report some of these as unsafe to prevent
+    //       failures.
+    if (isa<GenericTypeParamDecl>(decl) ||
+        isa<OpaqueTypeDecl>(decl) ||
+        isa<ParamDecl>(decl) ||
+        isa<EnumCaseDecl>(decl) ||
+        isa<EnumElementDecl>(decl))
+      return;
+
+    if (!isa<ValueDecl>(decl) && !isa<ExtensionDecl>(decl))
+      return;
+
+    // Don't look at decls inside functions and
+    // check the ValueDecls themselves.
+    auto declIsSafe = DC->isLocalContext() ||
+                      declIsDeserializationSafe(decl);
+#ifdef NDEBUG
+    // In release builds, bail right away if the decl is safe.
+    // In debug builds, wait to bail after the debug prints and asserts.
+    if (declIsSafe)
+      return;
+#endif
+
+    // Write a human readable name to an identifier.
+    SmallString<64> out;
+    llvm::raw_svector_ostream outStream(out);
+    if (auto val = dyn_cast<ValueDecl>(decl)) {
+      outStream << val->getName();
+    } else if (auto ext = dyn_cast<ExtensionDecl>(decl)) {
+      outStream << "extension ";
+      if (auto nominalType = ext->getExtendedNominal())
+        outStream << nominalType->getName();
+    }
+    auto name = S.getASTContext().getIdentifier(out);
+
+    LLVM_DEBUG(
+      llvm::dbgs() << "Serialization safety, "
+                   << (declIsSafe? "safe" : "unsafe")
+                   << ": '" << name << "'\n";
+      assert((declIsSafe || !fromModuleInterface) &&
+             "All swiftinterface decls should be deserialization safe");
+    );
+
+#ifndef NDEBUG
+    if (declIsSafe)
+      return;
+#endif
+
+    auto abbrCode = S.DeclTypeAbbrCodes[DeserializationSafetyLayout::Code];
+    DeserializationSafetyLayout::emitRecord(S.Out, S.ScratchRecord, abbrCode,
+                                            S.addDeclBaseNameRef(name));
+  }
+
   void writeForeignErrorConvention(const ForeignErrorConvention &fec) {
     using namespace decls_block;
 
@@ -3405,6 +3572,8 @@ public:
     if (D->isInvalid())
       writeDeclErrorFlag();
 
+    writeDeserializationSafety(D);
+
     // Emit attributes (if any).
     for (auto Attr : D->getAttrs())
       writeDeclAttribute(D, Attr);

test/Serialization/Safety/unsafe-decls.swift

@@ -0,0 +1,93 @@
+// RUN: %empty-directory(%t)
+
+// REQUIRES: asserts
+
+// RUN: %target-swift-frontend -emit-module %s \
+// RUN:   -enable-library-evolution -swift-version 5 \
+// RUN:   -enable-deserialization-safety \
+// RUN:   -Xllvm -debug-only=Serialization 2>&1 \
+// RUN:   | %FileCheck --check-prefixes=SAFETY-PRIVATE,SAFETY-INTERNAL %s
+
+// RUN: %target-swift-frontend -emit-module %s \
+// RUN:   -enable-library-evolution -swift-version 5 \
+// RUN:   -enable-deserialization-safety \
+// RUN:   -Xllvm -debug-only=Serialization \
+// RUN:   -enable-testing 2>&1 \
+// RUN:   | %FileCheck --check-prefixes=SAFETY-PRIVATE,NO-SAFETY-INTERNAL-NOT %s
+
+/// Don't mark decls as unsafe when private import is enabled.
+// RUN: %target-swift-frontend -emit-module %s \
+// RUN:   -enable-library-evolution -swift-version 5 \
+// RUN:   -enable-deserialization-safety \
+// RUN:   -Xllvm -debug-only=Serialization \
+// RUN:   -enable-private-imports 2>&1 \
+// RUN:   | %FileCheck --check-prefixes=DISABLED %s
+
+/// Don't mark decls as unsafe without library evolution.
+// RUN: %target-swift-frontend -emit-module %s \
+// RUN:   -enable-deserialization-safety -swift-version 5 \
+// RUN:   -Xllvm -debug-only=Serialization 2>&1 \
+// RUN:   | %FileCheck --check-prefixes=DISABLED %s
+
+// DISABLED-NOT: Serialization safety
+
+public protocol PublicProto {}
+// SAFETY-PRIVATE: Serialization safety, safe: 'PublicProto'
+internal protocol InternalProto {}
+// SAFETY-INTERNAL: Serialization safety, unsafe: 'InternalProto'
+// NO-SAFETY-INTERNAL: Serialization safety, safe: 'InternalProto'
+private protocol PrivateProto {}
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'PrivateProto'
+fileprivate protocol FileprivateProto {}
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'FileprivateProto'
+
+internal struct InternalStruct : PublicProto {
+// SAFETY-INTERNAL: Serialization safety, unsafe: 'InternalStruct'
+// NO-SAFETY-INTERNAL: Serialization safety, safe: 'InternalStruct'
+}
+
+public struct PublicStruct {
+// SAFETY-PRIVATE: Serialization safety, safe: 'PublicStruct'
+
+    public typealias publicTypealias = Int
+// SAFETY-PRIVATE: Serialization safety, safe: 'publicTypealias'
+    typealias internalTypealias = Int
+// SAFETY-INTERNAL: Serialization safety, unsafe: 'internalTypealias'
+// NO-SAFETY-INTERNAL: Serialization safety, safe: 'internalTypealias'
+// SAFETY-PRIVATE: Serialization safety, safe: 'localTypealias'
+
+    public init(publicInit a: Int) {}
+// SAFETY-PRIVATE: Serialization safety, safe: 'init(publicInit:)'
+    internal init(internalInit a: Int) {}
+// SAFETY-INTERNAL: Serialization safety, unsafe: 'init(internalInit:)'
+// NO-SAFETY-INTERNAL: Serialization safety, safe: 'init(internalInit:)'
+    private init(privateInit a: Int) {}
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'init(privateInit:)'
+    fileprivate init(fileprivateInit a: Int) {}
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'init(fileprivateInit:)'
+
+    @inlinable public func inlinableFunc() {
+        typealias localTypealias = Int
+    }
+// SAFETY-PRIVATE: Serialization safety, safe: 'inlinableFunc()'
+    public func publicFunc() {}
+// SAFETY-PRIVATE: Serialization safety, safe: 'publicFunc()'
+    public func opaqueTypeFunc() -> some PublicProto {
+        return InternalStruct()
+    }
+// SAFETY-PRIVATE: Serialization safety, safe: 'opaqueTypeFunc()'
+    internal func internalFunc() {}
+// SAFETY-INTERNAL: Serialization safety, unsafe: 'internalFunc()'
+// NO-SAFETY-INTERNAL: Serialization safety, safe: 'internalFunc()'
+    private func privateFunc() {}
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'privateFunc()'
+    fileprivate func fileprivateFunc() {}
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'fileprivateFunc()'
+
+    public var publicProperty = "Some text"
+// SAFETY-PRIVATE: Serialization safety, safe: 'publicProperty'
+    public private(set) var halfPublicProperty = "Some text"
+// SAFETY-PRIVATE: Serialization safety, safe: 'halfPublicProperty'
+    private var privateProperty = "Some text"
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'privateProperty'
+}

test/Serialization/Safety/unsafe-extensions.swift

@@ -0,0 +1,137 @@
+// RUN: %empty-directory(%t)
+
+// REQUIRES: asserts
+
+// RUN: %target-swift-frontend -emit-module %s \
+// RUN:   -enable-library-evolution -swift-version 5 \
+// RUN:   -enable-deserialization-safety \
+// RUN:   -Xllvm -debug-only=Serialization 2>&1 \
+// RUN:   | %FileCheck --check-prefixes=SAFETY-PRIVATE,SAFETY-INTERNAL %s
+
+// RUN: %target-swift-frontend -emit-module %s \
+// RUN:   -enable-library-evolution -swift-version 5 \
+// RUN:   -enable-deserialization-safety \
+// RUN:   -Xllvm -debug-only=Serialization \
+// RUN:   -enable-testing 2>&1 \
+// RUN:   | %FileCheck --check-prefixes=SAFETY-PRIVATE,NO-SAFETY-INTERNAL %s
+
+/// Don't mark decls as unsafe when private import is enabled.
+// RUN: %target-swift-frontend -emit-module %s \
+// RUN:   -enable-library-evolution -swift-version 5 \
+// RUN:   -enable-deserialization-safety \
+// RUN:   -Xllvm -debug-only=Serialization \
+// RUN:   -enable-private-imports 2>&1 \
+// RUN:   | %FileCheck --check-prefixes=DISABLED %s
+
+/// Don't mark decls as unsafe without library evolution.
+// RUN: %target-swift-frontend -emit-module %s \
+// RUN:   -enable-deserialization-safety -swift-version 5 \
+// RUN:   -Xllvm -debug-only=Serialization 2>&1 \
+// RUN:   | %FileCheck --check-prefixes=DISABLED %s
+
+// DISABLED-NOT: Serialization safety
+
+/// Public
+// SAFETY-PRIVATE: Serialization safety, safe: 'PublicProto'
+public protocol PublicProto {}
+public struct ExtendedPublic {
+// SAFETY-PRIVATE: Serialization safety, safe: 'ExtendedPublic'
+}
+extension ExtendedPublic {
+// SAFETY-PRIVATE: Serialization safety, safe: 'extension ExtendedPublic'
+    public func publicMember() {}
+}
+extension ExtendedPublic : PublicProto {
+// SAFETY-PRIVATE: Serialization safety, safe: 'extension ExtendedPublic'
+}
+
+/// Internal
+internal protocol InternalProto {}
+// SAFETY-INTERNAL: Serialization safety, unsafe: 'InternalProto'
+// NO-SAFETY-INTERNAL: Serialization safety, safe: 'InternalProto'
+internal struct ExtendedInternal {}
+// SAFETY-INTERNAL: Serialization safety, unsafe: 'ExtendedInternal'
+// NO-SAFETY-INTERNAL: Serialization safety, safe: 'ExtendedInternal'
+extension ExtendedInternal {
+// SAFETY-INTERNAL: Serialization safety, unsafe: 'extension ExtendedInternal'
+// NO-SAFETY-INTERNAL: Serialization safety, safe: 'extension ExtendedInternal'
+    internal func internalMember() {}
+}
+extension ExtendedInternal : InternalProto {}
+// SAFETY-INTERNAL: Serialization safety, unsafe: 'extension ExtendedInternal'
+// NO-SAFETY-INTERNAL: Serialization safety, safe: 'extension ExtendedInternal'
+
+/// Private
+private protocol PrivateProto {}
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'PrivateProto'
+private struct ExtendedPrivate {}
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'ExtendedPrivate'
+extension ExtendedPrivate {
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'extension ExtendedPrivate'
+    private func privateMember() {}
+}
+extension ExtendedPrivate : PrivateProto {}
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'extension ExtendedPrivate'
+
+/// Fileprivate
+private protocol FileprivateProto {}
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'FileprivateProto'
+private struct ExtendedFileprivate {}
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'ExtendedFileprivate'
+extension ExtendedFileprivate {
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'extension ExtendedFileprivate'
+    private func fileprivateMember() {}
+}
+extension ExtendedFileprivate : FileprivateProto {}
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'extension ExtendedFileprivate'
+
+/// Back to public
+extension ExtendedPublic : InternalProto {
+// SAFETY-INTERNAL: Serialization safety, unsafe: 'extension ExtendedPublic'
+// NO-SAFETY-INTERNAL: Serialization safety, safe: 'extension ExtendedPublic'
+}
+extension ExtendedPublic : PrivateProto {
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'extension ExtendedPublic'
+}
+extension ExtendedPublic : FileprivateProto {
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'extension ExtendedPublic'
+}
+
+extension ExtendedPublic {
+// SAFETY-INTERNAL: Serialization safety, unsafe: 'extension ExtendedPublic'
+// NO-SAFETY-INTERNAL: Serialization safety, safe: 'extension ExtendedPublic'
+    internal func internalMemberToPublic() {}
+}
+
+extension ExtendedPublic {
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'extension ExtendedPublic'
+    private func privateMemberToPublic() {}
+}
+
+extension ExtendedPublic {
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'extension ExtendedPublic'
+    fileprivate func fileprivateMemberToPublic() {}
+}
+
+extension ExtendedInternal {
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'extension ExtendedInternal'
+    private func privateMemberToInternal() {}
+}
+
+extension ExtendedInternal {
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'extension ExtendedInternal'
+    fileprivate func fileprivateMemberToInternal() {}
+}
+
+/// Members are serialized last
+// SAFETY-INTERNAL: Serialization safety, unsafe: 'internalMember()'
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'privateMember()'
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'fileprivateMember()'
+
+// SAFETY-INTERNAL: Serialization safety, unsafe: 'internalMemberToPublic()'
+// NO-SAFETY-INTERNAL: Serialization safety, safe: 'internalMemberToPublic()'
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'privateMemberToPublic()'
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'fileprivateMemberToPublic()'
+
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'privateMemberToInternal()'
+// SAFETY-PRIVATE: Serialization safety, unsafe: 'fileprivateMemberToInternal()'