Merge pull request #31553 from nkcsgexi/62612027

DependencyScanner: honor additional compiler flags in interfaces files when collecting imports
This commit is contained in:
Xi Ge
2020-05-05 13:52:20 -07:00
committed by GitHub
16 changed files with 220 additions and 77 deletions

View File

@@ -121,6 +121,7 @@ namespace swift {
class UnifiedStatsReporter;
class IndexSubset;
struct SILAutoDiffDerivativeFunctionKey;
struct SubASTContextDelegate;
enum class KnownProtocolKind : uint8_t;
@@ -719,7 +720,8 @@ public:
Optional<ModuleDependencies> getModuleDependencies(
StringRef moduleName,
bool isUnderlyingClangModule,
ModuleDependenciesCache &cache);
ModuleDependenciesCache &cache,
SubASTContextDelegate &delegate);
/// Load extensions to the given nominal type from the external
/// module loaders.

View File

@@ -25,6 +25,7 @@
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/ADT/TinyPtrVector.h"
#include "swift/AST/ModuleDependencies.h"
namespace llvm {
class FileCollector;
@@ -84,6 +85,15 @@ public:
std::shared_ptr<clang::DependencyCollector> getClangCollector();
};
/// Abstract interface to run an action in a sub ASTContext.
struct SubASTContextDelegate {
virtual bool runInSubContext(ASTContext &ctx, StringRef interfacePath,
llvm::function_ref<bool(ASTContext&)> action) {
llvm_unreachable("function should be overriden");
}
virtual ~SubASTContextDelegate() = default;
};
/// Abstract interface that loads named modules into the AST.
class ModuleLoader {
virtual void anchor();
@@ -186,7 +196,7 @@ public:
/// if no such module exists.
virtual Optional<ModuleDependencies> getModuleDependencies(
StringRef moduleName,
ModuleDependenciesCache &cache) = 0;
ModuleDependenciesCache &cache, SubASTContextDelegate &delegate) = 0;
};
} // namespace swift

View File

@@ -370,7 +370,8 @@ public:
void verifyAllModules() override;
Optional<ModuleDependencies> getModuleDependencies(
StringRef moduleName, ModuleDependenciesCache &cache) override;
StringRef moduleName, ModuleDependenciesCache &cache,
SubASTContextDelegate &delegate) override;
/// Add dependency information for the bridging header.
///

View File

@@ -110,6 +110,7 @@
#include "swift/Basic/LLVM.h"
#include "swift/Frontend/ModuleInterfaceSupport.h"
#include "swift/Serialization/SerializedModuleLoader.h"
#include "llvm/Support/StringSaver.h"
namespace clang {
class CompilerInstance;
@@ -123,6 +124,7 @@ namespace swift {
class LangOptions;
class SearchPathOptions;
class CompilerInvocation;
/// A ModuleLoader that runs a subordinate \c CompilerInvocation and
/// \c CompilerInstance to convert .swiftinterface files to .swiftmodule
@@ -201,6 +203,18 @@ public:
std::string
getModuleCachePathFromClang(const clang::CompilerInstance &Instance);
bool extractSwiftInterfaceVersionAndArgs(SourceManager &SM,
DiagnosticEngine &Diags,
StringRef InterfacePath,
version::Version &Vers,
StringRef &CompilerVersion,
llvm::StringSaver &SubArgSaver,
SmallVectorImpl<const char *> &SubArgs,
SourceLoc diagnosticLoc = SourceLoc());
void inheritOptionsForBuildingInterface(CompilerInvocation &Invok,
const SearchPathOptions &SearchPathOpts,
const LangOptions &LangOpts);
}
#endif

View File

@@ -93,7 +93,8 @@ public:
}
Optional<ModuleDependencies> getModuleDependencies(
StringRef moduleName, ModuleDependenciesCache &cache) override {
StringRef moduleName, ModuleDependenciesCache &cache,
SubASTContextDelegate &delegate) override {
// FIXME: Implement?
return None;
}

View File

@@ -194,7 +194,8 @@ public:
virtual void verifyAllModules() override;
virtual Optional<ModuleDependencies> getModuleDependencies(
StringRef moduleName, ModuleDependenciesCache &cache) override;
StringRef moduleName, ModuleDependenciesCache &cache,
SubASTContextDelegate &delegate) override;
};
/// Imports serialized Swift modules into an ASTContext.

View File

@@ -1462,13 +1462,14 @@ void ASTContext::addModuleLoader(std::unique_ptr<ModuleLoader> loader,
Optional<ModuleDependencies> ASTContext::getModuleDependencies(
StringRef moduleName, bool isUnderlyingClangModule,
ModuleDependenciesCache &cache) {
ModuleDependenciesCache &cache, SubASTContextDelegate &delegate) {
for (auto &loader : getImpl().ModuleLoaders) {
if (isUnderlyingClangModule &&
loader.get() != getImpl().TheClangModuleLoader)
continue;
if (auto dependencies = loader->getModuleDependencies(moduleName, cache))
if (auto dependencies = loader->getModuleDependencies(moduleName, cache,
delegate))
return dependencies;
}

View File

@@ -224,7 +224,8 @@ static void recordModuleDependencies(
}
Optional<ModuleDependencies> ClangImporter::getModuleDependencies(
StringRef moduleName, ModuleDependenciesCache &cache) {
StringRef moduleName, ModuleDependenciesCache &cache,
SubASTContextDelegate &delegate) {
// Check whether there is already a cached result.
if (auto found = cache.findDependencies(
moduleName, ModuleDependenciesKind::Clang))

View File

@@ -12,6 +12,7 @@
#define DEBUG_TYPE "textual-module-interface"
#include "swift/Frontend/ModuleInterfaceLoader.h"
#include "ModuleInterfaceBuilder.h"
#include "swift/AST/ASTContext.h"
#include "swift/AST/DiagnosticsFrontend.h"
@@ -82,19 +83,37 @@ void ModuleInterfaceBuilder::configureSubInvocationInputsAndOutputs(
.setMainAndSupplementaryOutputs({MainOut}, {SOPs});
}
void swift::inheritOptionsForBuildingInterface(
CompilerInvocation &Invok,
const SearchPathOptions &SearchPathOpts,
const LangOptions &LangOpts) {
// Start with a SubInvocation that copies various state from our
// invoking ASTContext.
Invok.setImportSearchPaths(SearchPathOpts.ImportSearchPaths);
Invok.setFrameworkSearchPaths(SearchPathOpts.FrameworkSearchPaths);
Invok.setSDKPath(SearchPathOpts.SDKPath);
Invok.setInputKind(InputFileKind::SwiftModuleInterface);
Invok.setRuntimeResourcePath(SearchPathOpts.RuntimeResourcePath);
Invok.setTargetTriple(LangOpts.Target);
// Inhibit warnings from the SubInvocation since we are assuming the user
// is not in a position to fix them.
Invok.getDiagnosticOptions().SuppressWarnings = true;
// Inherit this setting down so that it can affect error diagnostics (mostly
// by making them non-fatal).
Invok.getLangOptions().DebuggerSupport = LangOpts.DebuggerSupport;
// Disable this; deinitializers always get printed with `@objc` even in
// modules that don't import Foundation.
Invok.getLangOptions().EnableObjCAttrRequiresFoundation = false;
}
void ModuleInterfaceBuilder::configureSubInvocation(
const SearchPathOptions &SearchPathOpts,
const LangOptions &LangOpts,
ClangModuleLoader *ClangLoader) {
// Start with a SubInvocation that copies various state from our
// invoking ASTContext.
subInvocation.setImportSearchPaths(SearchPathOpts.ImportSearchPaths);
subInvocation.setFrameworkSearchPaths(SearchPathOpts.FrameworkSearchPaths);
subInvocation.setSDKPath(SearchPathOpts.SDKPath);
subInvocation.setInputKind(InputFileKind::SwiftModuleInterface);
subInvocation.setRuntimeResourcePath(SearchPathOpts.RuntimeResourcePath);
subInvocation.setTargetTriple(LangOpts.Target);
inheritOptionsForBuildingInterface(subInvocation, SearchPathOpts, LangOpts);
subInvocation.setModuleName(moduleName);
subInvocation.setClangModuleCachePath(moduleCachePath);
subInvocation.getFrontendOptions().PrebuiltModuleCachePath =
@@ -111,18 +130,6 @@ void ModuleInterfaceBuilder::configureSubInvocation(
}
}
// Inhibit warnings from the SubInvocation since we are assuming the user
// is not in a position to fix them.
subInvocation.getDiagnosticOptions().SuppressWarnings = true;
// Inherit this setting down so that it can affect error diagnostics (mostly
// by making them non-fatal).
subInvocation.getLangOptions().DebuggerSupport = LangOpts.DebuggerSupport;
// Disable this; deinitializers always get printed with `@objc` even in
// modules that don't import Foundation.
subInvocation.getLangOptions().EnableObjCAttrRequiresFoundation = false;
// Tell the subinvocation to serialize dependency hashes if asked to do so.
auto &frontendOpts = subInvocation.getFrontendOptions();
frontendOpts.SerializeModuleInterfaceDependencyHashes =
@@ -134,16 +141,22 @@ void ModuleInterfaceBuilder::configureSubInvocation(
remarkOnRebuildFromInterface;
}
bool ModuleInterfaceBuilder::extractSwiftInterfaceVersionAndArgs(
swift::version::Version &Vers, StringRef &CompilerVersion,
llvm::StringSaver &SubArgSaver, SmallVectorImpl<const char *> &SubArgs) {
llvm::vfs::FileSystem &fs = *sourceMgr.getFileSystem();
auto FileOrError = swift::vfs::getFileOrSTDIN(fs, interfacePath);
bool swift::extractSwiftInterfaceVersionAndArgs(
SourceManager &SM,
DiagnosticEngine &Diags,
StringRef InterfacePath,
version::Version &Vers,
StringRef &CompilerVersion,
llvm::StringSaver &SubArgSaver,
SmallVectorImpl<const char *> &SubArgs,
SourceLoc diagnosticLoc) {
llvm::vfs::FileSystem &fs = *SM.getFileSystem();
auto FileOrError = swift::vfs::getFileOrSTDIN(fs, InterfacePath);
if (!FileOrError) {
// Don't use this->diagnose() because it'll just try to re-open
// interfacePath.
diags.diagnose(diagnosticLoc, diag::error_open_input_file,
interfacePath, FileOrError.getError().message());
Diags.diagnose(diagnosticLoc, diag::error_open_input_file,
InterfacePath, FileOrError.getError().message());
return true;
}
auto SB = FileOrError.get()->getBuffer();
@@ -151,18 +164,21 @@ bool ModuleInterfaceBuilder::extractSwiftInterfaceVersionAndArgs(
auto CompRe = getSwiftInterfaceCompilerVersionRegex();
auto FlagRe = getSwiftInterfaceModuleFlagsRegex();
SmallVector<StringRef, 1> VersMatches, FlagMatches, CompMatches;
if (!VersRe.match(SB, &VersMatches)) {
diagnose(diag::error_extracting_version_from_module_interface);
ModuleInterfaceBuilder::diagnose(Diags, SM, InterfacePath, diagnosticLoc,
diag::error_extracting_version_from_module_interface);
return true;
}
if (!FlagRe.match(SB, &FlagMatches)) {
diagnose(diag::error_extracting_flags_from_module_interface);
ModuleInterfaceBuilder::diagnose(Diags, SM, InterfacePath, diagnosticLoc,
diag::error_extracting_version_from_module_interface);
return true;
}
assert(VersMatches.size() == 2);
assert(FlagMatches.size() == 2);
// FIXME We should diagnose this at a location that makes sense:
Vers = swift::version::Version(VersMatches[1], SourceLoc(), &diags);
Vers = swift::version::Version(VersMatches[1], SourceLoc(), &Diags);
llvm::cl::TokenizeGNUCommandLine(FlagMatches[1], SubArgSaver, SubArgs);
if (CompRe.match(SB, &CompMatches)) {
@@ -177,6 +193,13 @@ bool ModuleInterfaceBuilder::extractSwiftInterfaceVersionAndArgs(
return false;
}
bool ModuleInterfaceBuilder::extractSwiftInterfaceVersionAndArgs(
swift::version::Version &Vers, StringRef &CompilerVersion,
llvm::StringSaver &SubArgSaver, SmallVectorImpl<const char *> &SubArgs) {
return swift::extractSwiftInterfaceVersionAndArgs(sourceMgr, diags,
interfacePath, Vers, CompilerVersion, SubArgSaver, SubArgs, diagnosticLoc);
}
bool ModuleInterfaceBuilder::collectDepsForSerialization(
CompilerInstance &SubInstance, SmallVectorImpl<FileDependency> &Deps,
bool IsHashBased) {

View File

@@ -48,17 +48,31 @@ class ModuleInterfaceBuilder {
CompilerInvocation subInvocation;
SmallVector<StringRef, 3> extraDependencies;
public:
/// Emit a diagnostic tied to this declaration.
template<typename ...ArgTypes>
static InFlightDiagnostic diagnose(
DiagnosticEngine &Diags,
SourceManager &SM,
StringRef InterfacePath,
SourceLoc Loc,
Diag<ArgTypes...> ID,
typename detail::PassArgument<ArgTypes>::type... Args) {
if (Loc.isInvalid()) {
// Diagnose this inside the interface file, if possible.
Loc = SM.getLocFromExternalSource(InterfacePath, 1, 1);
}
return Diags.diagnose(Loc, ID, std::move(Args)...);
}
private:
/// Emit a diagnostic tied to this declaration.
template<typename ...ArgTypes>
InFlightDiagnostic diagnose(
Diag<ArgTypes...> ID,
typename detail::PassArgument<ArgTypes>::type... Args) const {
SourceLoc loc = diagnosticLoc;
if (loc.isInvalid()) {
// Diagnose this inside the interface file, if possible.
loc = sourceMgr.getLocFromExternalSource(interfacePath, 1, 1);
}
return diags.diagnose(loc, ID, std::move(Args)...);
return diagnose(diags, sourceMgr, interfacePath, diagnosticLoc,
ID, std::move(Args)...);
}
void configureSubInvocationInputsAndOutputs(StringRef OutPath);

View File

@@ -24,6 +24,7 @@
#include "swift/Basic/STLExtras.h"
#include "swift/Frontend/Frontend.h"
#include "swift/Frontend/FrontendOptions.h"
#include "swift/Frontend/ModuleInterfaceLoader.h"
#include "swift/Strings.h"
#include "clang/Basic/Module.h"
#include "llvm/ADT/SetVector.h"
@@ -58,13 +59,51 @@ static void findAllImportedClangModules(ASTContext &ctx, StringRef moduleName,
}
}
struct InterfaceSubASTContextDelegate: SubASTContextDelegate {
bool runInSubContext(ASTContext &ctx, StringRef interfacePath,
llvm::function_ref<bool(ASTContext&)> action) override {
// Parse the interface file using the current context to get the additional
// compiler arguments we should use when creating a sub-ASTContext.
// These arguments are in "swift-module-flags:"
version::Version Vers;
StringRef CompilerVersion;
llvm::BumpPtrAllocator Allocator;
llvm::StringSaver SubArgSaver(Allocator);
SmallVector<const char *, 64> SubArgs;
if (extractSwiftInterfaceVersionAndArgs(ctx.SourceMgr, ctx.Diags,
interfacePath, Vers, CompilerVersion,
SubArgSaver, SubArgs)) {
return true;
}
CompilerInvocation invok;
// Inherit options from the parent ASTContext so we have all search paths, etc.
inheritOptionsForBuildingInterface(invok, ctx.SearchPathOpts, ctx.LangOpts);
CompilerInstance inst;
// Use the additional flags to setup the compiler instance.
if (invok.parseArgs(SubArgs, ctx.Diags)) {
return true;
}
if (inst.setup(invok)) {
return true;
}
// Add the diag consumers to the sub context to make sure we don't lose
// diagnostics.
for (auto *consumer: ctx.Diags.getConsumers()) {
inst.getDiags().addConsumer(*consumer);
}
// Run the action under the sub-ASTContext.
return action(inst.getASTContext());
}
};
/// Resolve the direct dependencies of the given module.
static std::vector<ModuleDependencyID> resolveDirectDependencies(
ASTContext &ctx, ModuleDependencyID module,
ModuleDependenciesCache &cache) {
auto knownDependencies = *cache.findDependencies(module.first, module.second);
auto isSwift = knownDependencies.isSwiftModule();
InterfaceSubASTContextDelegate ASTDelegate;
// Find the dependencies of every module this module directly depends on.
std::vector<ModuleDependencyID> result;
for (auto dependsOn : knownDependencies.getModuleDependencies()) {
@@ -73,7 +112,7 @@ static std::vector<ModuleDependencyID> resolveDirectDependencies(
// Retrieve the dependencies for this module.
if (auto found = ctx.getModuleDependencies(
dependsOn, onlyClangModule, cache)) {
dependsOn, onlyClangModule, cache, ASTDelegate)) {
result.push_back({dependsOn, found->getKind()});
}
}
@@ -115,7 +154,7 @@ static std::vector<ModuleDependencyID> resolveDirectDependencies(
// directly depends on these.
for (const auto &clangDep : allClangModules) {
if (auto found = ctx.getModuleDependencies(
clangDep, /*onlyClangModule=*/false, cache)) {
clangDep, /*onlyClangModule=*/false, cache, ASTDelegate)) {
if (found->getKind() == ModuleDependenciesKind::Swift)
result.push_back({clangDep, found->getKind()});
}

View File

@@ -33,14 +33,15 @@ class ModuleDependencyScanner : public SerializedModuleLoaderBase {
ErrorOr<ModuleDependencies> scanInterfaceFile(
Twine moduleInterfacePath);
SubASTContextDelegate &astDelegate;
public:
Optional<ModuleDependencies> dependencies;
ModuleDependencyScanner(ASTContext &ctx, ModuleLoadingMode LoadMode,
Identifier moduleName)
Identifier moduleName, SubASTContextDelegate &astDelegate)
: SerializedModuleLoaderBase(ctx, nullptr, LoadMode,
/*IgnoreSwiftSourceInfoFile=*/true),
moduleName(moduleName) { }
moduleName(moduleName), astDelegate(astDelegate) { }
virtual std::error_code findModuleFilesInDirectory(
AccessPathElem ModuleID,
@@ -94,42 +95,47 @@ public:
ErrorOr<ModuleDependencies> ModuleDependencyScanner::scanInterfaceFile(
Twine moduleInterfacePath) {
// Open the interface file.
auto &fs = *Ctx.SourceMgr.getFileSystem();
auto interfaceBuf = fs.getBufferForFile(moduleInterfacePath);
if (!interfaceBuf)
return interfaceBuf.getError();
// Create a source file.
unsigned bufferID = Ctx.SourceMgr.addNewSourceBuffer(std::move(interfaceBuf.get()));
auto moduleDecl = ModuleDecl::create(moduleName, Ctx);
auto sourceFile = new (Ctx) SourceFile(
*moduleDecl, SourceFileKind::Interface, bufferID);
// Create a module filename.
// FIXME: Query the module interface loader to determine an appropriate
// name for the module, which includes an appropriate hash.
auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile);
llvm::SmallString<32> modulePath = moduleName.str();
llvm::sys::path::replace_extension(modulePath, newExt);
// Walk the source file to find the import declarations.
llvm::StringSet<> alreadyAddedModules;
auto dependencies = ModuleDependencies::forSwiftInterface(
ModuleDependencies Result = ModuleDependencies::forSwiftInterface(
modulePath.str().str(), moduleInterfacePath.str());
std::error_code code;
auto hasError = astDelegate.runInSubContext(Ctx,
moduleInterfacePath.str(),
[&](ASTContext &Ctx) {
// Open the interface file.
auto &fs = *Ctx.SourceMgr.getFileSystem();
auto interfaceBuf = fs.getBufferForFile(moduleInterfacePath);
if (!interfaceBuf) {
code = interfaceBuf.getError();
return true;
}
// FIXME: Suppressing diagnostics here is incorrect, and is currently
// used to paper over the fact that we should be creating a fresh
// ASTContext using the command-line arguments from the .swiftinterface
// file.
DiagnosticSuppression suppression(Ctx.Diags);
dependencies.addModuleDependencies(*sourceFile, alreadyAddedModules);
return dependencies;
// Create a source file.
unsigned bufferID = Ctx.SourceMgr.addNewSourceBuffer(std::move(interfaceBuf.get()));
auto moduleDecl = ModuleDecl::create(moduleName, Ctx);
auto sourceFile = new (Ctx) SourceFile(
*moduleDecl, SourceFileKind::Interface, bufferID);
// Walk the source file to find the import declarations.
llvm::StringSet<> alreadyAddedModules;
Result.addModuleDependencies(*sourceFile, alreadyAddedModules);
return false;
});
if (hasError) {
return code;
}
return Result;
}
Optional<ModuleDependencies> SerializedModuleLoaderBase::getModuleDependencies(
StringRef moduleName, ModuleDependenciesCache &cache) {
StringRef moduleName, ModuleDependenciesCache &cache,
SubASTContextDelegate &delegate) {
// Check whether we've cached this result.
if (auto found = cache.findDependencies(
moduleName, ModuleDependenciesKind::Swift))
@@ -137,7 +143,7 @@ Optional<ModuleDependencies> SerializedModuleLoaderBase::getModuleDependencies(
// Check whether there is a module with this name that we can import.
auto moduleId = Ctx.getIdentifier(moduleName);
ModuleDependencyScanner scanner(Ctx, LoadMode, moduleId);
ModuleDependencyScanner scanner(Ctx, LoadMode, moduleId, delegate);
if (!scanner.canImportModule({moduleId, SourceLoc()}))
return None;

View File

@@ -0,0 +1 @@
void funcG(void);

View File

@@ -22,3 +22,8 @@ module F {
header "F.h"
export *
}
module G {
header "G.h"
export *
}

View File

@@ -0,0 +1,9 @@
// swift-interface-format-version: 1.0
// swift-module-flags: -module-name G -swift-version 5
#if swift(>=5.0)
@_exported import G
public func overlayFuncG() { }
#endif

View File

@@ -1,6 +1,6 @@
// RUN: %empty-directory(%t)
// RUN: mkdir -p %t/clang-module-cache
// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/Inputs/CHeaders/Bridging.h
// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4
// Check the contents of the JSON output
// RUN: %FileCheck %s < %t/deps.json
@@ -20,6 +20,7 @@
import C
import E
import G
// CHECK: "mainModuleName": "deps"
@@ -36,6 +37,9 @@ import E
// CHECK-NEXT: "swift": "E"
// CHECK-NEXT: }
// CHECK-NEXT: {
// CHECK-NEXT: "swift": "G"
// CHECK-NEXT: }
// CHECK-NEXT: {
// CHECK-NEXT: "swift": "Swift"
// CHECK-NEXT: }
// CHECK-NEXT: {
@@ -90,6 +94,16 @@ import E
// CHECK: "moduleInterfacePath"
// CHECK-SAME: E.swiftinterface
/// --------Swift module G
// CHECK-LABEL: "modulePath": "G.swiftmodule"
// CHECK: "directDependencies"
// CHECK-NEXT: {
// CHECK-NEXT: "clang": "G"
// CHECK-NEXT: },
// CHECK-NEXT: {
// CHECK-NEXT: "swift": "G"
// CHECK-NEXT: }
/// --------Swift module Swift
// CHECK-LABEL: "modulePath": "Swift.swiftmodule",
@@ -124,6 +138,7 @@ import E
// Check make-style dependencies
// CHECK-MAKE-DEPS: module_deps.swift
// CHECK-MAKE-DEPS-SAME: A.swiftinterface
// CHECK-MAKE-DEPS-SAME: G.swiftinterface
// CHECK-MAKE-DEPS-SAME: Swift.swiftmodule
// CHECK-MAKE-DEPS-SAME: B.h
// CHECK-MAKE-DEPS-SAME: F.h