[CAS] Improve swift cas options

Using the same CASOption from clang to communicate CAS configurations so
it is easier to exchange CAS configuration between them.
This commit is contained in:
Steven Wu
2023-06-12 15:33:35 -07:00
parent b1b16981fe
commit 2b7d38dc16
15 changed files with 289 additions and 39 deletions

View File

@@ -494,7 +494,6 @@ REMARK(replay_output, none, "replay output file '%0': key '%1'", (StringRef, Str
REMARK(output_cache_miss, none, "cache miss output file '%0': key '%1'", (StringRef, StringRef))
// CAS related diagnostics
ERROR(error_create_cas, none, "failed to create CAS '%0' (%1)", (StringRef, StringRef))
ERROR(error_invalid_cas_id, none, "invalid CASID '%0' (%1)", (StringRef, StringRef))
ERROR(error_cas, none, "CAS error encountered: %0", (StringRef))

View File

@@ -23,6 +23,7 @@
#include "swift/Basic/LLVM.h"
#include "swift/Basic/Version.h"
#include "swift/Config.h"
#include "clang/CAS/CASOptions.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/SmallString.h"
@@ -820,7 +821,7 @@ namespace swift {
std::string Optimization;
/// clang CASOptions.
std::string CASPath;
llvm::Optional<clang::CASOptions> CASOpts;
/// Cache key for imported bridging header.
std::string BridgingHeaderPCHCacheKey;

View File

@@ -21,6 +21,7 @@
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/StringMap.h"
#include "clang/CAS/CASOptions.h"
#include <string>
#include <vector>
@@ -125,8 +126,8 @@ public:
/// Use CAS.
bool EnableCAS = false;
/// The CAS Path.
std::string CASPath;
/// CASOptions
clang::CASOptions CASOpts;
/// CASFS Root.
std::vector<std::string> CASFSRootIDs;

View File

@@ -1816,6 +1816,14 @@ def cas_path: Separate<["-"], "cas-path">,
Flags<[FrontendOption, NewDriverOnlyOption]>,
HelpText<"Path to CAS">, MetaVarName<"<path>">;
def cas_plugin_path: Separate<["-"], "cas-plugin-path">,
Flags<[FrontendOption, NewDriverOnlyOption]>,
HelpText<"Path to CAS Plugin">, MetaVarName<"<path>">;
def cas_plugin_option: Separate<["-"], "cas-plugin-option">,
Flags<[FrontendOption, NewDriverOnlyOption]>,
HelpText<"Option pass to CAS Plugin">, MetaVarName<"<name>=<option>">;
def allow_unstable_cache_key_for_testing: Flag<["-"], "allow-unstable-cache-key-for-testing">,
Flags<[FrontendOption, HelpHidden, NoDriverOption]>,
HelpText<"Allow compilation caching with unstable inputs for testing purpose">;

View File

@@ -454,17 +454,12 @@ void SwiftDependencyScanningService::setupCachingDependencyScanningService(
auto CachingFS =
llvm::cas::createCachingOnDiskFileSystem(Instance.getObjectStore());
if (!CachingFS) {
Instance.getDiags().diagnose(SourceLoc(), diag::error_create_cas,
"CachingOnDiskFS",
Instance.getDiags().diagnose(SourceLoc(), diag::error_cas,
toString(CachingFS.takeError()));
return;
}
CacheFS = std::move(*CachingFS);
clang::CASOptions CASOpts;
CASOpts.CASPath = Instance.getInvocation().getFrontendOptions().CASPath;
CASOpts.ensurePersistentCAS();
UseClangIncludeTree =
Instance.getInvocation().getClangImporterOptions().UseClangIncludeTree;
const clang::tooling::dependencies::ScanningOutputFormat ClangScanningFormat =
@@ -474,8 +469,9 @@ void SwiftDependencyScanningService::setupCachingDependencyScanningService(
ClangScanningService.emplace(
clang::tooling::dependencies::ScanningMode::DependencyDirectivesScan,
ClangScanningFormat, CASOpts, Instance.getSharedCASInstance(),
Instance.getSharedCacheInstance(),
ClangScanningFormat,
Instance.getInvocation().getFrontendOptions().CASOpts,
Instance.getSharedCASInstance(), Instance.getSharedCacheInstance(),
UseClangIncludeTree ? nullptr : CacheFS,
/* ReuseFileManager */ false, /* OptimizeArgs */ false);
}

View File

@@ -709,13 +709,28 @@ importer::getNormalInvocationArguments(
llvm::sys::path::get_separator() +
"apinotes").str());
if (!importerOpts.CASPath.empty()) {
if (importerOpts.CASOpts) {
invocationArgStrs.push_back("-Xclang");
invocationArgStrs.push_back("-fno-pch-timestamp");
if (!importerOpts.CASOpts->CASPath.empty()) {
invocationArgStrs.push_back("-Xclang");
invocationArgStrs.push_back("-fcas-path");
invocationArgStrs.push_back("-Xclang");
invocationArgStrs.push_back(importerOpts.CASPath);
invocationArgStrs.push_back(importerOpts.CASOpts->CASPath);
}
if (!importerOpts.CASOpts->PluginPath.empty()) {
invocationArgStrs.push_back("-Xclang");
invocationArgStrs.push_back("-fno-pch-timestamp");
invocationArgStrs.push_back("-fcas-plugin-path");
invocationArgStrs.push_back("-Xclang");
invocationArgStrs.push_back(importerOpts.CASOpts->PluginPath);
for (auto Opt : importerOpts.CASOpts->PluginOptions) {
invocationArgStrs.push_back("-Xclang");
invocationArgStrs.push_back("-fcas-plugin-option");
invocationArgStrs.push_back("-Xclang");
invocationArgStrs.push_back(
(llvm::Twine(Opt.first) + "=" + Opt.second).str());
}
}
}
}

View File

@@ -238,10 +238,21 @@ void ClangImporter::recordModuleDependencies(
std::string IncludeTree =
clangModuleDep.IncludeTreeID ? *clangModuleDep.IncludeTreeID : "";
if (!RootID.empty() || !IncludeTree.empty()) {
if (ctx.ClangImporterOpts.CASOpts) {
swiftArgs.push_back("-enable-cas");
if (!ctx.ClangImporterOpts.CASOpts->CASPath.empty()) {
swiftArgs.push_back("-cas-path");
swiftArgs.push_back(ctx.ClangImporterOpts.CASPath);
swiftArgs.push_back(ctx.ClangImporterOpts.CASOpts->CASPath);
}
if (!ctx.ClangImporterOpts.CASOpts->PluginPath.empty()) {
swiftArgs.push_back("-cas-plugin-path");
swiftArgs.push_back(ctx.ClangImporterOpts.CASOpts->PluginPath);
for (auto Opt : ctx.ClangImporterOpts.CASOpts->PluginOptions) {
swiftArgs.push_back("-cas-plugin-option");
swiftArgs.push_back(
(llvm::Twine(Opt.first) + "=" + Opt.second).str());
}
}
}
if (!RootID.empty()) {
@@ -334,10 +345,20 @@ void ClangImporter::recordBridgingHeaderOptions(
llvm::for_each(clangArgs, addClangArg);
if (!ctx.ClangImporterOpts.CASPath.empty()) {
if (ctx.ClangImporterOpts.CASOpts) {
swiftArgs.push_back("-enable-cas");
if (!ctx.ClangImporterOpts.CASOpts->CASPath.empty()) {
swiftArgs.push_back("-cas-path");
swiftArgs.push_back(ctx.ClangImporterOpts.CASPath);
swiftArgs.push_back(ctx.ClangImporterOpts.CASOpts->CASPath);
}
if (!ctx.ClangImporterOpts.CASOpts->PluginPath.empty()) {
swiftArgs.push_back("-cas-plugin-path");
swiftArgs.push_back(ctx.ClangImporterOpts.CASOpts->PluginPath);
for (auto Opt : ctx.ClangImporterOpts.CASOpts->PluginOptions) {
swiftArgs.push_back("-cas-plugin-option");
swiftArgs.push_back((llvm::Twine(Opt.first) + "=" + Opt.second).str());
}
}
}
if (auto Tree = deps.IncludeTreeID) {

View File

@@ -352,8 +352,16 @@ bool ArgsToFrontendOptionsConverter::convert(
}
Opts.EnableCAS = Args.hasArg(OPT_enable_cas);
Opts.CASPath =
Opts.CASOpts.CASPath =
Args.getLastArgValue(OPT_cas_path, llvm::cas::getDefaultOnDiskCASPath());
Opts.CASOpts.PluginPath = Args.getLastArgValue(OPT_cas_plugin_path);
for (StringRef Opt : Args.getAllArgValues(OPT_cas_plugin_option)) {
StringRef Name, Value;
std::tie(Name, Value) = Opt.split('=');
Opts.CASOpts.PluginOptions.emplace_back(std::string(Name),
std::string(Value));
}
Opts.CASFSRootIDs = Args.getAllArgValues(OPT_cas_fs);
Opts.ClangIncludeTrees = Args.getAllArgValues(OPT_clang_include_tree_root);

View File

@@ -1506,7 +1506,7 @@ static bool ParseClangImporterArgs(ClangImporterOptions &Opts,
// Forward the FrontendOptions to clang importer option so it can be
// accessed when creating clang module compilation invocation.
if (FrontendOpts.EnableCAS)
Opts.CASPath = FrontendOpts.CASPath;
Opts.CASOpts = FrontendOpts.CASOpts;
return false;
}

View File

@@ -406,14 +406,13 @@ bool CompilerInstance::setupCASIfNeeded(ArrayRef<const char *> Args) {
if (!Opts.EnableCAS)
return false;
auto MaybeCache = llvm::cas::createOnDiskUnifiedCASDatabases(Opts.CASPath);
if (!MaybeCache) {
Diagnostics.diagnose(SourceLoc(), diag::error_create_cas, Opts.CASPath,
toString(MaybeCache.takeError()));
auto MaybeDB= Opts.CASOpts.getOrCreateDatabases();
if (!MaybeDB) {
Diagnostics.diagnose(SourceLoc(), diag::error_cas,
toString(MaybeDB.takeError()));
return true;
}
CAS = std::move(MaybeCache->first);
ResultCache = std::move(MaybeCache->second);
std::tie(CAS, ResultCache) = *MaybeDB;
// create baseline key.
auto BaseKey = createCompileJobBaseCacheKey(*CAS, Args);

View File

@@ -39,6 +39,7 @@
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/Twine.h"
#include "llvm/CAS/ActionCache.h"
#include "llvm/CAS/ObjectStore.h"
#include "llvm/Support/CommandLine.h"
@@ -1600,12 +1601,23 @@ void InterfaceSubContextDelegateImpl::inheritOptionsForBuildingInterface(
GenericArgs.push_back(clangImporterOpts.BuildSessionFilePath);
}
if (!clangImporterOpts.CASPath.empty()) {
genericSubInvocation.getClangImporterOptions().CASPath =
clangImporterOpts.CASPath;
if (clangImporterOpts.CASOpts) {
genericSubInvocation.getClangImporterOptions().CASOpts =
clangImporterOpts.CASOpts;
GenericArgs.push_back("-enable-cas");
if (!clangImporterOpts.CASOpts->CASPath.empty()) {
GenericArgs.push_back("-cas-path");
GenericArgs.push_back(clangImporterOpts.CASPath);
GenericArgs.push_back(clangImporterOpts.CASOpts->CASPath);
}
if (!clangImporterOpts.CASOpts->PluginPath.empty()) {
GenericArgs.push_back("-cas-plugin-path");
GenericArgs.push_back(clangImporterOpts.CASOpts->PluginPath);
for (auto Opt : clangImporterOpts.CASOpts->PluginOptions) {
GenericArgs.push_back("-cas-plugin-option");
std::string pair = (llvm::Twine(Opt.first) + "=" + Opt.second).str();
GenericArgs.push_back(ArgSaver.save(pair));
}
}
}
if (clangImporterOpts.UseClangIncludeTree) {
@@ -1702,7 +1714,7 @@ InterfaceSubContextDelegateImpl::InterfaceSubContextDelegateImpl(
// required by sourcekitd.
subClangImporterOpts.DetailedPreprocessingRecord =
clangImporterOpts.DetailedPreprocessingRecord;
subClangImporterOpts.CASPath = clangImporterOpts.CASPath;
subClangImporterOpts.CASOpts = clangImporterOpts.CASOpts;
// If the compiler has been asked to be strict with ensuring downstream dependencies
// get the parent invocation's context, or this is an Explicit build, inherit the

186
test/CAS/plugin_cas.swift Normal file
View File

@@ -0,0 +1,186 @@
// REQUIRES: objc_interop
// RUN: %empty-directory(%t)
// RUN: mkdir -p %t/clang-module-cache
// RUN: mkdir -p %t/cas
// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s \
// RUN: -o %t/deps.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift \
// RUN: -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h \
// RUN: -swift-version 4 -enable-cas -clang-include-tree \
// RUN: -cas-path %t/cas \
// RUN: -cas-plugin-path %llvm_libs_dir/libCASPluginTest%llvm_plugin_ext \
// RUN: -cas-plugin-option first-prefix=myfirst- -cas-plugin-option second-prefix=mysecond- \
// RUN: -cas-plugin-option upstream-path=%t/cas-upstream
// Check the contents of the JSON output
// RUN: %FileCheck -check-prefix CHECK -check-prefix CHECK_NO_CLANG_TARGET %s < %t/deps.json
// Check the contents of the JSON output
// RUN: %FileCheck %s -check-prefix CHECK -check-prefix CHECK-NO-SEARCH-PATHS < %t/deps.json
// Check the make-style dependencies file
// RUN: %FileCheck %s -check-prefix CHECK-MAKE-DEPS < %t/deps.d
// Ensure that scanning with `-clang-target` makes sure that Swift modules' respective PCM-dependency-build-argument sets do not contain target triples.
// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s \
// RUN: -o %t/deps_clang_target.json -I %S/../ScanDependencies/Inputs/CHeaders \
// RUN: -I %S/../ScanDependencies/Inputs/Swift -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h \
// RUN: -swift-version 4 -clang-target %target-cpu-apple-macosx10.14 -enable-cas -clang-include-tree \
// RUN: -cas-path %t/cas \
// RUN: -cas-plugin-path %llvm_libs_dir/libCASPluginTest%llvm_plugin_ext \
// RUN: -cas-plugin-option first-prefix=myfirst- -cas-plugin-option second-prefix=mysecond- \
// RUN: -cas-plugin-option upstream-path=%t/cas-upstream
// Check the contents of the JSON output
// RUN: %FileCheck -check-prefix CHECK_CLANG_TARGET %s < %t/deps_clang_target.json
import C
import E
import G
import SubE
// CHECK: "mainModuleName": "deps"
/// --------Main module
// CHECK-LABEL: "modulePath": "deps.swiftmodule",
// CHECK-NEXT: sourceFiles
// CHECK-NEXT: plugin_cas.swift
// CHECK-NEXT: ],
// CHECK-NEXT: "directDependencies": [
// CHECK-DAG: "swift": "A"
// CHECK-DAG: "clang": "C"
// CHECK-DAG: "swift": "E"
// CHECK-DAG: "swift": "F"
// CHECK-DAG: "swift": "G"
// CHECK-DAG: "swift": "SubE"
// CHECK-DAG: "swift": "Swift"
// CHECK-DAG: "swift": "SwiftOnoneSupport"
// CHECK-DAG: "swift": "_Concurrency"
// CHECK-DAG: "swift": "_cross_import_E"
// CHECK: ],
// CHECK: "extraPcmArgs": [
// CHECK-NEXT: "-Xcc",
// CHECK-NEXT: "-target",
// CHECK-NEXT: "-Xcc",
// CHECK: "-fapinotes-swift-version=4"
// CHECK-NOT: "error: cannot open Swift placeholder dependency module map from"
// CHECK: "bridgingHeader":
// CHECK-NEXT: "path":
// CHECK-SAME: Bridging.h
// CHECK-NEXT: "sourceFiles":
// CHECK-NEXT: Bridging.h
// CHECK-NEXT: BridgingOther.h
// CHECK: "moduleDependencies": [
// CHECK-NEXT: "F"
// CHECK-NEXT: ]
/// --------Swift module A
// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}A-{{.*}}.swiftmodule",
// CHECK: directDependencies
// CHECK-NEXT: {
// CHECK-DAG: "clang": "A"
// CHECK-DAG: "swift": "Swift"
// CHECK-NEXT: },
// CHECK: "details":
// CHECK: "moduleCacheKey":
/// --------Swift module F
// CHECK: "modulePath": "{{.*}}{{/|\\}}F-{{.*}}.swiftmodule",
// CHECK-NEXT: "sourceFiles": [
// CHECK-NEXT: ],
// CHECK-NEXT: "directDependencies": [
// CHECK-NEXT: {
// CHECK-DAG: "clang": "F"
// CHECK-DAG: "swift": "Swift"
// CHECK-DAG: "swift": "SwiftOnoneSupport"
// CHECK-NEXT: }
// CHECK-NEXT: ],
// CHECK: "details":
// CHECK: "moduleCacheKey":
/// --------Swift module G
// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}G-{{.*}}.swiftmodule"
// CHECK: "directDependencies"
// CHECK-NEXT: {
// CHECK-DAG: "clang": "G"
// CHECK-DAG: "swift": "Swift"
// CHECK-DAG: "swift": "SwiftOnoneSupport"
// CHECK: ],
// CHECK-NEXT: "details": {
// CHECK: "contextHash": "{{.*}}",
// CHECK: "commandLine": [
// CHECK: "-compile-module-from-interface"
// CHECK: "-target"
// CHECK: "-module-name"
// CHECK: "G"
// CHECK: "-swift-version"
// CHECK: "5"
// CHECK: ],
// CHECK_NO_CLANG_TARGET: "extraPcmArgs": [
// CHECK_NO_CLANG_TARGET-NEXT: "-Xcc",
// CHECK_NO_CLANG_TARGET-NEXT: "-target",
// CHECK_CLANG_TARGET: "extraPcmArgs": [
// CHECK_CLANG_TARGET-NEXT: "-Xcc",
// CHECK_CLANG_TARGET-NEXT: "-fapinotes-swift-version={{.*}}"
// CHECK_CLANG_TARGET-NEXT: ]
/// --------Swift module E
// CHECK: "swift": "E"
// CHECK-LABEL: modulePath": "{{.*}}{{/|\\}}E-{{.*}}.swiftmodule"
// CHECK: "directDependencies"
// CHECK-NEXT: {
// CHECK-NEXT: "swift": "Swift"
// CHECK: "moduleInterfacePath"
// CHECK-SAME: E.swiftinterface
/// --------Swift module Swift
// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}Swift-{{.*}}.swiftmodule",
// CHECK: directDependencies
// CHECK-NEXT: {
// CHECK-NEXT: "clang": "SwiftShims"
/// --------Clang module SwiftShims
// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}SwiftShims-{{.*}}.pcm",
// CHECK: "contextHash": "[[SHIMS_CONTEXT:.*]]",
// CHECK: "-o"
// CHECK-NEXT: SwiftShims-{{.*}}[[SHIMS_CONTEXT]].pcm
// CHECK-NO-SEARCH-PATHS-NOT: "-prebuilt-module-cache-path"
/// --------Clang module C
// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}C-{{.*}}.pcm",
// CHECK: "sourceFiles": [
// CHECK-DAG: module.modulemap
// CHECK-DAG: C.h
// CHECK: directDependencies
// CHECK-NEXT: {
// CHECK-NEXT: "clang": "B"
// CHECK: "moduleMapPath"
// CHECK-SAME: module.modulemap
// CHECK: "contextHash"
// CHECK-SAME: "{{.*}}"
/// --------Clang module B
// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}B-{{.*}}.pcm",
// CHECK: "contextHash": "[[B_CONTEXT:.*]]",
// CHECK: "-o"
// CHECK-NEXT: B-{{.*}}[[B_CONTEXT]].pcm
// Check make-style dependencies
// CHECK-MAKE-DEPS: plugin_cas.swift
// CHECK-MAKE-DEPS-SAME: A.swiftinterface
// CHECK-MAKE-DEPS-SAME: G.swiftinterface
// CHECK-MAKE-DEPS-SAME: B.h
// CHECK-MAKE-DEPS-SAME: F.h
// CHECK-MAKE-DEPS-SAME: Bridging.h
// CHECK-MAKE-DEPS-SAME: BridgingOther.h
// CHECK-MAKE-DEPS-SAME: module.modulemap

View File

@@ -70,6 +70,7 @@ function(get_test_dependencies SDK result_var_name)
arcmt-test
c-arcmt-test
c-index-test
CASPluginTest
clang
clang-cas-test
count

View File

@@ -220,15 +220,15 @@ config.llvm_obj_root = getattr(config, 'llvm_obj_root', None)
lto_flags = ""
use_just_built_liblto = ""
llvm_libs_dir = getattr(config, 'llvm_libs_dir', None)
llvm_plugin_ext = getattr(config, 'llvm_plugin_ext', None)
if platform.system() == 'OpenBSD':
llvm_libs_dir = getattr(config, 'llvm_libs_dir', None)
if not llvm_libs_dir:
lit_config.fatal('No LLVM libs dir set.')
config.environment['LD_LIBRARY_PATH'] = llvm_libs_dir
elif platform.system() == 'Darwin':
llvm_libs_dir = getattr(config, 'llvm_libs_dir', None)
if not llvm_libs_dir:
lit_config.fatal('No LLVM libs dir set.')
lto_flags = "-Xlinker -lto_library -Xlinker %s/libLTO.dylib" % llvm_libs_dir
@@ -236,6 +236,8 @@ elif platform.system() == 'Darwin':
config.substitutions.append( ('%lto_flags', lto_flags) )
config.substitutions.append( ('%use_just_built_liblto', use_just_built_liblto) )
config.substitutions.append( ('%llvm_libs_dir', llvm_libs_dir) )
config.substitutions.append( ('%llvm_plugin_ext', llvm_plugin_ext) )
def append_to_env_path(directory):
config.environment['PATH'] = \

View File

@@ -19,6 +19,7 @@ config.llvm_src_root = "@LLVM_MAIN_SRC_DIR@"
config.llvm_obj_root = "@LLVM_BINARY_DIR@"
config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
config.llvm_libs_dir = "@LLVM_LIBS_DIR@"
config.llvm_plugin_ext = "@LLVM_PLUGIN_EXT@"
config.llvm_code_generators = "@LLVM_TARGETS_TO_BUILD@".split(";")
config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
config.swift_src_root = lit_config.params.get("swift_src_root", "@SWIFT_SOURCE_DIR@")