averello/swift-mirror
1
0
Fork 0
mirror of https://github.com/apple/swift.git synced 2025-12-14 20:36:38 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
c3f8dacdf06cf73a7da77415fb0c9903fb58446d
swift-mirror/utils/build-script
Eric Miotto c3f8dacdf0 [Build] allow to generate symbols for a subset of binaries (#37120) 
This would be needed to reduce overall build times in scenarios when
generating symbols for all binaries is too expensive and/or not needed.

At the same time, introduce tests around the logic that handles symbols.

Addresses rdar://76865276
2021-05-13 07:32:41 -07:00

1482 lines
57 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
"""
The ultimate tool for building Swift.
"""
from __future__ import absolute_import, print_function, unicode_literals
import json
import os
import pipes
import platform
import sys
import time
from build_swift.build_swift import argparse
from build_swift.build_swift import defaults
from build_swift.build_swift import driver_arguments
from build_swift.build_swift import migration
from build_swift.build_swift import presets
from build_swift.build_swift.constants import BUILD_SCRIPT_IMPL_PATH
from build_swift.build_swift.constants import SWIFT_BUILD_ROOT
from build_swift.build_swift.constants import SWIFT_REPO_NAME
from build_swift.build_swift.constants import SWIFT_SOURCE_ROOT
import six
from swift_build_support.swift_build_support import build_graph
from swift_build_support.swift_build_support import products
from swift_build_support.swift_build_support import shell
from swift_build_support.swift_build_support import targets
from swift_build_support.swift_build_support import workspace
from swift_build_support.swift_build_support.cmake import CMake
from swift_build_support.swift_build_support.host_specific_configuration \
import HostSpecificConfiguration
from swift_build_support.swift_build_support.targets import StdlibDeploymentTarget
from swift_build_support.swift_build_support.toolchain import host_toolchain
# -----------------------------------------------------------------------------
# Constants
# These versions are community sourced. At any given time only the Xcode
# version used by Swift CI is officially supported. See ci.swift.org
_SUPPORTED_XCODE_BUILDS = [
("12.2 beta 3", "12B5035g"),
("12.2 Release Candidate", "12B5044c"),
("12.2", "12B45b"),
("12.3", "12C33"),
("12.4", "12D4e"),
("12.5 beta 3", "12E5244e"),
("12.5", "12E262")
]
# -----------------------------------------------------------------------------
# Helpers
def print_note(message, stream=sys.stdout):
"""Writes a diagnostic message to the given stream. By default this
function outputs to stdout.
"""
stream.write('[{}] NOTE: {}\n'.format(sys.argv[0], message))
stream.flush()
def fatal_error(message, stream=sys.stderr):
"""Writes a message to the given stream and exits. By default this
function outputs to stderr.
"""
stream.write('[{}] ERROR: {}\n'.format(sys.argv[0], message))
stream.flush()
sys.exit(1)
def clean_delay():
"""Provide a short delay so accidentally invoked clean builds can be
canceled.
"""
sys.stdout.write('Starting clean build in ')
for i in range(3, 0, -1):
sys.stdout.write('\b%d' % i)
sys.stdout.flush()
time.sleep(1)
print('\b\b\b\bnow.')
def exit_rejecting_arguments(message, parser=None):
print(message, file=sys.stderr)
if parser:
parser.print_usage(sys.stderr)
sys.exit(2) # 2 is the same as `argparse` error exit code.
def initialize_runtime_environment():
"""Change the program environment for building.
"""
# Set an appropriate default umask.
os.umask(0o022)
# Unset environment variables that might affect how tools behave.
for v in [
'MAKEFLAGS',
'SDKROOT',
'MACOSX_DEPLOYMENT_TARGET',
'IPHONEOS_DEPLOYMENT_TARGET',
'TVOS_DEPLOYMENT_TARGET',
'WATCHOS_DEPLOYMENT_TARGET']:
os.environ.pop(v, None)
# Set NINJA_STATUS to format ninja output
os.environ['NINJA_STATUS'] = '[%f/%t][%p][%es] '
class JSONDumper(json.JSONEncoder):
def __init__(self, *args, **kwargs):
json.JSONEncoder.__init__(
self, indent=2, separators=(',', ': '), sort_keys=True,
*args, **kwargs)
def default(self, o):
if hasattr(o, '__dict__'):
return vars(o)
return six.text_type(o)
def print_xcodebuild_versions(file=sys.stdout):
"""
Print the host machine's `xcodebuild` version, as well as version
information for all available SDKs.
"""
version = shell.capture(
['xcodebuild', '-version'], dry_run=False, echo=False).rstrip()
# Allow non-zero exit codes. Under certain obscure circumstances
# xcodebuild can exit with a non-zero exit code even when the SDK is
# usable.
sdks = shell.capture(
['xcodebuild', '-version', '-sdk'], dry_run=False, echo=False,
allow_non_zero_exit=True).rstrip()
fmt = '{version}\n\n--- SDK versions ---\n{sdks}\n'
print(fmt.format(version=version, sdks=sdks), file=file)
file.flush()
def validate_xcode_compatibility():
if sys.platform != 'darwin':
return
if os.getenv("SKIP_XCODE_VERSION_CHECK"):
print("note: skipping Xcode version check")
return
version = shell.capture(
['xcodebuild', '-version'], dry_run=False, echo=False).strip()
valid_build_numbers = tuple(x[1] for x in _SUPPORTED_XCODE_BUILDS)
if not version.endswith(valid_build_numbers):
valid_versions_string = "\n".join(
"{} ({})".format(*x) for x in _SUPPORTED_XCODE_BUILDS)
raise SystemExit(
"error: using unsupported Xcode version:\n\n{}\n\n"
"Install one of:\n{}\n\n"
"Or set 'SKIP_XCODE_VERSION_CHECK=1' in the environment".format(
version, valid_versions_string
)
)
def tar(source, destination):
"""
Create a gzip archive of the file at 'source' at the given
'destination' path.
"""
# We do not use `tarfile` here because:
# - We wish to support LZMA2 compression while also supporting Python 2.7.
# - We wish to explicitly set the owner and group of the archive.
args = ['tar', '-c', '-z', '-f', destination]
if platform.system() != 'Darwin' and platform.system() != 'Windows':
args += ['--owner=0', '--group=0']
# Discard stderr output such as 'tar: Failed to open ...'. We'll detect
# these cases using the exit code, which should cause 'shell.call' to
# raise.
shell.call(args + [source], stderr=shell.DEVNULL)
# -----------------------------------------------------------------------------
# Argument Validation
def validate_arguments(toolchain, args):
if toolchain.cc is None or toolchain.cxx is None:
fatal_error(
"can't find clang (please install clang-3.5 or a "
"later version)")
if toolchain.cmake is None:
fatal_error("can't find CMake (please install CMake)")
if args.distcc:
if toolchain.distcc is None:
fatal_error(
"can't find distcc (please install distcc)")
if toolchain.distcc_pump is None:
fatal_error(
"can't find distcc-pump (please install distcc-pump)")
if args.sccache:
if toolchain.sccache is None:
fatal_error(
"can't find sccache (please install sccache)")
if args.host_target is None or args.stdlib_deployment_targets is None:
fatal_error("unknown operating system")
if args.symbols_package:
if not os.path.isabs(args.symbols_package):
print(
'--symbols-package must be an absolute path '
'(was \'{}\')'.format(args.symbols_package))
return 1
if not args.install_symroot:
fatal_error(
"--install-symroot is required when specifying "
"--symbols-package.")
if args.android:
if args.android_ndk is None or \
args.android_api_level is None or \
args.android_icu_uc is None or \
args.android_icu_uc_include is None or \
args.android_icu_i18n is None or \
args.android_icu_i18n_include is None or \
args.android_icu_data is None:
fatal_error(
"when building for Android, --android-ndk, "
"--android-api-level, --android-icu-uc, "
"--android-icu-uc-include, --android-icu-i18n, "
"--android-icu-i18n-include, and --android-icu-data "
"must be specified")
targets_needing_toolchain = [
'build_indexstoredb',
'build_playgroundsupport',
'build_sourcekitlsp',
'build_toolchainbenchmarks',
'build_swift_inspect',
'tsan_libdispatch_test',
]
has_target_needing_toolchain = \
bool(sum(getattr(args, x) for x in targets_needing_toolchain))
if args.legacy_impl and has_target_needing_toolchain:
fatal_error(
"--legacy-impl is incompatible with building packages needing "
"a toolchain (%s)" % ", ".join(targets_needing_toolchain))
def default_stdlib_deployment_targets(args):
"""
Return targets for the Swift stdlib, based on the build machine.
If the build machine is not one of the recognized ones, return None.
"""
host_target = StdlibDeploymentTarget.host_target()
if host_target is None:
return None
# OS X build machines configure all Darwin platforms by default.
# Put iOS native targets last so that we test them last
# (it takes a long time).
if host_target == StdlibDeploymentTarget.OSX.x86_64:
targets = [host_target]
if args.build_ios and args.build_ios_simulator:
targets.extend(StdlibDeploymentTarget.iOSSimulator.targets)
if args.build_ios and args.build_ios_device:
targets.extend(StdlibDeploymentTarget.iOS.targets)
if args.build_tvos and args.build_tvos_simulator:
targets.extend(StdlibDeploymentTarget.AppleTVSimulator.targets)
if args.build_tvos and args.build_tvos_device:
targets.extend(StdlibDeploymentTarget.AppleTV.targets)
if args.build_watchos and args.build_watchos_simulator:
targets.extend(StdlibDeploymentTarget
.AppleWatchSimulator.targets)
if args.build_watchos and args.build_watchos_device:
targets.extend(StdlibDeploymentTarget.AppleWatch.targets)
return targets
else:
# All other machines only configure their host stdlib by default.
return [host_target]
def apply_default_arguments(toolchain, args):
# Infer if ninja is required
ninja_required = (
args.build_foundation or
args.build_indexstoredb or
args.build_sourcekitlsp or
args.cmake_generator == 'Ninja'
)
if ninja_required and toolchain.ninja is None:
args.build_ninja = True
# Set the default stdlib-deployment-targets, if none were provided.
if args.stdlib_deployment_targets is None:
stdlib_targets = default_stdlib_deployment_targets(args)
args.stdlib_deployment_targets = [
target.name for target in stdlib_targets]
# SwiftPM and XCTest have a dependency on Foundation.
# On OS X, Foundation is built automatically using xcodebuild.
# On Linux, we must ensure that it is built manually.
if ((args.build_swiftpm or args.build_xctest) and
platform.system() != "Darwin"):
args.build_foundation = True
# Foundation has a dependency on libdispatch.
# On OS X, libdispatch is provided by the OS.
# On Linux, we must ensure that it is built manually.
if (args.build_foundation and
platform.system() != "Darwin"):
args.build_libdispatch = True
if args.build_subdir is None:
args.build_subdir = \
workspace.compute_build_subdir(args)
if args.relocate_xdg_cache_home_under_build_subdir:
cache_under_build_subdir = os.path.join(
SWIFT_BUILD_ROOT, args.build_subdir, '.cache')
if args.verbose_build:
print("Relocating XDG_CACHE_HOME to {}".format(cache_under_build_subdir))
workspace.relocate_xdg_cache_home_under(cache_under_build_subdir)
if args.install_destdir is None:
args.install_destdir = os.path.join(
SWIFT_BUILD_ROOT, args.build_subdir,
'{}-{}'.format('toolchain', args.host_target))
# Add optional stdlib-deployment-targets
if args.android:
if args.android_arch == "armv7":
args.stdlib_deployment_targets.append(
StdlibDeploymentTarget.Android.armv7.name)
elif args.android_arch == "aarch64":
args.stdlib_deployment_targets.append(
StdlibDeploymentTarget.Android.aarch64.name)
elif args.android_arch == "x86_64":
args.stdlib_deployment_targets.append(
StdlibDeploymentTarget.Android.x86_64.name)
# Infer platform flags from manually-specified configure targets.
# This doesn't apply to Darwin platforms, as they are
# already configured. No building without the platform flag, though.
android_tgts = [tgt for tgt in args.stdlib_deployment_targets
if StdlibDeploymentTarget.Android.contains(tgt)]
if not args.android and len(android_tgts) > 0:
# If building natively on an Android host, avoid the NDK
# cross-compilation configuration.
if not StdlibDeploymentTarget.Android.contains(StdlibDeploymentTarget
.host_target().name):
args.android = True
args.build_android = False
# Include the Darwin supported architectures in the CMake options.
if args.swift_darwin_supported_archs:
args.extra_cmake_options.append(
'-DSWIFT_DARWIN_SUPPORTED_ARCHS:STRING={}'.format(
args.swift_darwin_supported_archs))
# Remove unsupported Darwin archs from the standard library
# deployment targets.
supported_archs = args.swift_darwin_supported_archs.split(';')
targets = StdlibDeploymentTarget.get_targets_by_name(
args.stdlib_deployment_targets)
args.stdlib_deployment_targets = [
target.name
for target in targets
if (target.platform.is_darwin and
target.arch in supported_archs)
]
# Filter out any macOS stdlib deployment targets that are not supported
# by the macOS SDK.
targets = StdlibDeploymentTarget.get_targets_by_name(
args.stdlib_deployment_targets)
args.stdlib_deployment_targets = [
target.name
for target in targets
if (not target.platform.is_darwin or
target.platform.sdk_supports_architecture(
target.arch, args.darwin_xcrun_toolchain))
]
# Include the Darwin module-only architectures in the CMake options.
if args.swift_darwin_module_archs:
args.extra_cmake_options.append(
'-DSWIFT_DARWIN_MODULE_ARCHS:STRING={}'.format(
args.swift_darwin_module_archs))
# -----------------------------------------------------------------------------
# Build Script Impl Wrapping
class BuildScriptInvocation(object):
"""Represent a single build script invocation.
"""
def __init__(self, toolchain, args):
self.toolchain = toolchain
self.args = args
self.workspace = workspace.Workspace(
source_root=SWIFT_SOURCE_ROOT,
build_root=os.path.join(SWIFT_BUILD_ROOT, args.build_subdir))
self.build_libparser_only = args.build_libparser_only
@property
def install_all(self):
return self.args.install_all or self.args.infer_dependencies
def build_ninja(self):
if not os.path.exists(self.workspace.source_dir("ninja")):
fatal_error(
"can't find source directory for ninja "
"(tried %s)" % (self.workspace.source_dir("ninja")))
ninja_build = products.Ninja.new_builder(
args=self.args,
toolchain=self.toolchain,
workspace=self.workspace,
host=StdlibDeploymentTarget.get_target_for_name(
self.args.host_target))
ninja_build.build()
self.toolchain.ninja = ninja_build.ninja_bin_path
def convert_to_impl_arguments(self):
"""convert_to_impl_arguments() -> (env, args)
Convert the invocation to an environment and list of arguments suitable
for invoking `build-script-impl`.
"""
# Create local shadows, for convenience.
args = self.args
toolchain = self.toolchain
cmake = CMake(args=args,
toolchain=self.toolchain)
impl_args = [
"--workspace", self.workspace.source_root,
"--build-dir", self.workspace.build_root,
"--install-prefix", args.install_prefix,
"--host-target", args.host_target,
"--stdlib-deployment-targets={}".format(
" ".join(args.stdlib_deployment_targets)),
"--host-cc", toolchain.cc,
"--host-cxx", toolchain.cxx,
"--darwin-xcrun-toolchain", args.darwin_xcrun_toolchain,
"--darwin-deployment-version-osx=%s" % (
args.darwin_deployment_version_osx),
"--darwin-deployment-version-ios=%s" % (
args.darwin_deployment_version_ios),
"--darwin-deployment-version-tvos=%s" % (
args.darwin_deployment_version_tvos),
"--darwin-deployment-version-watchos=%s" % (
args.darwin_deployment_version_watchos),
"--cmake", toolchain.cmake,
"--cmark-build-type", args.cmark_build_variant,
"--llvm-build-type", args.llvm_build_variant,
"--swift-build-type", args.swift_build_variant,
"--swift-stdlib-build-type", args.swift_stdlib_build_variant,
"--lldb-build-type", args.lldb_build_variant,
"--foundation-build-type", args.foundation_build_variant,
"--libdispatch-build-type", args.libdispatch_build_variant,
"--libicu-build-type", args.libicu_build_variant,
"--xctest-build-type", args.build_variant,
"--llbuild-build-type", args.build_variant,
"--swift-enable-assertions", str(args.swift_assertions).lower(),
"--swift-stdlib-enable-assertions", str(
args.swift_stdlib_assertions).lower(),
"--swift-analyze-code-coverage", str(
args.swift_analyze_code_coverage).lower(),
"--llbuild-enable-assertions", str(
args.llbuild_assertions).lower(),
"--lldb-assertions", str(
args.lldb_assertions).lower(),
"--cmake-generator", args.cmake_generator,
"--build-jobs", str(args.build_jobs),
"--common-cmake-options=%s" % ' '.join(
pipes.quote(opt) for opt in cmake.common_options()),
"--build-args=%s" % ' '.join(
pipes.quote(arg) for arg in cmake.build_args()),
"--dsymutil-jobs", str(args.dsymutil_jobs),
]
# Compute any product specific cmake arguments.
#
# NOTE: The sum(list(...)) is b/c compute_product_classes returns a
# tuple of lists of which the first is the build-script-impl products
# and the second is the non-build-script-impl-products. It guarantees
# that when we concatenate these two lists together we get a valid
# dependency graph.
for product_class in sum(list(self.compute_product_classes()), []):
if not product_class.is_build_script_impl_product():
continue
product_name = product_class.product_name()
product_source_name = product_class.product_source_name()
source_dir = self.workspace.source_dir(product_source_name)
if not os.path.exists(source_dir):
fatal_error(
"can't find source directory for %s "
"(tried %s)" % (product_name, source_dir))
product = product_class(
args=args,
toolchain=self.toolchain,
source_dir=source_dir,
# FIXME: This is incorrect since it always assumes the host
# target I think?
build_dir=self.workspace.build_dir(
args.host_target, product_name))
cmake_opts = product.cmake_options
# FIXME: We should be using pipes.quote here but we run into issues
# with build-script-impl/cmake not being happy with all of the
# extra "'" in the strings. To fix this easily, we really need to
# just invoke cmake from build-script directly rather than futzing
# with build-script-impl. This makes even more sense since there
# really isn't a security issue here.
if cmake_opts:
impl_args += [
"--{}-cmake-options={}".format(
product_name, ' '.join(cmake_opts))
]
if args.build_stdlib_deployment_targets:
impl_args += [
"--build-stdlib-deployment-targets", " ".join(
args.build_stdlib_deployment_targets)]
if args.cross_compile_hosts:
impl_args += [
"--cross-compile-hosts", " ".join(args.cross_compile_hosts)]
if args.test_paths:
impl_args += ["--test-paths", " ".join(args.test_paths)]
if toolchain.ninja:
impl_args += ["--ninja-bin=%s" % toolchain.ninja]
if args.distcc:
impl_args += [
"--distcc",
"--distcc-pump=%s" % toolchain.distcc_pump
]
if args.sccache:
args.cmake_c_launcher = toolchain.sccache
args.cmake_cxx_launcher = toolchain.sccache
# *NOTE* We use normal cmake to pass through tsan/ubsan options. We do
# NOT pass them to build-script-impl.
if args.enable_asan:
impl_args += ["--enable-asan"]
# If we are on linux, disable leak detection when running ASAN. We
# have a separate bot that checks for leaks.
if platform.system() == 'Linux':
os.environ['ASAN_OPTIONS'] = 'detect_leaks=0'
if args.enable_ubsan:
impl_args += ["--enable-ubsan"]
# If we have lsan, we need to export our suppression list. The actual
# passing in of the LSAN flag is done via the normal cmake method. We
# do not pass the flag to build-script-impl.
if args.enable_lsan:
supp_file = os.path.join(SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME,
"utils",
"lsan_leaks_suppression_list.txt")
os.environ['LSAN_OPTIONS'] = 'suppressions={}'.format(supp_file)
if args.verbose_build:
impl_args += ["--verbose-build"]
if args.install_symroot:
impl_args += [
"--install-symroot", os.path.abspath(args.install_symroot)
]
if args.install_destdir:
impl_args += [
"--install-destdir", os.path.abspath(args.install_destdir)
]
if args.skip_build:
impl_args += ["--skip-build"]
if not args.build_benchmarks:
impl_args += ["--skip-build-benchmarks"]
if args.swift_disable_dead_stripping:
args.extra_cmake_options.append('-DSWIFT_DISABLE_DEAD_STRIPPING:BOOL=TRUE')
# Then add subproject install flags that either skip building them /or/
# if we are going to build them and install_all is set, we also install
# them.
conditional_subproject_configs = [
(args.build_cmark, "cmark"),
(args.build_llvm, "llvm"),
(args.build_swift, "swift"),
(args.build_foundation, "foundation"),
(args.build_xctest, "xctest"),
(args.build_lldb, "lldb"),
(args.build_llbuild, "llbuild"),
(args.build_libcxx, "libcxx"),
(args.build_libdispatch, "libdispatch"),
(args.build_libicu, "libicu")
]
for (should_build, string_name) in conditional_subproject_configs:
if not should_build and not self.args.infer_dependencies:
impl_args += ["--skip-build-{}".format(string_name)]
elif self.install_all:
impl_args += ["--install-{}".format(string_name)]
if args.build_swift_dynamic_stdlib:
impl_args += ["--build-swift-dynamic-stdlib"]
if args.build_swift_static_stdlib:
impl_args += ["--build-swift-static-stdlib"]
if args.build_swift_stdlib_unittest_extra:
impl_args += ["--build-swift-stdlib-unittest-extra"]
if args.build_swift_dynamic_sdk_overlay:
impl_args += ["--build-swift-dynamic-sdk-overlay"]
if args.build_swift_static_sdk_overlay:
impl_args += ["--build-swift-static-sdk-overlay"]
if not args.build_android:
impl_args += ["--skip-build-android"]
if not args.build_clang_tools_extra:
impl_args += ["--skip-build-clang-tools-extra"]
if not args.test and not args.long_test and not args.stress_test:
impl_args += ["--skip-test-swift"]
if not args.test:
impl_args += [
"--skip-test-cmark",
"--skip-test-lldb",
"--skip-test-llbuild",
"--skip-test-xctest",
"--skip-test-foundation",
"--skip-test-libdispatch",
"--skip-test-libicu",
]
if args.build_runtime_with_host_compiler:
impl_args += ["--build-runtime-with-host-compiler"]
if args.validation_test:
impl_args += ["--validation-test"]
if args.long_test:
impl_args += ["--long-test"]
if args.stress_test:
impl_args += ["--stress-test"]
if args.skip_local_build:
impl_args += ["--skip-local-build"]
if args.only_executable_test:
impl_args += ["--only-executable-test"]
if not args.benchmark:
impl_args += ["--skip-test-benchmarks"]
if args.build_libparser_only:
impl_args += ["--build-libparser-only"]
if args.android:
impl_args += [
"--android-arch", args.android_arch,
"--android-ndk", args.android_ndk,
"--android-api-level", args.android_api_level,
"--android-ndk-gcc-version", args.android_ndk_gcc_version,
"--android-icu-uc", args.android_icu_uc,
"--android-icu-uc-include", args.android_icu_uc_include,
"--android-icu-i18n", args.android_icu_i18n,
"--android-icu-i18n-include", args.android_icu_i18n_include,
"--android-icu-data", args.android_icu_data,
]
# If building natively on an Android host, only pass the API level.
if StdlibDeploymentTarget.Android.contains(StdlibDeploymentTarget
.host_target().name):
impl_args += ["--android-api-level", args.android_api_level]
if args.android_deploy_device_path:
impl_args += [
"--android-deploy-device-path",
args.android_deploy_device_path,
]
if platform.system() == 'Darwin':
impl_args += [
"--toolchain-prefix",
targets.darwin_toolchain_prefix(
args.install_prefix),
"--host-lipo", toolchain.lipo,
]
# Isolate build from the system; Darwin toolchains build against SDKs.
# For additional isolation, disable pkg-config. Homebrew's pkg-config
# prioritizes CommandLineTools paths, resulting in compile errors.
args.extra_cmake_options += [
'-DCMAKE_IGNORE_PATH=/usr/lib;/usr/local/lib;/lib',
'-DPKG_CONFIG_EXECUTABLE=/usr/bin/false',
]
if toolchain.libtool is not None:
impl_args += [
"--host-libtool", toolchain.libtool,
]
if args.native_clang_tools_path is not None:
impl_args += [
"--native-clang-tools-path=%s" % args.native_clang_tools_path
]
if args.native_llvm_tools_path is not None:
impl_args += [
"--native-llvm-tools-path=%s" % args.native_llvm_tools_path
]
if args.native_swift_tools_path is not None:
impl_args += [
"--native-swift-tools-path=%s" % args.native_swift_tools_path
]
# If we have extra_swift_args, combine all of them together and then
# add them as one command.
if args.extra_swift_args:
impl_args += [
"--extra-swift-args=%s" % ';'.join(args.extra_swift_args)
]
# Enable macCatalyst
if args.maccatalyst:
(args.extra_cmake_options
.append('-DSWIFT_ENABLE_MACCATALYST:BOOL=TRUE'))
if args.maccatalyst_ios_tests:
impl_args += ["--darwin-test-maccatalyst-ios-like=1"]
# If we have extra_cmake_options, combine all of them together and then
# add them as one command.
if args.extra_cmake_options:
impl_args += [
"--extra-cmake-options=%s" % ' '.join(
pipes.quote(opt) for opt in args.extra_cmake_options)
]
if args.lto_type is not None:
impl_args += [
"--llvm-enable-lto=%s" % args.lto_type,
"--swift-tools-enable-lto=%s" % args.lto_type
]
if args.llvm_max_parallel_lto_link_jobs is not None:
impl_args += [
"--llvm-num-parallel-lto-link-jobs=%s" %
min(args.llvm_max_parallel_lto_link_jobs, args.build_jobs)
]
if args.swift_tools_max_parallel_lto_link_jobs is not None:
impl_args += [
"--swift-tools-num-parallel-lto-link-jobs=%s" %
min(args.swift_tools_max_parallel_lto_link_jobs,
args.build_jobs)
]
impl_args += args.build_script_impl_args
if args.dry_run:
impl_args += ["--dry-run"]
if args.clang_profile_instr_use:
impl_args += [
"--clang-profile-instr-use=%s" %
os.path.abspath(args.clang_profile_instr_use)
]
if args.lit_args:
impl_args += ["--llvm-lit-args=%s" % args.lit_args]
if args.coverage_db:
impl_args += [
"--coverage-db=%s" %
os.path.abspath(args.coverage_db)
]
if args.llvm_install_components:
impl_args += [
"--llvm-install-components=%s" % args.llvm_install_components
]
if not args.clean_llbuild:
impl_args += [
"--skip-clean-llbuild"
]
if args.llvm_ninja_targets:
impl_args += [
"--llvm-ninja-targets=%s" % ' '.join(args.llvm_ninja_targets)
]
if args.llvm_ninja_targets_for_cross_compile_hosts:
impl_args += [
"--llvm-ninja-targets-for-cross-compile-hosts=%s" %
' '.join(args.llvm_ninja_targets_for_cross_compile_hosts)
]
if args.darwin_symroot_path_filters:
impl_args += [
"--darwin_symroot_path_filters=%s" %
' '.join(args.darwin_symroot_path_filters)
]
# Compute the set of host-specific variables, which we pass through to
# the build script via environment variables.
host_specific_variables = self.compute_host_specific_variables()
impl_env = {}
for (host_target, options) in host_specific_variables.items():
for (name, value) in options.items():
# We mangle into an environment variable we can easily evaluate
# from the `build-script-impl`.
impl_env["HOST_VARIABLE_{}__{}".format(
host_target.replace("-", "_"), name)] = value
return (impl_env, impl_args)
def compute_host_specific_variables(self):
"""compute_host_specific_variables(args) -> dict
Compute the host-specific options, organized as a dictionary keyed by
host of options.
"""
args = self.args
options = {}
for host_target in [args.host_target] + args.cross_compile_hosts:
# Compute the host specific configuration.
try:
config = HostSpecificConfiguration(host_target, args)
except argparse.ArgumentError as e:
exit_rejecting_arguments(six.text_type(e))
# Convert into `build-script-impl` style variables.
options[host_target] = {
"SWIFT_SDKS": " ".join(sorted(
config.sdks_to_configure)),
"SWIFT_STDLIB_TARGETS": " ".join(
config.swift_stdlib_build_targets),
"SWIFT_BENCHMARK_TARGETS": " ".join(
config.swift_benchmark_build_targets),
"SWIFT_RUN_BENCHMARK_TARGETS": " ".join(
config.swift_benchmark_run_targets),
"SWIFT_TEST_TARGETS": " ".join(
config.swift_test_run_targets),
}
return options
def compute_product_classes(self):
"""compute_product_classes() -> (list, list, list)
Compute the list first of all pre-build-script-impl products, then all
build-script-impl products and then all non-build-script-impl products.
It is assumed that concatenating the three lists together will result in a
valid dependency graph for the compilation.
"""
before_impl_product_classes = []
# If --skip-early-swift-driver is passed in, swift will be built
# as usual, but relying on its own C++-based (Legacy) driver.
# Otherwise, we build an "early" swift-driver using the host
# toolchain, which the later-built compiler will forward
# `swiftc` invocations to. That is, if we find a Swift compiler
# in the host toolchain. If the host toolchain is not equpipped with
# a Swift compiler, a warning is emitted. In the future, it may become
# mandatory that the host toolchain come with its own `swiftc`.
if self.args.build_early_swift_driver:
before_impl_product_classes.append(products.EarlySwiftDriver)
# FIXME: This is a weird division (returning a list of class objects),
# but it matches the existing structure of the `build-script-impl`.
impl_product_classes = []
if self.args.build_cmark:
impl_product_classes.append(products.CMark)
# If --skip-build-llvm is passed in, LLVM cannot be completely disabled, as
# Swift still needs a few LLVM targets like tblgen to be built for it to be
# configured. Instead, handle this in build-script-impl for now.
impl_product_classes.append(products.LLVM)
if self.args.build_libcxx:
impl_product_classes.append(products.LibCXX)
if self.args.build_libicu:
impl_product_classes.append(products.LibICU)
if self.args.build_swift:
impl_product_classes.append(products.Swift)
if self.args.build_lldb:
impl_product_classes.append(products.LLDB)
if self.args.build_libdispatch:
impl_product_classes.append(products.LibDispatch)
if self.args.build_foundation:
impl_product_classes.append(products.Foundation)
if self.args.build_xctest:
impl_product_classes.append(products.XCTest)
if self.args.build_llbuild:
impl_product_classes.append(products.LLBuild)
# Sanity check that all of our impl classes are actually
# build_script_impl products.
for prod in impl_product_classes:
assert(prod.is_build_script_impl_product())
product_classes = []
if self.args.build_swiftpm:
product_classes.append(products.SwiftPM)
if self.args.build_swiftsyntax:
product_classes.append(products.SwiftSyntax)
if self.args.build_skstresstester:
product_classes.append(products.SKStressTester)
if self.args.build_swiftformat:
product_classes.append(products.SwiftFormat)
if self.args.build_swiftevolve:
product_classes.append(products.SwiftEvolve)
if self.args.build_indexstoredb:
product_classes.append(products.IndexStoreDB)
if self.args.build_playgroundsupport:
product_classes.append(products.PlaygroundSupport)
if self.args.build_sourcekitlsp:
product_classes.append(products.SourceKitLSP)
if self.args.build_toolchainbenchmarks:
product_classes.append(products.Benchmarks)
if self.args.build_swift_inspect:
product_classes.append(products.SwiftInspect)
if self.args.tsan_libdispatch_test:
product_classes.append(products.TSanLibDispatch)
# Keep SwiftDriver at last.
# swift-driver's integration with the build scripts is not fully
# supported. Using swift-driver to build these products may hit
# failures.
if self.args.build_swift_driver or self.args.install_swift_driver:
product_classes.append(products.SwiftDriver)
# Sanity check that all of our non-impl classes are actually
# not build_script_impl products.
for prod in product_classes:
assert(not prod.is_build_script_impl_product())
# Now that we have our two lists of product_classes, if we are asked to
# infer dependencies, infer the dependencies now and then re-split the
# list.
if self.args.infer_dependencies:
combined = impl_product_classes + product_classes
if self.args.verbose_build:
print("-- Build Graph Inference --")
print("Initial Product List:")
for p in combined:
print(" {}".format(p.product_name()))
# Now that we have produced the schedule, resplit. We require our
# dependencies to respect our build-script-impl property. This means
# that no build-script-impl products can have dependencies on
# non-build-script-impl products. Otherwise, it would be unsafe to
# re-order build-script-impl products in front of non
# build-script-impl products.
before_impl_product_classes = []
impl_product_classes = []
product_classes = []
is_darwin = platform.system() == 'Darwin'
final_schedule = build_graph.produce_scheduled_build(combined)[0]
for p in final_schedule:
if is_darwin and p.is_nondarwin_only_build_product():
continue
if p.is_build_script_impl_product():
impl_product_classes.append(p)
elif p.is_before_build_script_impl_product():
before_impl_product_classes.append(p)
else:
product_classes.append(p)
if self.args.verbose_build:
print("Final Build Order:")
for p in before_impl_product_classes:
print(" {}".format(p.product_name()))
for p in impl_product_classes:
print(" {}".format(p.product_name()))
for p in product_classes:
print(" {}".format(p.product_name()))
return (before_impl_product_classes, impl_product_classes, product_classes)
def execute(self):
"""Execute the invocation with the configured arguments."""
# Convert to a build-script-impl invocation.
(self.impl_env, self.impl_args) = self.convert_to_impl_arguments()
# If using the legacy implementation, delegate all behavior to
# `build-script-impl`.
if self.args.legacy_impl:
# Execute the underlying build script implementation.
shell.call_without_sleeping([BUILD_SCRIPT_IMPL_PATH] + self.impl_args,
env=self.impl_env, echo=True)
return
# Otherwise, we compute and execute the individual actions ourselves.
# Compute the list of hosts to operate on.
all_host_names = [
self.args.host_target] + self.args.cross_compile_hosts
all_hosts = [StdlibDeploymentTarget.get_target_for_name(name)
for name in all_host_names]
# Compute the list of product classes to operate on.
#
# FIXME: This should really be per-host, but the current structure
# matches that of `build-script-impl`.
(before_impl_product_classes, impl_product_classes, product_classes) =\
self.compute_product_classes()
# Execute each "pass".
# Pre-build-script-impl products...
# Note: currently only supports building for the host.
for host_target in [self.args.host_target]:
for product_class in before_impl_product_classes:
if product_class.is_build_script_impl_product():
continue
if not product_class.is_before_build_script_impl_product():
continue
# Execute clean, build, test, install
self.execute_product_build_steps(product_class, host_target)
# Build...
for host_target in all_hosts:
# FIXME: We should only compute these once.
try:
config = HostSpecificConfiguration(host_target.name, self.args)
except argparse.ArgumentError as e:
exit_rejecting_arguments(six.text_type(e))
print("Building the standard library for: {}".format(
" ".join(config.swift_stdlib_build_targets)))
if config.swift_test_run_targets and (
self.args.test or self.args.long_test):
print("Running Swift tests for: {}".format(
" ".join(config.swift_test_run_targets)))
if config.swift_benchmark_run_targets and self.args.benchmark:
print("Running Swift benchmarks for: {}".format(
" ".join(config.swift_benchmark_run_targets)))
for product_class in impl_product_classes:
self._execute_build_action(host_target, product_class)
# Test...
for host_target in all_hosts:
for product_class in impl_product_classes:
self._execute_test_action(host_target, product_class)
# Install...
for host_target in all_hosts:
for product_class in impl_product_classes:
self._execute_install_action(host_target, product_class)
# Core Lipo...
self._execute_merged_host_lipo_core_action()
# Non-build-script-impl products...
# Note: currently only supports building for the host.
for host_target in [self.args.host_target]:
for product_class in product_classes:
if product_class.is_build_script_impl_product():
continue
# Execute clean, build, test, install
self.execute_product_build_steps(product_class, host_target)
# Extract symbols...
for host_target in all_hosts:
self._execute_extract_symbols_action(host_target)
# Package...
for host_target in all_hosts:
self._execute_package_action(host_target)
# Lipo...
self._execute_merged_host_lipo_action()
def _execute_build_action(self, host_target, product_class):
action_name = "{}-{}-build".format(host_target.name,
product_class.product_name())
self._execute_action(action_name)
def _execute_test_action(self, host_target, product_class):
action_name = "{}-{}-test".format(host_target.name,
product_class.product_name())
self._execute_action(action_name)
def _execute_install_action(self, host_target, product_class):
action_name = "{}-{}-install".format(host_target.name,
product_class.product_name())
self._execute_action(action_name)
def _execute_extract_symbols_action(self, host_target):
action_name = "{}-extractsymbols".format(host_target.name)
self._execute_action(action_name)
def _execute_package_action(self, host_target):
action_name = "{}-package".format(host_target.name)
self._execute_action(action_name)
def _execute_merged_host_lipo_action(self):
self._execute_action("merged-hosts-lipo")
def _execute_merged_host_lipo_core_action(self):
self._execute_action("merged-hosts-lipo-core")
def _execute_action(self, action_name):
shell.call_without_sleeping(
[BUILD_SCRIPT_IMPL_PATH] + self.impl_args +
["--only-execute", action_name],
env=self.impl_env, echo=self.args.verbose_build)
def execute_product_build_steps(self, product_class, host_target):
product_source = product_class.product_source_name()
product_name = product_class.product_name()
if product_class.is_swiftpm_unified_build_product():
build_dir = self.workspace.swiftpm_unified_build_dir(
host_target)
else:
build_dir = self.workspace.build_dir(
host_target, product_name)
product = product_class(
args=self.args,
toolchain=self.toolchain,
source_dir=self.workspace.source_dir(product_source),
build_dir=build_dir)
if product.should_clean(host_target):
print("--- Cleaning %s ---" % product_name)
product.clean(host_target)
if product.should_build(host_target):
print("--- Building %s ---" % product_name)
product.build(host_target)
if product.should_test(host_target):
print("--- Running tests for %s ---" % product_name)
product.test(host_target)
print("--- Finished tests for %s ---" % product_name)
if product.should_install(host_target) or \
(self.install_all and product.should_build(host_target)):
print("--- Installing %s ---" % product_name)
product.install(host_target)
# -----------------------------------------------------------------------------
# Main (preset)
def parse_preset_args():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description="""Builds Swift using a preset.""")
parser.add_argument(
"-n", "--dry-run",
help="print the commands that would be executed, but do not execute "
"them",
action="store_true",
default=False)
parser.add_argument(
"--preset-file",
help="load presets from the specified file",
metavar="PATH",
action="append",
dest="preset_file_names",
default=[])
parser.add_argument(
"--preset",
help="use the specified option preset",
metavar="NAME")
parser.add_argument(
"--show-presets",
help="list all presets and exit",
action=argparse.actions.StoreTrueAction,
nargs=argparse.Nargs.OPTIONAL)
parser.add_argument(
"--distcc",
help="use distcc",
action=argparse.actions.StoreTrueAction,
nargs=argparse.Nargs.OPTIONAL,
default=os.environ.get('USE_DISTCC') == '1')
parser.add_argument(
"--sccache",
help="use sccache",
action=argparse.actions.StoreTrueAction,
nargs=argparse.Nargs.OPTIONAL,
default=os.environ.get('SWIFT_USE_SCCACHE') == '1')
parser.add_argument(
"--cmake-c-launcher",
help="the absolute path to set CMAKE_C_COMPILER_LAUNCHER",
metavar="PATH")
parser.add_argument(
"--cmake-cxx-launcher",
help="the absolute path to set CMAKE_CXX_COMPILER_LAUNCHER",
metavar="PATH")
parser.add_argument(
"-j", "--jobs",
help="the number of parallel build jobs to use",
type=int,
dest="build_jobs")
parser.add_argument(
"preset_substitutions_raw",
help="'name=value' pairs that are substituted in the preset",
nargs="*",
metavar="SUBSTITUTION")
parser.add_argument(
"--expand-invocation", "--expand-build-script-invocation",
help="Print the expanded build-script invocation generated "
"by the preset, but do not run the preset",
action=argparse.actions.StoreTrueAction,
nargs=argparse.Nargs.OPTIONAL)
parser.add_argument(
"--swiftsyntax-install-prefix",
help="specify the directory to where SwiftSyntax should be installed")
parser.add_argument(
"--build-dir",
help="specify the directory where build artifact should be stored")
parser.add_argument(
"--dump-config",
help="instead of building, write JSON to stdout containing "
"various values used to build in this configuration",
action="store_true",
default=False)
parser.add_argument(
"--reconfigure",
help="Reconfigure all projects as we build",
action="store_true",
default=False)
return parser.parse_args()
def main_preset():
args = parse_preset_args()
if len(args.preset_file_names) == 0:
args.preset_file_names = [
os.path.join(
SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME, "utils",
"build-presets.ini")
]
user_presets_file = os.path.join(os.path.expanduser("~"),
'.swift-build-presets')
if os.path.isfile(user_presets_file):
args.preset_file_names.append(user_presets_file)
preset_parser = presets.PresetParser()
try:
preset_parser.read_files(args.preset_file_names)
except presets.PresetError as e:
fatal_error(six.text_type(e))
if args.show_presets:
for name in sorted(preset_parser.preset_names,
key=lambda name: name.lower()):
print(name)
return 0
if not args.preset:
fatal_error("missing --preset option")
args.preset_substitutions = {}
for arg in args.preset_substitutions_raw:
name, value = arg.split("=", 1)
args.preset_substitutions[name] = value
try:
preset = preset_parser.get_preset(
args.preset,
vars=args.preset_substitutions)
except presets.PresetError as e:
fatal_error(six.text_type(e))
preset_args = migration.migrate_swift_sdks(preset.args)
if args.distcc and (args.cmake_c_launcher or args.cmake_cxx_launcher):
fatal_error(
'--distcc can not be used with' +
' --cmake-c-launcher or --cmake-cxx-launcher')
if args.sccache and (args.cmake_c_launcher or args.cmake_cxx_launcher):
fatal_error(
'--sccache can not be used with' +
' --cmake-c-launcher or --cmake-cxx-launcher')
build_script_args = [sys.argv[0]]
if args.dry_run:
build_script_args += ["--dry-run"]
if args.dump_config:
build_script_args += ["--dump-config"]
build_script_args += preset_args
if args.distcc:
build_script_args += ["--distcc"]
if args.sccache:
build_script_args += ["--sccache"]
if args.build_jobs:
build_script_args += ["--jobs", str(args.build_jobs)]
if args.swiftsyntax_install_prefix:
build_script_args += ["--install-swiftsyntax",
"--install-destdir",
args.swiftsyntax_install_prefix]
if args.build_dir:
build_script_args += ["--build-dir", args.build_dir]
if args.cmake_c_launcher:
build_script_args += ["--cmake-c-launcher", args.cmake_c_launcher]
if args.cmake_cxx_launcher:
build_script_args += ["--cmake-cxx-launcher", args.cmake_cxx_launcher]
printable_command = shell.quote_command(build_script_args)
if args.expand_invocation:
print(printable_command)
return 0
if args.reconfigure:
build_script_args += ["--reconfigure"]
print_note('using preset "{}", which expands to \n\n{}\n'.format(
args.preset, printable_command))
shell.call_without_sleeping(build_script_args)
return 0
# -----------------------------------------------------------------------------
# Main (normal)
def main_normal():
parser = driver_arguments.create_argument_parser()
args = migration.parse_args(parser, sys.argv[1:])
if args.build_script_impl_args:
# If we received any impl args, check if `build-script-impl` would
# accept them or not before any further processing.
try:
migration.check_impl_args(BUILD_SCRIPT_IMPL_PATH,
args.build_script_impl_args)
except ValueError as e:
exit_rejecting_arguments(e, parser)
if '--check-args-only' in args.build_script_impl_args:
return 0
shell.dry_run = args.dry_run
# Prepare and validate toolchain
if args.darwin_xcrun_toolchain is None:
xcrun_toolchain = os.environ.get('TOOLCHAINS',
defaults.DARWIN_XCRUN_TOOLCHAIN)
print_note('Using toolchain {}'.format(xcrun_toolchain))
args.darwin_xcrun_toolchain = xcrun_toolchain
toolchain = host_toolchain(xcrun_toolchain=args.darwin_xcrun_toolchain)
os.environ['TOOLCHAINS'] = args.darwin_xcrun_toolchain
if args.host_cc is not None:
toolchain.cc = args.host_cc
if args.host_cxx is not None:
toolchain.cxx = args.host_cxx
if args.host_lipo is not None:
toolchain.lipo = args.host_lipo
if args.host_libtool is not None:
toolchain.libtool = args.host_libtool
if args.cmake is not None:
toolchain.cmake = args.cmake
if args.sccache:
print("Ensuring the sccache server is running...")
# Use --show-stats to ensure the server is started, because using
# --start-server will fail if the server is already running.
# Capture the output because we don't want to see the stats.
shell.capture([toolchain.sccache, "--show-stats"])
cmake = CMake(args=args, toolchain=toolchain)
# Check the CMake version is sufficient on Linux and build from source
# if not.
cmake_path = cmake.check_cmake_version(SWIFT_SOURCE_ROOT, SWIFT_BUILD_ROOT)
if cmake_path is not None:
toolchain.cmake = cmake_path
args.cmake = cmake_path
# Preprocess the arguments to apply defaults.
apply_default_arguments(toolchain, args)
# Validate the arguments.
validate_arguments(toolchain, args)
# Create the build script invocation.
invocation = BuildScriptInvocation(toolchain, args)
# Sanitize the runtime environment.
initialize_runtime_environment()
# Show SDKs, if requested.
if args.show_sdks:
print_xcodebuild_versions()
if args.dump_config:
print(JSONDumper().encode(invocation))
return 0
# Clean build directory if requested.
if args.clean:
clean_delay()
shell.rmtree(invocation.workspace.build_root)
# Create build directory.
shell.makedirs(invocation.workspace.build_root)
# Create .build_script_log
if not args.dry_run:
build_script_log = os.path.join(invocation.workspace.build_root,
".build_script_log")
open(build_script_log, 'w').close()
# Build ninja if required, which will update the toolchain.
if args.build_ninja:
invocation.build_ninja()
# Execute the underlying build script implementation.
invocation.execute()
if args.symbols_package:
print('--- Creating symbols package ---')
print('-- Package file: {} --'.format(args.symbols_package))
if platform.system() == 'Darwin':
prefix = targets.darwin_toolchain_prefix(args.install_prefix)
prefix = os.path.join(args.host_target, prefix.lstrip('/'))
else:
prefix = args.install_prefix
# As a security measure, `tar` normally strips leading '/' from paths
# it is archiving. To stay safe, we change working directories, then
# run `tar` without the leading '/' (we remove it ourselves to keep
# `tar` from emitting a warning).
with shell.pushd(args.install_symroot):
tar(source=prefix.lstrip('/'),
destination=args.symbols_package)
return 0
# -----------------------------------------------------------------------------
def main():
if not SWIFT_SOURCE_ROOT:
fatal_error(
"could not infer source root directory " +
"(forgot to set $SWIFT_SOURCE_ROOT environment variable?)")
if not os.path.isdir(SWIFT_SOURCE_ROOT):
fatal_error(
"source root directory \'" + SWIFT_SOURCE_ROOT +
"\' does not exist " +
"(forgot to set $SWIFT_SOURCE_ROOT environment variable?)")
validate_xcode_compatibility()
# Determine if we are invoked in the preset mode and dispatch accordingly.
if any([(opt.startswith("--preset") or opt == "--show-presets")
for opt in sys.argv[1:]]):
return main_preset()
else:
return main_normal()
if __name__ == "__main__":
try:
sys.exit(main())
except KeyboardInterrupt:
sys.exit(1)
Reference in New Issue View Git Blame Copy Permalink