averello/swift-mirror
1
0
Fork 0
mirror of https://github.com/apple/swift.git synced 2025-12-21 12:14:44 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
3a551f8fcce271bbeffb65c9fe4aa0b167e77a7f
swift-mirror/utils/build-script
Eric Miotto 3a551f8fcc [build] allow to specify ninja targets to build for LLVM (#36910) 
This is meant to support scenarios in which we need to build as little as
LLVM as possible for performance reasons (e.g. when enabling LTO).

While LLVM CMake build system offers options in this sense,
in our investigation they turned out not to be suitable,
since either they are not granular enough (`LLVM_INCLUDE/BUILD` flags)
or they require active opt out for any new tool added
(`*_BUILD_*_TOOL` flags)

When using this mechanism, there is the possibility to specify different
targets to use for cross-compile hosts.

Supports rdar://32019390
2021-04-15 12:45:00 -07:00

1438 lines
55 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
"""
The ultimate tool for building Swift.
"""
from __future__ import absolute_import, print_function, unicode_literals
import json
import os
import pipes
import platform
import sys
import time
from build_swift.build_swift import argparse
from build_swift.build_swift import defaults
from build_swift.build_swift import driver_arguments
from build_swift.build_swift import migration
from build_swift.build_swift import presets
from build_swift.build_swift.constants import BUILD_SCRIPT_IMPL_PATH
from build_swift.build_swift.constants import SWIFT_BUILD_ROOT
from build_swift.build_swift.constants import SWIFT_REPO_NAME
from build_swift.build_swift.constants import SWIFT_SOURCE_ROOT
import six
from swift_build_support.swift_build_support import build_graph
from swift_build_support.swift_build_support import products
from swift_build_support.swift_build_support import shell
from swift_build_support.swift_build_support import targets
from swift_build_support.swift_build_support import workspace
from swift_build_support.swift_build_support.cmake import CMake
from swift_build_support.swift_build_support.host_specific_configuration \
import HostSpecificConfiguration
from swift_build_support.swift_build_support.targets import StdlibDeploymentTarget
from swift_build_support.swift_build_support.toolchain import host_toolchain
# -----------------------------------------------------------------------------
# Constants
# These versions are community sourced. At any given time only the Xcode
# version used by Swift CI is officially supported. See ci.swift.org
_SUPPORTED_XCODE_BUILDS = [
("12.2 beta 3", "12B5035g"),
("12.2 Release Candidate", "12B5044c"),
("12.2", "12B45b"),
("12.3", "12C33"),
("12.4", "12D4e"),
]
# -----------------------------------------------------------------------------
# Helpers
def print_note(message, stream=sys.stdout):
"""Writes a diagnostic message to the given stream. By default this
function outputs to stdout.
"""
stream.write('[{}] NOTE: {}\n'.format(sys.argv[0], message))
stream.flush()
def fatal_error(message, stream=sys.stderr):
"""Writes a message to the given stream and exits. By default this
function outputs to stderr.
"""
stream.write('[{}] ERROR: {}\n'.format(sys.argv[0], message))
stream.flush()
sys.exit(1)
def clean_delay():
"""Provide a short delay so accidentally invoked clean builds can be
canceled.
"""
sys.stdout.write('Starting clean build in ')
for i in range(3, 0, -1):
sys.stdout.write('\b%d' % i)
sys.stdout.flush()
time.sleep(1)
print('\b\b\b\bnow.')
def exit_rejecting_arguments(message, parser=None):
print(message, file=sys.stderr)
if parser:
parser.print_usage(sys.stderr)
sys.exit(2) # 2 is the same as `argparse` error exit code.
def initialize_runtime_environment():
"""Change the program environment for building.
"""
# Set an appropriate default umask.
os.umask(0o022)
# Unset environment variables that might affect how tools behave.
for v in [
'MAKEFLAGS',
'SDKROOT',
'MACOSX_DEPLOYMENT_TARGET',
'IPHONEOS_DEPLOYMENT_TARGET',
'TVOS_DEPLOYMENT_TARGET',
'WATCHOS_DEPLOYMENT_TARGET']:
os.environ.pop(v, None)
# Set NINJA_STATUS to format ninja output
os.environ['NINJA_STATUS'] = '[%f/%t][%p][%es] '
class JSONDumper(json.JSONEncoder):
def __init__(self, *args, **kwargs):
json.JSONEncoder.__init__(
self, indent=2, separators=(',', ': '), sort_keys=True,
*args, **kwargs)
def default(self, o):
if hasattr(o, '__dict__'):
return vars(o)
return six.text_type(o)
def print_xcodebuild_versions(file=sys.stdout):
"""
Print the host machine's `xcodebuild` version, as well as version
information for all available SDKs.
"""
version = shell.capture(
['xcodebuild', '-version'], dry_run=False, echo=False).rstrip()
# Allow non-zero exit codes. Under certain obscure circumstances
# xcodebuild can exit with a non-zero exit code even when the SDK is
# usable.
sdks = shell.capture(
['xcodebuild', '-version', '-sdk'], dry_run=False, echo=False,
allow_non_zero_exit=True).rstrip()
fmt = '{version}\n\n--- SDK versions ---\n{sdks}\n'
print(fmt.format(version=version, sdks=sdks), file=file)
file.flush()
def validate_xcode_compatibility():
if sys.platform != 'darwin':
return
if os.getenv("SKIP_XCODE_VERSION_CHECK"):
print("note: skipping Xcode version check")
return
version = shell.capture(
['xcodebuild', '-version'], dry_run=False, echo=False).strip()
valid_build_numbers = tuple(x[1] for x in _SUPPORTED_XCODE_BUILDS)
if not version.endswith(valid_build_numbers):
valid_versions_string = "\n".join(
"{} ({})".format(*x) for x in _SUPPORTED_XCODE_BUILDS)
raise SystemExit(
"error: using unsupported Xcode version:\n\n{}\n\n"
"Install one of:\n{}\n\n"
"Or set 'SKIP_XCODE_VERSION_CHECK=1' in the environment".format(
version, valid_versions_string
)
)
def tar(source, destination):
"""
Create a gzip archive of the file at 'source' at the given
'destination' path.
"""
# We do not use `tarfile` here because:
# - We wish to support LZMA2 compression while also supporting Python 2.7.
# - We wish to explicitly set the owner and group of the archive.
args = ['tar', '-c', '-z', '-f', destination]
if platform.system() != 'Darwin' and platform.system() != 'Windows':
args += ['--owner=0', '--group=0']
# Discard stderr output such as 'tar: Failed to open ...'. We'll detect
# these cases using the exit code, which should cause 'shell.call' to
# raise.
shell.call(args + [source], stderr=shell.DEVNULL)
# -----------------------------------------------------------------------------
# Argument Validation
def validate_arguments(toolchain, args):
if toolchain.cc is None or toolchain.cxx is None:
fatal_error(
"can't find clang (please install clang-3.5 or a "
"later version)")
if toolchain.cmake is None:
fatal_error("can't find CMake (please install CMake)")
if args.distcc:
if toolchain.distcc is None:
fatal_error(
"can't find distcc (please install distcc)")
if toolchain.distcc_pump is None:
fatal_error(
"can't find distcc-pump (please install distcc-pump)")
if args.sccache:
if toolchain.sccache is None:
fatal_error(
"can't find sccache (please install sccache)")
if args.host_target is None or args.stdlib_deployment_targets is None:
fatal_error("unknown operating system")
if args.symbols_package:
if not os.path.isabs(args.symbols_package):
print(
'--symbols-package must be an absolute path '
'(was \'{}\')'.format(args.symbols_package))
return 1
if not args.install_symroot:
fatal_error(
"--install-symroot is required when specifying "
"--symbols-package.")
if args.android:
if args.android_ndk is None or \
args.android_api_level is None or \
args.android_icu_uc is None or \
args.android_icu_uc_include is None or \
args.android_icu_i18n is None or \
args.android_icu_i18n_include is None or \
args.android_icu_data is None:
fatal_error(
"when building for Android, --android-ndk, "
"--android-api-level, --android-icu-uc, "
"--android-icu-uc-include, --android-icu-i18n, "
"--android-icu-i18n-include, and --android-icu-data "
"must be specified")
targets_needing_toolchain = [
'build_indexstoredb',
'build_playgroundsupport',
'build_sourcekitlsp',
'build_toolchainbenchmarks',
'build_swift_inspect',
'tsan_libdispatch_test',
]
has_target_needing_toolchain = \
bool(sum(getattr(args, x) for x in targets_needing_toolchain))
if args.legacy_impl and has_target_needing_toolchain:
fatal_error(
"--legacy-impl is incompatible with building packages needing "
"a toolchain (%s)" % ", ".join(targets_needing_toolchain))
def default_stdlib_deployment_targets(args):
"""
Return targets for the Swift stdlib, based on the build machine.
If the build machine is not one of the recognized ones, return None.
"""
host_target = StdlibDeploymentTarget.host_target()
if host_target is None:
return None
# OS X build machines configure all Darwin platforms by default.
# Put iOS native targets last so that we test them last
# (it takes a long time).
if host_target == StdlibDeploymentTarget.OSX.x86_64:
targets = [host_target]
if args.build_ios and args.build_ios_simulator:
targets.extend(StdlibDeploymentTarget.iOSSimulator.targets)
if args.build_ios and args.build_ios_device:
targets.extend(StdlibDeploymentTarget.iOS.targets)
if args.build_tvos and args.build_tvos_simulator:
targets.extend(StdlibDeploymentTarget.AppleTVSimulator.targets)
if args.build_tvos and args.build_tvos_device:
targets.extend(StdlibDeploymentTarget.AppleTV.targets)
if args.build_watchos and args.build_watchos_simulator:
targets.extend(StdlibDeploymentTarget
.AppleWatchSimulator.targets)
if args.build_watchos and args.build_watchos_device:
targets.extend(StdlibDeploymentTarget.AppleWatch.targets)
return targets
else:
# All other machines only configure their host stdlib by default.
return [host_target]
def apply_default_arguments(toolchain, args):
# Infer if ninja is required
ninja_required = (
args.build_foundation or
args.build_indexstoredb or
args.build_sourcekitlsp or
args.cmake_generator == 'Ninja'
)
if ninja_required and toolchain.ninja is None:
args.build_ninja = True
# Set the default stdlib-deployment-targets, if none were provided.
if args.stdlib_deployment_targets is None:
stdlib_targets = default_stdlib_deployment_targets(args)
args.stdlib_deployment_targets = [
target.name for target in stdlib_targets]
# SwiftPM and XCTest have a dependency on Foundation.
# On OS X, Foundation is built automatically using xcodebuild.
# On Linux, we must ensure that it is built manually.
if ((args.build_swiftpm or args.build_xctest) and
platform.system() != "Darwin"):
args.build_foundation = True
# Foundation has a dependency on libdispatch.
# On OS X, libdispatch is provided by the OS.
# On Linux, we must ensure that it is built manually.
if (args.build_foundation and
platform.system() != "Darwin"):
args.build_libdispatch = True
if args.build_subdir is None:
args.build_subdir = \
workspace.compute_build_subdir(args)
if args.relocate_xdg_cache_home_under_build_subdir:
cache_under_build_subdir = os.path.join(
SWIFT_BUILD_ROOT, args.build_subdir, '.cache')
if args.verbose_build:
print("Relocating XDG_CACHE_HOME to {}".format(cache_under_build_subdir))
workspace.relocate_xdg_cache_home_under(cache_under_build_subdir)
if args.install_destdir is None:
args.install_destdir = os.path.join(
SWIFT_BUILD_ROOT, args.build_subdir,
'{}-{}'.format('toolchain', args.host_target))
# Add optional stdlib-deployment-targets
if args.android:
if args.android_arch == "armv7":
args.stdlib_deployment_targets.append(
StdlibDeploymentTarget.Android.armv7.name)
elif args.android_arch == "aarch64":
args.stdlib_deployment_targets.append(
StdlibDeploymentTarget.Android.aarch64.name)
elif args.android_arch == "x86_64":
args.stdlib_deployment_targets.append(
StdlibDeploymentTarget.Android.x86_64.name)
# Infer platform flags from manually-specified configure targets.
# This doesn't apply to Darwin platforms, as they are
# already configured. No building without the platform flag, though.
android_tgts = [tgt for tgt in args.stdlib_deployment_targets
if StdlibDeploymentTarget.Android.contains(tgt)]
if not args.android and len(android_tgts) > 0:
# If building natively on an Android host, avoid the NDK
# cross-compilation configuration.
if not StdlibDeploymentTarget.Android.contains(StdlibDeploymentTarget
.host_target().name):
args.android = True
args.build_android = False
# Include the Darwin supported architectures in the CMake options.
if args.swift_darwin_supported_archs:
args.extra_cmake_options.append(
'-DSWIFT_DARWIN_SUPPORTED_ARCHS:STRING={}'.format(
args.swift_darwin_supported_archs))
# Remove unsupported Darwin archs from the standard library
# deployment targets.
supported_archs = args.swift_darwin_supported_archs.split(';')
targets = StdlibDeploymentTarget.get_targets_by_name(
args.stdlib_deployment_targets)
args.stdlib_deployment_targets = [
target.name
for target in targets
if (target.platform.is_darwin and
target.arch in supported_archs)
]
# Filter out any macOS stdlib deployment targets that are not supported
# by the macOS SDK.
targets = StdlibDeploymentTarget.get_targets_by_name(
args.stdlib_deployment_targets)
args.stdlib_deployment_targets = [
target.name
for target in targets
if (not target.platform.is_darwin or
target.platform.sdk_supports_architecture(
target.arch, args.darwin_xcrun_toolchain))
]
# Include the Darwin module-only architectures in the CMake options.
if args.swift_darwin_module_archs:
args.extra_cmake_options.append(
'-DSWIFT_DARWIN_MODULE_ARCHS:STRING={}'.format(
args.swift_darwin_module_archs))
# -----------------------------------------------------------------------------
# Build Script Impl Wrapping
class BuildScriptInvocation(object):
"""Represent a single build script invocation.
"""
def __init__(self, toolchain, args):
self.toolchain = toolchain
self.args = args
self.workspace = workspace.Workspace(
source_root=SWIFT_SOURCE_ROOT,
build_root=os.path.join(SWIFT_BUILD_ROOT, args.build_subdir))
self.build_libparser_only = args.build_libparser_only
@property
def install_all(self):
return self.args.install_all or self.args.infer_dependencies
def build_ninja(self):
if not os.path.exists(self.workspace.source_dir("ninja")):
fatal_error(
"can't find source directory for ninja "
"(tried %s)" % (self.workspace.source_dir("ninja")))
ninja_build = products.Ninja.new_builder(
args=self.args,
toolchain=self.toolchain,
workspace=self.workspace,
host=StdlibDeploymentTarget.get_target_for_name(
self.args.host_target))
ninja_build.build()
self.toolchain.ninja = ninja_build.ninja_bin_path
def convert_to_impl_arguments(self):
"""convert_to_impl_arguments() -> (env, args)
Convert the invocation to an environment and list of arguments suitable
for invoking `build-script-impl`.
"""
# Create local shadows, for convenience.
args = self.args
toolchain = self.toolchain
cmake = CMake(args=args,
toolchain=self.toolchain)
impl_args = [
"--workspace", self.workspace.source_root,
"--build-dir", self.workspace.build_root,
"--install-prefix", args.install_prefix,
"--host-target", args.host_target,
"--stdlib-deployment-targets={}".format(
" ".join(args.stdlib_deployment_targets)),
"--host-cc", toolchain.cc,
"--host-cxx", toolchain.cxx,
"--darwin-xcrun-toolchain", args.darwin_xcrun_toolchain,
"--darwin-deployment-version-osx=%s" % (
args.darwin_deployment_version_osx),
"--darwin-deployment-version-ios=%s" % (
args.darwin_deployment_version_ios),
"--darwin-deployment-version-tvos=%s" % (
args.darwin_deployment_version_tvos),
"--darwin-deployment-version-watchos=%s" % (
args.darwin_deployment_version_watchos),
"--cmake", toolchain.cmake,
"--cmark-build-type", args.cmark_build_variant,
"--llvm-build-type", args.llvm_build_variant,
"--swift-build-type", args.swift_build_variant,
"--swift-stdlib-build-type", args.swift_stdlib_build_variant,
"--lldb-build-type", args.lldb_build_variant,
"--foundation-build-type", args.foundation_build_variant,
"--libdispatch-build-type", args.libdispatch_build_variant,
"--libicu-build-type", args.libicu_build_variant,
"--xctest-build-type", args.build_variant,
"--llbuild-build-type", args.build_variant,
"--swift-enable-assertions", str(args.swift_assertions).lower(),
"--swift-stdlib-enable-assertions", str(
args.swift_stdlib_assertions).lower(),
"--swift-analyze-code-coverage", str(
args.swift_analyze_code_coverage).lower(),
"--llbuild-enable-assertions", str(
args.llbuild_assertions).lower(),
"--lldb-assertions", str(
args.lldb_assertions).lower(),
"--cmake-generator", args.cmake_generator,
"--build-jobs", str(args.build_jobs),
"--common-cmake-options=%s" % ' '.join(
pipes.quote(opt) for opt in cmake.common_options()),
"--build-args=%s" % ' '.join(
pipes.quote(arg) for arg in cmake.build_args()),
"--dsymutil-jobs", str(args.dsymutil_jobs),
]
# Compute any product specific cmake arguments.
#
# NOTE: The sum(list(...)) is b/c compute_product_classes returns a
# tuple of lists of which the first is the build-script-impl products
# and the second is the non-build-script-impl-products. It guarantees
# that when we concatenate these two lists together we get a valid
# dependency graph.
for product_class in sum(list(self.compute_product_classes()), []):
if not product_class.is_build_script_impl_product():
continue
product_name = product_class.product_name()
product_source_name = product_class.product_source_name()
source_dir = self.workspace.source_dir(product_source_name)
if not os.path.exists(source_dir):
fatal_error(
"can't find source directory for %s "
"(tried %s)" % (product_name, source_dir))
product = product_class(
args=args,
toolchain=self.toolchain,
source_dir=source_dir,
# FIXME: This is incorrect since it always assumes the host
# target I think?
build_dir=self.workspace.build_dir(
args.host_target, product_name))
cmake_opts = product.cmake_options
# FIXME: We should be using pipes.quote here but we run into issues
# with build-script-impl/cmake not being happy with all of the
# extra "'" in the strings. To fix this easily, we really need to
# just invoke cmake from build-script directly rather than futzing
# with build-script-impl. This makes even more sense since there
# really isn't a security issue here.
if cmake_opts:
impl_args += [
"--{}-cmake-options={}".format(
product_name, ' '.join(cmake_opts))
]
if args.build_stdlib_deployment_targets:
impl_args += [
"--build-stdlib-deployment-targets", " ".join(
args.build_stdlib_deployment_targets)]
if args.cross_compile_hosts:
impl_args += [
"--cross-compile-hosts", " ".join(args.cross_compile_hosts)]
if args.test_paths:
impl_args += ["--test-paths", " ".join(args.test_paths)]
if toolchain.ninja:
impl_args += ["--ninja-bin=%s" % toolchain.ninja]
if args.distcc:
impl_args += [
"--distcc",
"--distcc-pump=%s" % toolchain.distcc_pump
]
if args.sccache:
args.cmake_c_launcher = toolchain.sccache
args.cmake_cxx_launcher = toolchain.sccache
# *NOTE* We use normal cmake to pass through tsan/ubsan options. We do
# NOT pass them to build-script-impl.
if args.enable_asan:
impl_args += ["--enable-asan"]
# If we are on linux, disable leak detection when running ASAN. We
# have a separate bot that checks for leaks.
if platform.system() == 'Linux':
os.environ['ASAN_OPTIONS'] = 'detect_leaks=0'
if args.enable_ubsan:
impl_args += ["--enable-ubsan"]
# If we have lsan, we need to export our suppression list. The actual
# passing in of the LSAN flag is done via the normal cmake method. We
# do not pass the flag to build-script-impl.
if args.enable_lsan:
supp_file = os.path.join(SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME,
"utils",
"lsan_leaks_suppression_list.txt")
os.environ['LSAN_OPTIONS'] = 'suppressions={}'.format(supp_file)
if args.verbose_build:
impl_args += ["--verbose-build"]
if args.install_symroot:
impl_args += [
"--install-symroot", os.path.abspath(args.install_symroot)
]
if args.install_destdir:
impl_args += [
"--install-destdir", os.path.abspath(args.install_destdir)
]
if args.skip_build:
impl_args += ["--skip-build"]
if not args.build_benchmarks:
impl_args += ["--skip-build-benchmarks"]
if args.swift_disable_dead_stripping:
args.extra_cmake_options.append('-DSWIFT_DISABLE_DEAD_STRIPPING:BOOL=TRUE')
# Then add subproject install flags that either skip building them /or/
# if we are going to build them and install_all is set, we also install
# them.
conditional_subproject_configs = [
(args.build_cmark, "cmark"),
(args.build_llvm, "llvm"),
(args.build_swift, "swift"),
(args.build_foundation, "foundation"),
(args.build_xctest, "xctest"),
(args.build_lldb, "lldb"),
(args.build_llbuild, "llbuild"),
(args.build_libcxx, "libcxx"),
(args.build_libdispatch, "libdispatch"),
(args.build_libicu, "libicu")
]
for (should_build, string_name) in conditional_subproject_configs:
if not should_build and not self.args.infer_dependencies:
impl_args += ["--skip-build-{}".format(string_name)]
elif self.install_all:
impl_args += ["--install-{}".format(string_name)]
if args.build_swift_dynamic_stdlib:
impl_args += ["--build-swift-dynamic-stdlib"]
if args.build_swift_static_stdlib:
impl_args += ["--build-swift-static-stdlib"]
if args.build_swift_stdlib_unittest_extra:
impl_args += ["--build-swift-stdlib-unittest-extra"]
if args.build_swift_dynamic_sdk_overlay:
impl_args += ["--build-swift-dynamic-sdk-overlay"]
if args.build_swift_static_sdk_overlay:
impl_args += ["--build-swift-static-sdk-overlay"]
if not args.build_android:
impl_args += ["--skip-build-android"]
if not args.build_clang_tools_extra:
impl_args += ["--skip-build-clang-tools-extra"]
if not args.test and not args.long_test and not args.stress_test:
impl_args += ["--skip-test-swift"]
if not args.test:
impl_args += [
"--skip-test-cmark",
"--skip-test-lldb",
"--skip-test-llbuild",
"--skip-test-xctest",
"--skip-test-foundation",
"--skip-test-libdispatch",
"--skip-test-libicu",
]
if args.build_runtime_with_host_compiler:
impl_args += ["--build-runtime-with-host-compiler"]
if args.validation_test:
impl_args += ["--validation-test"]
if args.long_test:
impl_args += ["--long-test"]
if args.stress_test:
impl_args += ["--stress-test"]
if args.skip_local_build:
impl_args += ["--skip-local-build"]
if args.only_executable_test:
impl_args += ["--only-executable-test"]
if not args.benchmark:
impl_args += ["--skip-test-benchmarks"]
if args.build_libparser_only:
impl_args += ["--build-libparser-only"]
if args.android:
impl_args += [
"--android-arch", args.android_arch,
"--android-ndk", args.android_ndk,
"--android-api-level", args.android_api_level,
"--android-ndk-gcc-version", args.android_ndk_gcc_version,
"--android-icu-uc", args.android_icu_uc,
"--android-icu-uc-include", args.android_icu_uc_include,
"--android-icu-i18n", args.android_icu_i18n,
"--android-icu-i18n-include", args.android_icu_i18n_include,
"--android-icu-data", args.android_icu_data,
]
# If building natively on an Android host, only pass the API level.
if StdlibDeploymentTarget.Android.contains(StdlibDeploymentTarget
.host_target().name):
impl_args += ["--android-api-level", args.android_api_level]
if args.android_deploy_device_path:
impl_args += [
"--android-deploy-device-path",
args.android_deploy_device_path,
]
if platform.system() == 'Darwin':
impl_args += [
"--toolchain-prefix",
targets.darwin_toolchain_prefix(
args.install_prefix),
"--host-lipo", toolchain.lipo,
]
# Isolate build from the system; Darwin toolchains build against SDKs.
# For additional isolation, disable pkg-config. Homebrew's pkg-config
# prioritizes CommandLineTools paths, resulting in compile errors.
args.extra_cmake_options += [
'-DCMAKE_IGNORE_PATH=/usr/lib;/usr/local/lib;/lib',
'-DPKG_CONFIG_EXECUTABLE=/usr/bin/false',
]
if toolchain.libtool is not None:
impl_args += [
"--host-libtool", toolchain.libtool,
]
if args.native_clang_tools_path is not None:
impl_args += [
"--native-clang-tools-path=%s" % args.native_clang_tools_path
]
if args.native_llvm_tools_path is not None:
impl_args += [
"--native-llvm-tools-path=%s" % args.native_llvm_tools_path
]
if args.native_swift_tools_path is not None:
impl_args += [
"--native-swift-tools-path=%s" % args.native_swift_tools_path
]
# If we have extra_swift_args, combine all of them together and then
# add them as one command.
if args.extra_swift_args:
impl_args += [
"--extra-swift-args=%s" % ';'.join(args.extra_swift_args)
]
# Enable macCatalyst
if args.maccatalyst:
(args.extra_cmake_options
.append('-DSWIFT_ENABLE_MACCATALYST:BOOL=TRUE'))
if args.maccatalyst_ios_tests:
impl_args += ["--darwin-test-maccatalyst-ios-like=1"]
# If we have extra_cmake_options, combine all of them together and then
# add them as one command.
if args.extra_cmake_options:
impl_args += [
"--extra-cmake-options=%s" % ' '.join(
pipes.quote(opt) for opt in args.extra_cmake_options)
]
if args.lto_type is not None:
impl_args += [
"--llvm-enable-lto=%s" % args.lto_type,
"--swift-tools-enable-lto=%s" % args.lto_type
]
if args.llvm_max_parallel_lto_link_jobs is not None:
impl_args += [
"--llvm-num-parallel-lto-link-jobs=%s" %
min(args.llvm_max_parallel_lto_link_jobs, args.build_jobs)
]
if args.swift_tools_max_parallel_lto_link_jobs is not None:
impl_args += [
"--swift-tools-num-parallel-lto-link-jobs=%s" %
min(args.swift_tools_max_parallel_lto_link_jobs,
args.build_jobs)
]
impl_args += args.build_script_impl_args
if args.dry_run:
impl_args += ["--dry-run"]
if args.clang_profile_instr_use:
impl_args += [
"--clang-profile-instr-use=%s" %
os.path.abspath(args.clang_profile_instr_use)
]
if args.lit_args:
impl_args += ["--llvm-lit-args=%s" % args.lit_args]
if args.coverage_db:
impl_args += [
"--coverage-db=%s" %
os.path.abspath(args.coverage_db)
]
if args.llvm_install_components:
impl_args += [
"--llvm-install-components=%s" % args.llvm_install_components
]
if not args.clean_llbuild:
impl_args += [
"--skip-clean-llbuild"
]
if args.llvm_ninja_targets:
impl_args += [
"--llvm-ninja-targets=%s" % ' '.join(args.llvm_ninja_targets)
]
if args.llvm_ninja_targets_for_cross_compile_hosts:
impl_args += [
"--llvm-ninja-targets-for-cross-compile-hosts=%s" %
' '.join(args.llvm_ninja_targets_for_cross_compile_hosts)
]
# Compute the set of host-specific variables, which we pass through to
# the build script via environment variables.
host_specific_variables = self.compute_host_specific_variables()
impl_env = {}
for (host_target, options) in host_specific_variables.items():
for (name, value) in options.items():
# We mangle into an environment variable we can easily evaluate
# from the `build-script-impl`.
impl_env["HOST_VARIABLE_{}__{}".format(
host_target.replace("-", "_"), name)] = value
return (impl_env, impl_args)
def compute_host_specific_variables(self):
"""compute_host_specific_variables(args) -> dict
Compute the host-specific options, organized as a dictionary keyed by
host of options.
"""
args = self.args
options = {}
for host_target in [args.host_target] + args.cross_compile_hosts:
# Compute the host specific configuration.
try:
config = HostSpecificConfiguration(host_target, args)
except argparse.ArgumentError as e:
exit_rejecting_arguments(six.text_type(e))
# Convert into `build-script-impl` style variables.
options[host_target] = {
"SWIFT_SDKS": " ".join(sorted(
config.sdks_to_configure)),
"SWIFT_STDLIB_TARGETS": " ".join(
config.swift_stdlib_build_targets),
"SWIFT_BENCHMARK_TARGETS": " ".join(
config.swift_benchmark_build_targets),
"SWIFT_RUN_BENCHMARK_TARGETS": " ".join(
config.swift_benchmark_run_targets),
"SWIFT_TEST_TARGETS": " ".join(
config.swift_test_run_targets),
}
return options
def compute_product_classes(self):
"""compute_product_classes() -> (list, list)
Compute the list first of all build-script-impl products and then all
non-build-script-impl products. It is assumed that concatenating the two
lists together will result in a valid dependency graph for the
compilation.
"""
# FIXME: This is a weird division (returning a list of class objects),
# but it matches the existing structure of the `build-script-impl`.
impl_product_classes = []
if self.args.build_cmark:
impl_product_classes.append(products.CMark)
# If --skip-build-llvm is passed in, LLVM cannot be completely disabled, as
# Swift still needs a few LLVM targets like tblgen to be built for it to be
# configured. Instead, handle this in build-script-impl for now.
impl_product_classes.append(products.LLVM)
if self.args.build_libcxx:
impl_product_classes.append(products.LibCXX)
if self.args.build_libicu:
impl_product_classes.append(products.LibICU)
if self.args.build_swift:
impl_product_classes.append(products.Swift)
if self.args.build_lldb:
impl_product_classes.append(products.LLDB)
if self.args.build_libdispatch:
impl_product_classes.append(products.LibDispatch)
if self.args.build_foundation:
impl_product_classes.append(products.Foundation)
if self.args.build_xctest:
impl_product_classes.append(products.XCTest)
if self.args.build_llbuild:
impl_product_classes.append(products.LLBuild)
# Sanity check that all of our impl classes are actually
# build_script_impl products.
for prod in impl_product_classes:
assert(prod.is_build_script_impl_product())
product_classes = []
if self.args.build_swiftpm:
product_classes.append(products.SwiftPM)
if self.args.build_swiftsyntax:
product_classes.append(products.SwiftSyntax)
if self.args.build_skstresstester:
product_classes.append(products.SKStressTester)
if self.args.build_swiftformat:
product_classes.append(products.SwiftFormat)
if self.args.build_swiftevolve:
product_classes.append(products.SwiftEvolve)
if self.args.build_indexstoredb:
product_classes.append(products.IndexStoreDB)
if self.args.build_playgroundsupport:
product_classes.append(products.PlaygroundSupport)
if self.args.build_sourcekitlsp:
product_classes.append(products.SourceKitLSP)
if self.args.build_toolchainbenchmarks:
product_classes.append(products.Benchmarks)
if self.args.build_swift_inspect:
product_classes.append(products.SwiftInspect)
if self.args.tsan_libdispatch_test:
product_classes.append(products.TSanLibDispatch)
# Keep SwiftDriver at last.
# swift-driver's integration with the build scripts is not fully
# supported. Using swift-driver to build these products may hit
# failures.
if self.args.build_swift_driver or self.args.install_swift_driver:
product_classes.append(products.SwiftDriver)
# Sanity check that all of our non-impl classes are actually
# not build_script_impl products.
for prod in product_classes:
assert(not prod.is_build_script_impl_product())
# Now that we have our two lists of product_classes, if we are asked to
# infer dependencies, infer the dependencies now and then re-split the
# list.
if self.args.infer_dependencies:
combined = impl_product_classes + product_classes
if self.args.verbose_build:
print("-- Build Graph Inference --")
print("Initial Product List:")
for p in combined:
print(" {}".format(p.product_name()))
# Now that we have produced the schedule, resplit. We require our
# dependencies to respect our build-script-impl property. This means
# that no build-script-impl products can have dependencies on
# non-build-script-impl products. Otherwise, it would be unsafe to
# re-order build-script-impl products in front of non
# build-script-impl products.
impl_product_classes = []
product_classes = []
is_darwin = platform.system() == 'Darwin'
final_schedule = build_graph.produce_scheduled_build(combined)[0]
for p in final_schedule:
if is_darwin and p.is_nondarwin_only_build_product():
continue
if p.is_build_script_impl_product():
impl_product_classes.append(p)
else:
product_classes.append(p)
if self.args.verbose_build:
print("Final Build Order:")
for p in impl_product_classes:
print(" {}".format(p.product_name()))
for p in product_classes:
print(" {}".format(p.product_name()))
return (impl_product_classes, product_classes)
def execute(self):
"""Execute the invocation with the configured arguments."""
# Convert to a build-script-impl invocation.
(self.impl_env, self.impl_args) = self.convert_to_impl_arguments()
# If using the legacy implementation, delegate all behavior to
# `build-script-impl`.
if self.args.legacy_impl:
# Execute the underlying build script implementation.
shell.call_without_sleeping([BUILD_SCRIPT_IMPL_PATH] + self.impl_args,
env=self.impl_env, echo=True)
return
# Otherwise, we compute and execute the individual actions ourselves.
# Compute the list of hosts to operate on.
all_host_names = [
self.args.host_target] + self.args.cross_compile_hosts
all_hosts = [StdlibDeploymentTarget.get_target_for_name(name)
for name in all_host_names]
# Compute the list of product classes to operate on.
#
# FIXME: This should really be per-host, but the current structure
# matches that of `build-script-impl`.
(impl_product_classes, product_classes) = self.compute_product_classes()
# Execute each "pass".
# Build...
for host_target in all_hosts:
# FIXME: We should only compute these once.
try:
config = HostSpecificConfiguration(host_target.name, self.args)
except argparse.ArgumentError as e:
exit_rejecting_arguments(six.text_type(e))
print("Building the standard library for: {}".format(
" ".join(config.swift_stdlib_build_targets)))
if config.swift_test_run_targets and (
self.args.test or self.args.long_test):
print("Running Swift tests for: {}".format(
" ".join(config.swift_test_run_targets)))
if config.swift_benchmark_run_targets and self.args.benchmark:
print("Running Swift benchmarks for: {}".format(
" ".join(config.swift_benchmark_run_targets)))
for product_class in impl_product_classes:
self._execute_build_action(host_target, product_class)
# Test...
for host_target in all_hosts:
for product_class in impl_product_classes:
self._execute_test_action(host_target, product_class)
# Install...
for host_target in all_hosts:
for product_class in impl_product_classes:
self._execute_install_action(host_target, product_class)
# Core Lipo...
self._execute_merged_host_lipo_core_action()
# Non-build-script-impl products...
# Note: currently only supports building for the host.
for host_target in [self.args.host_target]:
for product_class in product_classes:
if product_class.is_build_script_impl_product():
continue
product_source = product_class.product_source_name()
product_name = product_class.product_name()
if product_class.is_swiftpm_unified_build_product():
build_dir = self.workspace.swiftpm_unified_build_dir(
host_target)
else:
build_dir = self.workspace.build_dir(
host_target, product_name)
product = product_class(
args=self.args,
toolchain=self.toolchain,
source_dir=self.workspace.source_dir(product_source),
build_dir=build_dir)
if product.should_clean(host_target):
print("--- Cleaning %s ---" % product_name)
product.clean(host_target)
if product.should_build(host_target):
print("--- Building %s ---" % product_name)
product.build(host_target)
if product.should_test(host_target):
print("--- Running tests for %s ---" % product_name)
product.test(host_target)
print("--- Finished tests for %s ---" % product_name)
if product.should_install(host_target) or \
(self.install_all and product.should_build(host_target)):
print("--- Installing %s ---" % product_name)
product.install(host_target)
# Extract symbols...
for host_target in all_hosts:
self._execute_extract_symbols_action(host_target)
# Package...
for host_target in all_hosts:
self._execute_package_action(host_target)
# Lipo...
self._execute_merged_host_lipo_action()
def _execute_build_action(self, host_target, product_class):
action_name = "{}-{}-build".format(host_target.name,
product_class.product_name())
self._execute_action(action_name)
def _execute_test_action(self, host_target, product_class):
action_name = "{}-{}-test".format(host_target.name,
product_class.product_name())
self._execute_action(action_name)
def _execute_install_action(self, host_target, product_class):
action_name = "{}-{}-install".format(host_target.name,
product_class.product_name())
self._execute_action(action_name)
def _execute_extract_symbols_action(self, host_target):
action_name = "{}-extractsymbols".format(host_target.name)
self._execute_action(action_name)
def _execute_package_action(self, host_target):
action_name = "{}-package".format(host_target.name)
self._execute_action(action_name)
def _execute_merged_host_lipo_action(self):
self._execute_action("merged-hosts-lipo")
def _execute_merged_host_lipo_core_action(self):
self._execute_action("merged-hosts-lipo-core")
def _execute_action(self, action_name):
shell.call_without_sleeping(
[BUILD_SCRIPT_IMPL_PATH] + self.impl_args +
["--only-execute", action_name],
env=self.impl_env, echo=self.args.verbose_build)
# -----------------------------------------------------------------------------
# Main (preset)
def parse_preset_args():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description="""Builds Swift using a preset.""")
parser.add_argument(
"-n", "--dry-run",
help="print the commands that would be executed, but do not execute "
"them",
action="store_true",
default=False)
parser.add_argument(
"--preset-file",
help="load presets from the specified file",
metavar="PATH",
action="append",
dest="preset_file_names",
default=[])
parser.add_argument(
"--preset",
help="use the specified option preset",
metavar="NAME")
parser.add_argument(
"--show-presets",
help="list all presets and exit",
action=argparse.actions.StoreTrueAction,
nargs=argparse.Nargs.OPTIONAL)
parser.add_argument(
"--distcc",
help="use distcc",
action=argparse.actions.StoreTrueAction,
nargs=argparse.Nargs.OPTIONAL,
default=os.environ.get('USE_DISTCC') == '1')
parser.add_argument(
"--sccache",
help="use sccache",
action=argparse.actions.StoreTrueAction,
nargs=argparse.Nargs.OPTIONAL,
default=os.environ.get('SWIFT_USE_SCCACHE') == '1')
parser.add_argument(
"--cmake-c-launcher",
help="the absolute path to set CMAKE_C_COMPILER_LAUNCHER",
metavar="PATH")
parser.add_argument(
"--cmake-cxx-launcher",
help="the absolute path to set CMAKE_CXX_COMPILER_LAUNCHER",
metavar="PATH")
parser.add_argument(
"-j", "--jobs",
help="the number of parallel build jobs to use",
type=int,
dest="build_jobs")
parser.add_argument(
"preset_substitutions_raw",
help="'name=value' pairs that are substituted in the preset",
nargs="*",
metavar="SUBSTITUTION")
parser.add_argument(
"--expand-invocation", "--expand-build-script-invocation",
help="Print the expanded build-script invocation generated "
"by the preset, but do not run the preset",
action=argparse.actions.StoreTrueAction,
nargs=argparse.Nargs.OPTIONAL)
parser.add_argument(
"--swiftsyntax-install-prefix",
help="specify the directory to where SwiftSyntax should be installed")
parser.add_argument(
"--build-dir",
help="specify the directory where build artifact should be stored")
parser.add_argument(
"--dump-config",
help="instead of building, write JSON to stdout containing "
"various values used to build in this configuration",
action="store_true",
default=False)
parser.add_argument(
"--reconfigure",
help="Reconfigure all projects as we build",
action="store_true",
default=False)
return parser.parse_args()
def main_preset():
args = parse_preset_args()
if len(args.preset_file_names) == 0:
args.preset_file_names = [
os.path.join(
SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME, "utils",
"build-presets.ini")
]
user_presets_file = os.path.join(os.path.expanduser("~"),
'.swift-build-presets')
if os.path.isfile(user_presets_file):
args.preset_file_names.append(user_presets_file)
preset_parser = presets.PresetParser()
try:
preset_parser.read_files(args.preset_file_names)
except presets.PresetError as e:
fatal_error(six.text_type(e))
if args.show_presets:
for name in sorted(preset_parser.preset_names,
key=lambda name: name.lower()):
print(name)
return 0
if not args.preset:
fatal_error("missing --preset option")
args.preset_substitutions = {}
for arg in args.preset_substitutions_raw:
name, value = arg.split("=", 1)
args.preset_substitutions[name] = value
try:
preset = preset_parser.get_preset(
args.preset,
vars=args.preset_substitutions)
except presets.PresetError as e:
fatal_error(six.text_type(e))
preset_args = migration.migrate_swift_sdks(preset.args)
if args.distcc and (args.cmake_c_launcher or args.cmake_cxx_launcher):
fatal_error(
'--distcc can not be used with' +
' --cmake-c-launcher or --cmake-cxx-launcher')
if args.sccache and (args.cmake_c_launcher or args.cmake_cxx_launcher):
fatal_error(
'--sccache can not be used with' +
' --cmake-c-launcher or --cmake-cxx-launcher')
build_script_args = [sys.argv[0]]
if args.dry_run:
build_script_args += ["--dry-run"]
if args.dump_config:
build_script_args += ["--dump-config"]
build_script_args += preset_args
if args.distcc:
build_script_args += ["--distcc"]
if args.sccache:
build_script_args += ["--sccache"]
if args.build_jobs:
build_script_args += ["--jobs", str(args.build_jobs)]
if args.swiftsyntax_install_prefix:
build_script_args += ["--install-swiftsyntax",
"--install-destdir",
args.swiftsyntax_install_prefix]
if args.build_dir:
build_script_args += ["--build-dir", args.build_dir]
if args.cmake_c_launcher:
build_script_args += ["--cmake-c-launcher", args.cmake_c_launcher]
if args.cmake_cxx_launcher:
build_script_args += ["--cmake-cxx-launcher", args.cmake_cxx_launcher]
printable_command = shell.quote_command(build_script_args)
if args.expand_invocation:
print(printable_command)
return 0
if args.reconfigure:
build_script_args += ["--reconfigure"]
print_note('using preset "{}", which expands to \n\n{}\n'.format(
args.preset, printable_command))
shell.call_without_sleeping(build_script_args)
return 0
# -----------------------------------------------------------------------------
# Main (normal)
def main_normal():
parser = driver_arguments.create_argument_parser()
args = migration.parse_args(parser, sys.argv[1:])
if args.build_script_impl_args:
# If we received any impl args, check if `build-script-impl` would
# accept them or not before any further processing.
try:
migration.check_impl_args(BUILD_SCRIPT_IMPL_PATH,
args.build_script_impl_args)
except ValueError as e:
exit_rejecting_arguments(e, parser)
if '--check-args-only' in args.build_script_impl_args:
return 0
shell.dry_run = args.dry_run
# Prepare and validate toolchain
if args.darwin_xcrun_toolchain is None:
xcrun_toolchain = os.environ.get('TOOLCHAINS',
defaults.DARWIN_XCRUN_TOOLCHAIN)
print_note('Using toolchain {}'.format(xcrun_toolchain))
args.darwin_xcrun_toolchain = xcrun_toolchain
toolchain = host_toolchain(xcrun_toolchain=args.darwin_xcrun_toolchain)
os.environ['TOOLCHAINS'] = args.darwin_xcrun_toolchain
if args.host_cc is not None:
toolchain.cc = args.host_cc
if args.host_cxx is not None:
toolchain.cxx = args.host_cxx
if args.host_lipo is not None:
toolchain.lipo = args.host_lipo
if args.host_libtool is not None:
toolchain.libtool = args.host_libtool
if args.cmake is not None:
toolchain.cmake = args.cmake
if args.sccache:
print("Ensuring the sccache server is running...")
# Use --show-stats to ensure the server is started, because using
# --start-server will fail if the server is already running.
# Capture the output because we don't want to see the stats.
shell.capture([toolchain.sccache, "--show-stats"])
cmake = CMake(args=args, toolchain=toolchain)
# Check the CMake version is sufficient on Linux and build from source
# if not.
cmake_path = cmake.check_cmake_version(SWIFT_SOURCE_ROOT, SWIFT_BUILD_ROOT)
if cmake_path is not None:
toolchain.cmake = cmake_path
args.cmake = cmake_path
# Preprocess the arguments to apply defaults.
apply_default_arguments(toolchain, args)
# Validate the arguments.
validate_arguments(toolchain, args)
# Create the build script invocation.
invocation = BuildScriptInvocation(toolchain, args)
# Sanitize the runtime environment.
initialize_runtime_environment()
# Show SDKs, if requested.
if args.show_sdks:
print_xcodebuild_versions()
if args.dump_config:
print(JSONDumper().encode(invocation))
return 0
# Clean build directory if requested.
if args.clean:
clean_delay()
shell.rmtree(invocation.workspace.build_root)
# Create build directory.
shell.makedirs(invocation.workspace.build_root)
# Build ninja if required, which will update the toolchain.
if args.build_ninja:
invocation.build_ninja()
# Execute the underlying build script implementation.
invocation.execute()
if args.symbols_package:
print('--- Creating symbols package ---')
print('-- Package file: {} --'.format(args.symbols_package))
if platform.system() == 'Darwin':
prefix = targets.darwin_toolchain_prefix(args.install_prefix)
prefix = os.path.join(args.host_target, prefix.lstrip('/'))
else:
prefix = args.install_prefix
# As a security measure, `tar` normally strips leading '/' from paths
# it is archiving. To stay safe, we change working directories, then
# run `tar` without the leading '/' (we remove it ourselves to keep
# `tar` from emitting a warning).
with shell.pushd(args.install_symroot):
tar(source=prefix.lstrip('/'),
destination=args.symbols_package)
return 0
# -----------------------------------------------------------------------------
def main():
if not SWIFT_SOURCE_ROOT:
fatal_error(
"could not infer source root directory " +
"(forgot to set $SWIFT_SOURCE_ROOT environment variable?)")
if not os.path.isdir(SWIFT_SOURCE_ROOT):
fatal_error(
"source root directory \'" + SWIFT_SOURCE_ROOT +
"\' does not exist " +
"(forgot to set $SWIFT_SOURCE_ROOT environment variable?)")
validate_xcode_compatibility()
# Determine if we are invoked in the preset mode and dispatch accordingly.
if any([(opt.startswith("--preset") or opt == "--show-presets")
for opt in sys.argv[1:]]):
return main_preset()
else:
return main_normal()
if __name__ == "__main__":
try:
sys.exit(main())
except KeyboardInterrupt:
sys.exit(1)
Reference in New Issue View Git Blame Copy Permalink