#!/usr/bin/env python3 # utils/run-test - test runner for Swift -*- python -*- # # This source file is part of the Swift.org open source project # # Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors # Licensed under Apache License v2.0 with Runtime Library Exception # # See https://swift.org/LICENSE.txt for license information # See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors import multiprocessing import os import shutil import sys from build_swift.build_swift import argparse from build_swift.build_swift.constants import SWIFT_SOURCE_ROOT from swift_build_support.swift_build_support import shell from swift_build_support.swift_build_support.targets import StdlibDeploymentTarget TEST_MODES = [ 'optimize_none', 'optimize', 'optimize_unchecked', 'only_executable', 'only_non_executable', ] TEST_SUBSETS = [ 'primary', 'validation', 'all', 'only_validation', 'only_long', 'only_stress', ] SWIFT_SOURCE_DIR = os.path.join(SWIFT_SOURCE_ROOT, 'swift') TEST_SOURCE_DIR = os.path.join(SWIFT_SOURCE_DIR, 'test') VALIDATION_TEST_SOURCE_DIR = os.path.join(SWIFT_SOURCE_DIR, 'validation-test') def _get_default_llvm_source_dir(): legacy_llvm_dir_path = os.path.join(SWIFT_SOURCE_ROOT, 'llvm') if os.path.isdir(legacy_llvm_dir_path): return legacy_llvm_dir_path return os.path.join(SWIFT_SOURCE_ROOT, 'llvm-project', 'llvm') # Default path for "lit.py" executable. LIT_BIN_DEFAULT = os.path.join(os.environ.get("LLVM_SOURCE_DIR", _get_default_llvm_source_dir()), 'utils', 'lit', 'lit.py') host_target = StdlibDeploymentTarget.host_target().name def error_exit(msg): print("%s: %s" % (os.path.basename(sys.argv[0]), msg), file=sys.stderr) sys.exit(1) # Return true if the path looks like swift build directory. def is_swift_build_dir(path, unified_build_dir): if not unified_build_dir: tests_path = [path, "test-%s" % host_target] else: tests_path = [path, "tools", "swift", "test-%s" % host_target] return (os.path.exists(os.path.join(path, "CMakeCache.txt")) and os.path.isdir(os.path.join(*tests_path))) # Return true if the swift build directory is configured with `Xcode` # generator. def is_build_dir_xcode(path): return os.path.exists(os.path.join(path, 'Swift.xcodeproj')) # Return true if 'path' is sub path of 'd' def is_subpath(path, d): path, d = os.path.abspath(path), os.path.abspath(d) if os.path.isdir(path): path = os.path.join(path, '') d = os.path.join(d, '') return path.startswith(d) # Convert test path in source directory to corresponding path in build # directory. If the path is not sub path of test directories in source, # return the path as is. def normalize_test_path(path, build_dir, variant, unified_build_dir): if not unified_build_dir: tests_path = [build_dir] else: tests_path = [build_dir, "tools", "swift"] for d, prefix in [(TEST_SOURCE_DIR, 'test-%s'), (VALIDATION_TEST_SOURCE_DIR, 'validation-test-%s')]: if is_subpath(path, d): return os.path.normpath(os.path.join(*( tests_path + [prefix % variant, os.path.relpath(path, d)]))) return path def main(): parser = argparse.ArgumentParser() parser.add_argument("paths", type=os.path.realpath, nargs="*", metavar="PATH", help="paths to test. Accept multiple. " "If --build-dir is not specified, these paths " "must be test paths in the Swift build " "directory. (default: primary test suite if " "--build-dir is specified, none otherwise)") parser.add_argument("-v", "--verbose", action="store_true", help="run test with verbose output") parser.add_argument("--build-dir", type=os.path.realpath, metavar="PATH", help="Swift build directory") parser.add_argument("--build", choices=["true", "verbose", "skip"], default='true', help="build test dependencies before running tests " "(default: true)") parser.add_argument("--build-jobs", type=int, help="The number of parallel build jobs to use") parser.add_argument("--color", choices=["true", "false"], default="true", help="Whether to enable color output (default: true)") parser.add_argument("--target", type=argparse.types.ShellSplitType(), action=argparse.actions.AppendAction, dest="targets", help="stdlib deployment targets to test. Accept " "multiple (default: " + host_target + ")") parser.add_argument("--filter", type=str, metavar="REGEX", help="only run tests with paths matching the given " "regular expression") parser.add_argument("--mode", choices=TEST_MODES, default='optimize_none', help="test mode (default: optimize_none)") parser.add_argument("--subset", choices=TEST_SUBSETS, default='primary', help="test subset (default: primary)") parser.add_argument("--param", type=argparse.types.ShellSplitType(), action=argparse.actions.AppendAction, default=[], help="key=value parameters they are directly passed " "to lit command in addition to `mode` and " "`subset`. Accept multiple.") parser.add_argument("--result-dir", type=os.path.realpath, metavar="PATH", help="directory to store test results (default: none)") parser.add_argument("--lit", default=LIT_BIN_DEFAULT, metavar="PATH", help="lit.py executable path " "(default: ${LLVM_SOURCE_DIR}/utils/lit/lit.py)") parser.add_argument("--unified", action="store_true", help="The build directory is an unified LLVM build, " "not a standalone Swift build") args = parser.parse_args() targets = args.targets if not targets: targets = [host_target] paths = [] build_dir = args.build_dir if build_dir is not None: # Fixup build directory. # build_dir can be: # build-root/ # assuming we are to test host deployment target. # build-root/swift-{tool-deployment_target}/ for d in [ build_dir, os.path.join(build_dir, 'swift-%s' % host_target)]: if is_swift_build_dir(d, args.unified): build_dir = d break else: error_exit("'%s' is not a swift build directory" % args.build_dir) # If no path given, run primary test suite. if not args.paths: args.paths = [TEST_SOURCE_DIR] # $ run-test --build-dir= ... \ # --target macosx-x86_64 --target iphonesimulator-i386 for target in targets: paths += [normalize_test_path(p, build_dir, target, args.unified) for p in args.paths] else: # Otherwise, we assume all given paths are valid test paths in the # build_dir. paths = args.paths if not paths: parser.print_usage() error_exit("error: too few arguments") if args.build != 'skip': # Building dependencies requires `build_dir` set. # Traverse the first test path to find the `build_dir` d = os.path.dirname(paths[0]) while d not in ['', os.sep]: if is_swift_build_dir(d, args.unified): build_dir = d break d = os.path.dirname(d) else: error_exit("Can't infer swift build directory") # Ensure we have up to date test dependency if args.build != 'skip' and is_build_dir_xcode(build_dir): # We don't support Xcode Generator build yet. print("warning: Building Xcode project is not supported yet. " "Skipping...") sys.stdout.flush() elif args.build != 'skip': dependency_targets = ["all", "SwiftUnitTests"] upload_stdlib_targets = [] need_validation = any('/validation-test-' in path for path in paths) for target in targets: if args.mode != 'only_non_executable': upload_stdlib_targets += ["upload-stdlib-%s" % target] if need_validation: dependency_targets += ["swift-stdlib-%s" % target] else: dependency_targets += ["swift-test-stdlib-%s" % target] cmake_build = ['cmake', '--build', build_dir, '--'] if args.build == 'verbose': cmake_build += ['-v'] if args.build_jobs is not None: cmake_build += ['-j%d' % args.build_jobs] else: cmake_build += ['-j%d' % multiprocessing.cpu_count()] print("--- Building test dependencies %s ---" % ', '.join(dependency_targets)) sys.stdout.flush() shell.call(cmake_build + dependency_targets) shell.call(cmake_build + upload_stdlib_targets) print("--- Build finished ---") sys.stdout.flush() if args.result_dir is not None: # Clear result directory if os.path.exists(args.result_dir): shutil.rmtree(args.result_dir) os.makedirs(args.result_dir) if args.verbose: test_args = ["-a"] else: test_args = ["-sv"] # Test parameters. test_args += ['--param', 'swift_test_mode=%s' % args.mode, '--param', 'swift_test_subset=%s' % args.subset] for param in args.param: test_args += ['--param', param] # Enable colors if we're on a tty. if args.color == 'true' and sys.stdout.isatty(): test_args += ['--param', 'color_output'] if args.result_dir: test_args += ['--param', 'swift_test_results_dir=%s' % args.result_dir, '--xunit-xml-output=%s' % os.path.join(args.result_dir, 'lit-tests.xml')] if args.filter: test_args += ['--filter', args.filter] test_cmd = [sys.executable, args.lit] + test_args + paths # Do execute test shell.call(test_cmd) if __name__ == "__main__": main()