mirror of
https://github.com/apple/swift.git
synced 2025-12-14 20:36:38 +01:00
283713d61d
Stress tests are, by definition, stressful. They intentionally burn a lot of resources by using randomness to hopefully surface state machine bugs. Additionally, many stress tests are multi-threaded these days and they may attempt to use all of the available CPUs to better uncover bugs. In isolation, this is not a problem, but the test suite as a whole assumes that individual tests are single threaded and therefore running multiple stress tests at once can quickly spiral out of control. This change formalizes stress tests and then treats them like long tests, i.e. tested via 'check-swift-all' and otherwise opt-in. Finally, with this change, the CI build bots might need to change if they are still only testing 'validation' instead of all of the tests. I see three options: 1) Run all of the tests. -- There are very few long tests left these days, and the additional costs seems small relative to the cost of the whole validation test suite before this change. 2) Continue checking 'validation', now sans stress tests. 3) Check 'validation', *then* the stress tests. If the former doesn't pass, then there is no point in the latter, and by running the stress tests separately, they stand a better chance of uncovering bugs and not overwhelming build bot resources.
255 lines
9.0 KiB
Python
Executable File
255 lines
9.0 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# utils/run-test - test runner for Swift -*- python -*-
|
|
#
|
|
# This source file is part of the Swift.org open source project
|
|
#
|
|
# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
|
|
# Licensed under Apache License v2.0 with Runtime Library Exception
|
|
#
|
|
# See https://swift.org/LICENSE.txt for license information
|
|
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
|
|
|
|
from __future__ import print_function
|
|
|
|
import argparse
|
|
import multiprocessing
|
|
import os
|
|
import shutil
|
|
import sys
|
|
|
|
from swift_build_support.swift_build_support import (
|
|
arguments,
|
|
shell
|
|
)
|
|
|
|
from swift_build_support.swift_build_support.SwiftBuildSupport import \
|
|
SWIFT_SOURCE_ROOT
|
|
|
|
from swift_build_support.swift_build_support.targets import \
|
|
StdlibDeploymentTarget
|
|
|
|
|
|
TEST_MODES = [
|
|
'optimize_none',
|
|
'optimize',
|
|
'optimize_unchecked',
|
|
'only_executable',
|
|
'only_non_executable',
|
|
]
|
|
TEST_SUBSETS = [
|
|
'primary',
|
|
'validation',
|
|
'all',
|
|
'only_validation',
|
|
'only_long',
|
|
'only_stress',
|
|
]
|
|
|
|
SWIFT_SOURCE_DIR = os.path.join(SWIFT_SOURCE_ROOT, 'swift')
|
|
TEST_SOURCE_DIR = os.path.join(SWIFT_SOURCE_DIR, 'test')
|
|
VALIDATION_TEST_SOURCE_DIR = os.path.join(SWIFT_SOURCE_DIR, 'validation-test')
|
|
|
|
LIT_BIN_DEFAULT = os.path.join(SWIFT_SOURCE_ROOT, 'llvm',
|
|
'utils', 'lit', 'lit.py')
|
|
host_target = StdlibDeploymentTarget.host_target().name
|
|
|
|
|
|
def error_exit(msg):
|
|
print("%s: %s" % (os.path.basename(sys.argv[0]), msg), file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
# Return true if the path looks like swift build directory.
|
|
def is_swift_build_dir(path):
|
|
return (os.path.exists(os.path.join(path, "CMakeCache.txt")) and
|
|
os.path.isdir(os.path.join(path, "test-%s" % host_target)))
|
|
|
|
|
|
# Return true if the swift build directory is configured with `Xcode`
|
|
# generator.
|
|
def is_build_dir_xcode(path):
|
|
return os.path.exists(os.path.join(path, 'Swift.xcodeproj'))
|
|
|
|
|
|
# Return true if 'path' is sub path of 'd'
|
|
def is_subpath(path, d):
|
|
path, d = os.path.abspath(path), os.path.abspath(d)
|
|
if os.path.isdir(path):
|
|
path = os.path.join(path, '')
|
|
d = os.path.join(d, '')
|
|
return path.startswith(d)
|
|
|
|
|
|
# Convert test path in source directory to corresponding path in build
|
|
# directory. If the path is not sub path of test directories in source,
|
|
# return the path as is.
|
|
def normalize_test_path(path, build_dir, variant):
|
|
for d, prefix in [(TEST_SOURCE_DIR, 'test-%s'),
|
|
(VALIDATION_TEST_SOURCE_DIR, 'validation-test-%s')]:
|
|
if is_subpath(path, d):
|
|
return os.path.normpath(os.path.join(
|
|
build_dir,
|
|
prefix % variant,
|
|
os.path.relpath(path, d)))
|
|
return path
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("paths", type=os.path.realpath,
|
|
nargs="*", metavar="PATH",
|
|
help="paths to test. Accept multiple. "
|
|
"If --build-dir is not specified, these paths "
|
|
"must be test paths in the Swift build "
|
|
"directory. (default: primary test suite if "
|
|
"--build-dir is specified, none otherwise)")
|
|
parser.add_argument("-v", "--verbose", action="store_true",
|
|
help="run test with verbose output")
|
|
parser.add_argument("--build-dir", type=os.path.realpath, metavar="PATH",
|
|
help="Swift build directory")
|
|
parser.add_argument("--build",
|
|
choices=["true", "verbose", "skip"], default='true',
|
|
help="build test dependencies before running tests "
|
|
"(default: true)")
|
|
parser.add_argument("--target",
|
|
type=arguments.type.shell_split,
|
|
action=arguments.action.concat,
|
|
dest="targets",
|
|
help="stdlib deployment targets to test. Accept "
|
|
"multiple (default: " + host_target + ")")
|
|
parser.add_argument("--mode",
|
|
choices=TEST_MODES, default='optimize_none',
|
|
help="test mode (default: optimize_none)")
|
|
parser.add_argument("--subset",
|
|
choices=TEST_SUBSETS, default='primary',
|
|
help="test subset (default: primary)")
|
|
parser.add_argument("--param",
|
|
type=arguments.type.shell_split,
|
|
action=arguments.action.concat,
|
|
default=[],
|
|
help="key=value parameters they are directly passed "
|
|
"to lit command in addition to `mode` and "
|
|
"`subset`. Accept multiple.")
|
|
parser.add_argument("--result-dir", type=os.path.realpath, metavar="PATH",
|
|
help="directory to store test results (default: none)")
|
|
parser.add_argument("--lit", default=LIT_BIN_DEFAULT, metavar="PATH",
|
|
help="lit.py executable path "
|
|
"(default: ${LLVM_SOURCE_DIR}/utils/lit/lit.py)")
|
|
|
|
args = parser.parse_args()
|
|
|
|
targets = args.targets
|
|
if targets is None:
|
|
targets = [host_target]
|
|
|
|
paths = []
|
|
|
|
build_dir = args.build_dir
|
|
if build_dir is not None:
|
|
# Fixup build directory.
|
|
# build_dir can be:
|
|
# build-root/ # assuming we are to test host deployment target.
|
|
# build-root/swift-{tool-deployment_target}/
|
|
for d in [
|
|
build_dir,
|
|
os.path.join(build_dir, 'swift-%s' % host_target)]:
|
|
if is_swift_build_dir(d):
|
|
build_dir = d
|
|
break
|
|
else:
|
|
error_exit("'%s' is not a swift build directory" % args.build_dir)
|
|
|
|
# If no path given, run primary test suite.
|
|
if not args.paths:
|
|
args.paths = [TEST_SOURCE_DIR]
|
|
|
|
# $ run-test --build-dir=<swift-build-dir> <test-dir-in-source> ... \
|
|
# --target macosx-x86_64 --target iphonesimulator-i386
|
|
for target in targets:
|
|
paths += map(
|
|
lambda p: normalize_test_path(p, build_dir, target),
|
|
args.paths)
|
|
|
|
else:
|
|
# Otherwise, we assume all given paths are valid test paths in the
|
|
# build_dir.
|
|
paths = args.paths
|
|
if not paths:
|
|
parser.print_usage()
|
|
error_exit("error: too few arguments")
|
|
|
|
if args.build != 'skip':
|
|
# Building dependencies requires `build_dir` set.
|
|
# Traverse the first test path to find the `build_dir`
|
|
d = os.path.dirname(paths[0])
|
|
while d not in ['', os.sep]:
|
|
if is_swift_build_dir(d):
|
|
build_dir = d
|
|
break
|
|
d = os.path.dirname(d)
|
|
else:
|
|
error_exit("Can't infer swift build directory")
|
|
|
|
# Ensure we have up to date test dependency
|
|
if args.build != 'skip' and is_build_dir_xcode(build_dir):
|
|
# We don't support Xcode Generator build yet.
|
|
print("warning: Building Xcode project is not supported yet. "
|
|
"Skipping...")
|
|
sys.stdout.flush()
|
|
|
|
elif args.build != 'skip':
|
|
dependency_targets = ["all", "SwiftUnitTests"]
|
|
upload_stdlib_targets = []
|
|
need_validation = any('/validation-test-' in path for path in paths)
|
|
for target in targets:
|
|
upload_stdlib_targets += ["upload-stdlib-%s" % target]
|
|
if need_validation:
|
|
dependency_targets += ["swift-stdlib-%s" % target]
|
|
else:
|
|
dependency_targets += ["swift-test-stdlib-%s" % target]
|
|
|
|
cmake_build = ['cmake', '--build', build_dir, '--']
|
|
if args.build == 'verbose':
|
|
cmake_build += ['-v']
|
|
cmake_build += ['-j%d' % multiprocessing.cpu_count()]
|
|
|
|
print("--- Building test dependencies %s ---" %
|
|
', '.join(dependency_targets))
|
|
sys.stdout.flush()
|
|
shell.call(cmake_build + dependency_targets)
|
|
shell.call(cmake_build + upload_stdlib_targets)
|
|
print("--- Build finished ---")
|
|
sys.stdout.flush()
|
|
|
|
if args.result_dir is not None:
|
|
# Clear result directory
|
|
if os.path.exists(args.result_dir):
|
|
shutil.rmtree(args.result_dir)
|
|
os.makedirs(args.result_dir)
|
|
|
|
if args.verbose:
|
|
test_args = ["-a"]
|
|
else:
|
|
test_args = ["-sv"]
|
|
|
|
# Test parameters.
|
|
test_args += ['--param', 'swift_test_mode=%s' % args.mode,
|
|
'--param', 'swift_test_subset=%s' % args.subset]
|
|
|
|
for param in args.param:
|
|
test_args += ['--param', param]
|
|
|
|
if args.result_dir:
|
|
test_args += ['--param', 'swift_test_results_dir=%s' % args.result_dir,
|
|
'--xunit-xml-output=%s' % os.path.join(args.result_dir,
|
|
'lit-tests.xml')]
|
|
|
|
test_cmd = [sys.executable, args.lit] + test_args + paths
|
|
|
|
# Do execute test
|
|
shell.call(test_cmd)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|