mirror of
https://github.com/apple/swift.git
synced 2025-12-21 12:14:44 +01:00
283713d61d
Stress tests are, by definition, stressful. They intentionally burn a lot of resources by using randomness to hopefully surface state machine bugs. Additionally, many stress tests are multi-threaded these days and they may attempt to use all of the available CPUs to better uncover bugs. In isolation, this is not a problem, but the test suite as a whole assumes that individual tests are single threaded and therefore running multiple stress tests at once can quickly spiral out of control. This change formalizes stress tests and then treats them like long tests, i.e. tested via 'check-swift-all' and otherwise opt-in. Finally, with this change, the CI build bots might need to change if they are still only testing 'validation' instead of all of the tests. I see three options: 1) Run all of the tests. -- There are very few long tests left these days, and the additional costs seems small relative to the cost of the whole validation test suite before this change. 2) Continue checking 'validation', now sans stress tests. 3) Check 'validation', *then* the stress tests. If the former doesn't pass, then there is no point in the latter, and by running the stress tests separately, they stand a better chance of uncovering bugs and not overwhelming build bot resources.
75 lines
2.8 KiB
Python
75 lines
2.8 KiB
Python
# test/Unit/lit.cfg - Configuration for the 'lit' test runner. -*- python -*-
|
|
#
|
|
# This source file is part of the Swift.org open source project
|
|
#
|
|
# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
|
|
# Licensed under Apache License v2.0 with Runtime Library Exception
|
|
#
|
|
# See https://swift.org/LICENSE.txt for license information
|
|
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
|
|
|
|
import os
|
|
|
|
import lit.formats
|
|
import lit.util
|
|
|
|
###
|
|
|
|
# Check that the object root is known.
|
|
swift_obj_root = getattr(config, 'swift_obj_root', None)
|
|
if swift_obj_root is None:
|
|
# Check for 'swift_unit_site_config' user parameter, and use that if available.
|
|
site_cfg = lit_config.params.get('swift_unit_site_config', None)
|
|
if not site_cfg:
|
|
base_site_cfg = lit_config.params.get('swift_site_config', None)
|
|
if base_site_cfg:
|
|
test_dir = os.path.dirname(base_site_cfg)
|
|
site_cfg = os.path.join(test_dir, 'Unit', 'lit.site.cfg')
|
|
|
|
if site_cfg and os.path.exists(site_cfg):
|
|
lit_config.load_config(config, site_cfg)
|
|
raise SystemExit
|
|
|
|
lit_config.fatal("lit must be pointed at a build folder")
|
|
|
|
###
|
|
|
|
# name: The name of this test suite.
|
|
config.name = 'Swift-Unit'
|
|
|
|
# suffixes: A list of file extensions to treat as test files.
|
|
config.suffixes = []
|
|
|
|
# excludes: A path component to ignore when discovering tests.
|
|
# The Swift build system does not generate DWARF symbols in the unittests build
|
|
# directory. For build systems that do, however, files would be generated with
|
|
# the test suffix "Tests". These need to be excluded because otherwise
|
|
# `lit.formats.GoogleTest` tries to execute them.
|
|
# See http://reviews.llvm.org/D18647 for details.
|
|
config.excludes = ['DWARF']
|
|
|
|
# Exclude LongTests directories when not executing long tests.
|
|
swift_test_subset = lit_config.params.get('swift_test_subset', 'validation')
|
|
if swift_test_subset in ['primary', 'validation', 'only_validation']:
|
|
config.excludes += ['LongTests']
|
|
elif swift_test_subset == 'all':
|
|
pass
|
|
elif swift_test_subset == 'only_long':
|
|
# FIXME: this doesn't exclude not-long tests from the only_long subset.
|
|
# Currently those tests are very fast so it doesn't matter much.
|
|
pass
|
|
elif swift_test_subset == 'only_stress':
|
|
# FIXME: this doesn't exclude not-stress tests from the only_stress subset.
|
|
# Currently those tests are very fast so it doesn't matter much.
|
|
pass
|
|
else:
|
|
lit_config.fatal("Unknown test mode %r" % swift_test_subset)
|
|
|
|
# test_source_root: The root path where tests are located.
|
|
# test_exec_root: The root path where tests should be run.
|
|
config.test_exec_root = os.path.join(swift_obj_root, 'unittests')
|
|
config.test_source_root = config.test_exec_root
|
|
|
|
# testFormat: The test format to use to interpret tests.
|
|
config.test_format = lit.formats.GoogleTest(config.build_mode, 'Tests')
|