From bdffb35d17aa77ddcc5de5784a816e6af2d42aaa Mon Sep 17 00:00:00 2001 From: Pavol Vaskovic Date: Tue, 2 May 2017 16:20:30 +0200 Subject: [PATCH] SR-4572 Remove jinja2 dependency from benchmark test harness generation --- benchmark/CMakeLists.txt | 3 +- ...eLists.txt_template => CMakeLists.txt.gyb} | 105 +++++----- .../generate_harness/generate_harness.py | 189 +----------------- .../generate_harness/main.swift_template | 63 ------ benchmark/utils/main.swift | 27 ++- benchmark/utils/main.swift.gyb | 98 +++++++++ utils/gyb_benchmark_support.py | 58 ++++++ 7 files changed, 227 insertions(+), 316 deletions(-) rename benchmark/{scripts/generate_harness/CMakeLists.txt_template => CMakeLists.txt.gyb} (58%) delete mode 100644 benchmark/scripts/generate_harness/main.swift_template create mode 100644 benchmark/utils/main.swift.gyb create mode 100644 utils/gyb_benchmark_support.py diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 4054fac0808..8300cc034b0 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -1,8 +1,7 @@ # -*- mode: cmake -*- ################################################################################ # WARNING: This file is automatically generated from templates and should not -# be directly modified. Instead, make changes to -# scripts/generate_harness/CMakeLists.txt_template and run +# be directly modified. Instead, make changes to CMakeLists.text.gyb and run # scripts/generate_harness/generate_harness.py to regenerate this file. ################################################################################ diff --git a/benchmark/scripts/generate_harness/CMakeLists.txt_template b/benchmark/CMakeLists.txt.gyb similarity index 58% rename from benchmark/scripts/generate_harness/CMakeLists.txt_template rename to benchmark/CMakeLists.txt.gyb index d4f0abb42ab..0e55a53f0c9 100644 --- a/benchmark/scripts/generate_harness/CMakeLists.txt_template +++ b/benchmark/CMakeLists.txt.gyb @@ -1,39 +1,45 @@ # -*- mode: cmake -*- +% # Ignore the following warning. This _is_ the correct file to edit. ################################################################################ # WARNING: This file is automatically generated from templates and should not -# be directly modified. Instead, make changes to -# scripts/generate_harness/CMakeLists.txt_template and run +# be directly modified. Instead, make changes to CMakeLists.text.gyb and run # scripts/generate_harness/generate_harness.py to regenerate this file. ################################################################################ +%{ +from gyb_benchmark_support import ( + tests, + multisource_benches +) +}% cmake_minimum_required(VERSION 2.8.12) # Add path for custom CMake modules. list(APPEND CMAKE_MODULE_PATH - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules") + "$${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules") include(AddSwiftBenchmarkSuite) set(SWIFT_BENCH_MODULES -{% for test in tests %} - single-source/{{ test }} -{% endfor %} +% for test in tests: + single-source/${test} +% end ) set(SWIFT_MULTISOURCE_BENCHES -{% for multisource_bench in multisource_benches %} - multi-source/{{ multisource_bench.name }} -{% endfor %} +% for multisource_bench in multisource_benches: + multi-source/${multisource_bench.name} +% end ) -{% for multisource_bench in multisource_benches %} -set({{ multisource_bench.name }}_sources - {% for file in multisource_bench.files %} - multi-source/{{ multisource_bench.name }}/{{ file }} - {% endfor %} +% for multisource_bench in multisource_benches: +set(${multisource_bench.name}_sources +% for file in multisource_bench.files: + multi-source/${multisource_bench.name}/${file} +% end ) -{% endfor %} +% end set(BENCH_DRIVER_LIBRARY_MODULES utils/DriverUtils @@ -62,19 +68,19 @@ if(NOT SWIFT_EXEC) endif() if(NOT SWIFT_LIBRARY_PATH) - get_filename_component(tmp_dir "${SWIFT_EXEC}" DIRECTORY) - get_filename_component(tmp_dir "${tmp_dir}" DIRECTORY) - set(SWIFT_LIBRARY_PATH "${tmp_dir}/lib/swift") + get_filename_component(tmp_dir "$${SWIFT_EXEC}" DIRECTORY) + get_filename_component(tmp_dir "$${tmp_dir}" DIRECTORY) + set(SWIFT_LIBRARY_PATH "$${tmp_dir}/lib/swift") endif() # If the CMAKE_C_COMPILER is already clang, don't find it again, # thus allowing the --host-cc build-script argument to work here. -get_filename_component(c_compiler ${CMAKE_C_COMPILER} NAME) +get_filename_component(c_compiler $${CMAKE_C_COMPILER} NAME) -if(${c_compiler} STREQUAL "clang") - set(CLANG_EXEC ${CMAKE_C_COMPILER}) +if($${c_compiler} STREQUAL "clang") + set(CLANG_EXEC $${CMAKE_C_COMPILER}) else() - runcmd(COMMAND "xcrun" "-toolchain" "${SWIFT_DARWIN_XCRUN_TOOLCHAIN}" "-f" "clang" + runcmd(COMMAND "xcrun" "-toolchain" "$${SWIFT_DARWIN_XCRUN_TOOLCHAIN}" "-f" "clang" VARIABLE CLANG_EXEC ERROR "Unable to find Clang driver") endif() @@ -90,7 +96,7 @@ Available configurations: _SINGLEFILE, _MULTITHREADED") # where "_" is optional. if(NOT SWIFT_OPTIMIZATION_LEVELS) set(SWIFT_OPTIMIZATION_LEVELS "Onone" "O" "Ounchecked" - ${SWIFT_EXTRA_BENCH_CONFIGS}) + $${SWIFT_EXTRA_BENCH_CONFIGS}) endif() set(SWIFT_BENCHMARK_NUM_O_ITERATIONS "" CACHE STRING @@ -123,30 +129,30 @@ set(watchos_triple_platform "watchos") set(sdks) set(platforms) -foreach(platform ${ONLY_PLATFORMS}) +foreach(platform $${ONLY_PLATFORMS}) execute_process( - COMMAND "xcrun" "--sdk" "${platform}" "--show-sdk-path" - OUTPUT_VARIABLE ${platform}_sdk + COMMAND "xcrun" "--sdk" "$${platform}" "--show-sdk-path" + OUTPUT_VARIABLE $${platform}_sdk RESULT_VARIABLE result ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) - if("${result}" MATCHES "0") - list(APPEND sdks "${${platform}_sdk}") - list(APPEND platforms ${platform}) + if("$${result}" MATCHES "0") + list(APPEND sdks "$${$${platform}_sdk}") + list(APPEND platforms $${platform}) endif() endforeach() message("--") message("-- Swift Benchmark Suite:") -message("-- SWIFT_EXEC = ${SWIFT_EXEC}") -message("-- SWIFT_LIBRARY_PATH = ${SWIFT_LIBRARY_PATH}") -message("-- CLANG_EXEC = ${CLANG_EXEC}") -message("-- SWIFT_OPTIMIZATION_LEVELS = ${SWIFT_OPTIMIZATION_LEVELS}") -message("-- ONLY_PLATFORMS = ${ONLY_PLATFORMS}") +message("-- SWIFT_EXEC = $${SWIFT_EXEC}") +message("-- SWIFT_LIBRARY_PATH = $${SWIFT_LIBRARY_PATH}") +message("-- CLANG_EXEC = $${CLANG_EXEC}") +message("-- SWIFT_OPTIMIZATION_LEVELS = $${SWIFT_OPTIMIZATION_LEVELS}") +message("-- ONLY_PLATFORMS = $${ONLY_PLATFORMS}") -message("-- found platforms: ${platforms}") +message("-- found platforms: $${platforms}") message("-- found sdks:") -foreach(sdk ${sdks}) - message("-- ${sdk}") +foreach(sdk $${sdks}) + message("-- $${sdk}") endforeach() set(executable_targets) @@ -155,27 +161,26 @@ if(SWIFT_SDKS) set(IS_SWIFT_BUILD true) endif() -set(srcdir "${CMAKE_CURRENT_SOURCE_DIR}") +set(srcdir "$${CMAKE_CURRENT_SOURCE_DIR}") if(IS_SWIFT_BUILD) - get_filename_component(swift-bin-dir "${SWIFT_EXEC}" DIRECTORY) + get_filename_component(swift-bin-dir "$${SWIFT_EXEC}" DIRECTORY) else() - set(swift-bin-dir "${CMAKE_BINARY_DIR}/bin") + set(swift-bin-dir "$${CMAKE_BINARY_DIR}/bin") endif() -set(benchmark-bin-dir "${CMAKE_CURRENT_BINARY_DIR}/bin") -set(benchmark-lib-dir "${CMAKE_CURRENT_BINARY_DIR}/lib") -set(benchmark-lib-swift-dir "${CMAKE_CURRENT_BINARY_DIR}/lib/swift") +set(benchmark-bin-dir "$${CMAKE_CURRENT_BINARY_DIR}/bin") +set(benchmark-lib-dir "$${CMAKE_CURRENT_BINARY_DIR}/lib") +set(benchmark-lib-swift-dir "$${CMAKE_CURRENT_BINARY_DIR}/lib/swift") -file(MAKE_DIRECTORY "${swift-bin-dir}") -file(MAKE_DIRECTORY "${benchmark-bin-dir}") -file(MAKE_DIRECTORY "${benchmark-lib-dir}") -file(MAKE_DIRECTORY "${benchmark-lib-swift-dir}") +file(MAKE_DIRECTORY "$${swift-bin-dir}") +file(MAKE_DIRECTORY "$${benchmark-bin-dir}") +file(MAKE_DIRECTORY "$${benchmark-lib-dir}") +file(MAKE_DIRECTORY "$${benchmark-lib-swift-dir}") # Compile the perf test suite for each platform -foreach(platform ${platforms}) - swift_benchmark_compile(PLATFORM ${platform}) +foreach(platform $${platforms}) + swift_benchmark_compile(PLATFORM $${platform}) endforeach() add_subdirectory(scripts) - diff --git a/benchmark/scripts/generate_harness/generate_harness.py b/benchmark/scripts/generate_harness/generate_harness.py index 3809325f1fc..2e6120c9ee0 100755 --- a/benchmark/scripts/generate_harness/generate_harness.py +++ b/benchmark/scripts/generate_harness/generate_harness.py @@ -18,16 +18,11 @@ from __future__ import print_function import argparse import os -import re import subprocess -import jinja2 - script_dir = os.path.dirname(os.path.realpath(__file__)) perf_dir = os.path.realpath(os.path.join(script_dir, '../..')) gyb = os.path.realpath(os.path.join(perf_dir, '../utils/gyb')) -single_source_dir = os.path.join(perf_dir, 'single-source') -multi_source_dir = os.path.join(perf_dir, 'multi-source') parser = argparse.ArgumentParser() parser.add_argument("--output-dir", help="Output directory (for validation test)", @@ -35,138 +30,6 @@ parser.add_argument("--output-dir", args = parser.parse_args() output_dir = args.output_dir -template_map = { - 'CMakeLists.txt_template': os.path.join(output_dir, 'CMakeLists.txt'), - 'main.swift_template': os.path.join(output_dir, 'utils/main.swift') -} - -# The test suites. Currently, "other" and "string" -other_tests = [ - "Ackermann", - "Fibonacci", - "ExistentialTestArrayConditionalShift_ClassValueBuffer1", - "ExistentialTestArrayConditionalShift_ClassValueBuffer2", - "ExistentialTestArrayConditionalShift_ClassValueBuffer3", - "ExistentialTestArrayConditionalShift_ClassValueBuffer4", - "ExistentialTestArrayConditionalShift_IntValueBuffer0", - "ExistentialTestArrayConditionalShift_IntValueBuffer1", - "ExistentialTestArrayConditionalShift_IntValueBuffer2", - "ExistentialTestArrayConditionalShift_IntValueBuffer3", - "ExistentialTestArrayConditionalShift_IntValueBuffer4", - "ExistentialTestArrayMutating_ClassValueBuffer1", - "ExistentialTestArrayMutating_ClassValueBuffer2", - "ExistentialTestArrayMutating_ClassValueBuffer3", - "ExistentialTestArrayMutating_ClassValueBuffer4", - "ExistentialTestArrayMutating_IntValueBuffer0", - "ExistentialTestArrayMutating_IntValueBuffer1", - "ExistentialTestArrayMutating_IntValueBuffer2", - "ExistentialTestArrayMutating_IntValueBuffer3", - "ExistentialTestArrayMutating_IntValueBuffer4", - "ExistentialTestArrayOneMethodCall_ClassValueBuffer1", - "ExistentialTestArrayOneMethodCall_ClassValueBuffer2", - "ExistentialTestArrayOneMethodCall_ClassValueBuffer3", - "ExistentialTestArrayOneMethodCall_ClassValueBuffer4", - "ExistentialTestArrayOneMethodCall_IntValueBuffer0", - "ExistentialTestArrayOneMethodCall_IntValueBuffer1", - "ExistentialTestArrayOneMethodCall_IntValueBuffer2", - "ExistentialTestArrayOneMethodCall_IntValueBuffer3", - "ExistentialTestArrayOneMethodCall_IntValueBuffer4", - "ExistentialTestArrayShift_ClassValueBuffer1", - "ExistentialTestArrayShift_ClassValueBuffer2", - "ExistentialTestArrayShift_ClassValueBuffer3", - "ExistentialTestArrayShift_ClassValueBuffer4", - "ExistentialTestArrayShift_IntValueBuffer0", - "ExistentialTestArrayShift_IntValueBuffer1", - "ExistentialTestArrayShift_IntValueBuffer2", - "ExistentialTestArrayShift_IntValueBuffer3", - "ExistentialTestArrayShift_IntValueBuffer4", - "ExistentialTestArrayTwoMethodCalls_ClassValueBuffer1", - "ExistentialTestArrayTwoMethodCalls_ClassValueBuffer2", - "ExistentialTestArrayTwoMethodCalls_ClassValueBuffer3", - "ExistentialTestArrayTwoMethodCalls_ClassValueBuffer4", - "ExistentialTestArrayTwoMethodCalls_IntValueBuffer0", - "ExistentialTestArrayTwoMethodCalls_IntValueBuffer1", - "ExistentialTestArrayTwoMethodCalls_IntValueBuffer2", - "ExistentialTestArrayTwoMethodCalls_IntValueBuffer3", - "ExistentialTestArrayTwoMethodCalls_IntValueBuffer4", - "ExistentialTestMutatingAndNonMutating_ClassValueBuffer1", - "ExistentialTestMutatingAndNonMutating_ClassValueBuffer2", - "ExistentialTestMutatingAndNonMutating_ClassValueBuffer3", - "ExistentialTestMutatingAndNonMutating_ClassValueBuffer4", - "ExistentialTestMutatingAndNonMutating_IntValueBuffer0", - "ExistentialTestMutatingAndNonMutating_IntValueBuffer1", - "ExistentialTestMutatingAndNonMutating_IntValueBuffer2", - "ExistentialTestMutatingAndNonMutating_IntValueBuffer3", - "ExistentialTestMutatingAndNonMutating_IntValueBuffer4", - "ExistentialTestMutating_ClassValueBuffer1", - "ExistentialTestMutating_ClassValueBuffer2", - "ExistentialTestMutating_ClassValueBuffer3", - "ExistentialTestMutating_ClassValueBuffer4", - "ExistentialTestMutating_IntValueBuffer0", - "ExistentialTestMutating_IntValueBuffer1", - "ExistentialTestMutating_IntValueBuffer2", - "ExistentialTestMutating_IntValueBuffer3", - "ExistentialTestMutating_IntValueBuffer4", - "ExistentialTestOneMethodCall_ClassValueBuffer1", - "ExistentialTestOneMethodCall_ClassValueBuffer2", - "ExistentialTestOneMethodCall_ClassValueBuffer3", - "ExistentialTestOneMethodCall_ClassValueBuffer4", - "ExistentialTestOneMethodCall_IntValueBuffer0", - "ExistentialTestOneMethodCall_IntValueBuffer1", - "ExistentialTestOneMethodCall_IntValueBuffer2", - "ExistentialTestOneMethodCall_IntValueBuffer3", - "ExistentialTestOneMethodCall_IntValueBuffer4", - "ExistentialTestPassExistentialOneMethodCall_ClassValueBuffer1", - "ExistentialTestPassExistentialOneMethodCall_ClassValueBuffer2", - "ExistentialTestPassExistentialOneMethodCall_ClassValueBuffer3", - "ExistentialTestPassExistentialOneMethodCall_ClassValueBuffer4", - "ExistentialTestPassExistentialOneMethodCall_IntValueBuffer0", - "ExistentialTestPassExistentialOneMethodCall_IntValueBuffer1", - "ExistentialTestPassExistentialOneMethodCall_IntValueBuffer2", - "ExistentialTestPassExistentialOneMethodCall_IntValueBuffer3", - "ExistentialTestPassExistentialOneMethodCall_IntValueBuffer4", - "ExistentialTestPassExistentialTwoMethodCalls_ClassValueBuffer1", - "ExistentialTestPassExistentialTwoMethodCalls_ClassValueBuffer2", - "ExistentialTestPassExistentialTwoMethodCalls_ClassValueBuffer3", - "ExistentialTestPassExistentialTwoMethodCalls_ClassValueBuffer4", - "ExistentialTestPassExistentialTwoMethodCalls_IntValueBuffer0", - "ExistentialTestPassExistentialTwoMethodCalls_IntValueBuffer1", - "ExistentialTestPassExistentialTwoMethodCalls_IntValueBuffer2", - "ExistentialTestPassExistentialTwoMethodCalls_IntValueBuffer3", - "ExistentialTestPassExistentialTwoMethodCalls_IntValueBuffer4", - "ExistentialTestTwoMethodCalls_ClassValueBuffer1", - "ExistentialTestTwoMethodCalls_ClassValueBuffer2", - "ExistentialTestTwoMethodCalls_ClassValueBuffer3", - "ExistentialTestTwoMethodCalls_ClassValueBuffer4", - "ExistentialTestTwoMethodCalls_IntValueBuffer0", - "ExistentialTestTwoMethodCalls_IntValueBuffer1", - "ExistentialTestTwoMethodCalls_IntValueBuffer2", - "ExistentialTestTwoMethodCalls_IntValueBuffer3", - "ExistentialTestTwoMethodCalls_IntValueBuffer4", -] - -string_tests = [ - "StringWalkASCIIScalars", - "StringWalkASCIICharacters", - "StringWalkUnicodeScalars", - "StringWalkUnicodeCharacters", - "StringWalkMixedScalars", - "StringWalkMixedCharacters", - "StringWalkASCIIScalarsBackwards", - "StringWalkASCIICharactersBackwards", - "StringWalkUnicodeScalarsBackwards", - "StringWalkUnicodeCharactersBackwards", - "StringWalkMixedScalarsBackwards", - "StringWalkMixedCharactersBackwards", -] - -ignored_run_funcs = other_tests + string_tests - -template_loader = jinja2.FileSystemLoader(searchpath="/") -template_env = jinja2.Environment(loader=template_loader, trim_blocks=True, - lstrip_blocks=True) - - def all_files(directory, extension): # matching: [directory]/**/*[extension] return [ os.path.join(root, f) @@ -185,58 +48,10 @@ def will_write(filename): # ensure path to file exists before writing if __name__ == '__main__': # Generate Your Boilerplate gyb_files = all_files(perf_dir, '.gyb') + # FIXME make sure longer paths are done first, + # as CMakeLists and main.swift depend on the other gybs being generated first! for f in gyb_files: relative_path = os.path.relpath(f[:-4], perf_dir) out_file = os.path.join(output_dir, relative_path) will_write(out_file) subprocess.call([gyb, '--line-directive', '', '-o', out_file, f]) - - # CMakeList single-source - test_files = all_files(single_source_dir, '.swift') - tests = sorted(os.path.basename(x).split('.')[0] for x in test_files) - - # CMakeList multi-source - class MultiSourceBench(object): - def __init__(self, path): - self.name = os.path.basename(path) - self.files = [x for x in os.listdir(path) - if x.endswith('.swift')] - - if os.path.isdir(multi_source_dir): - multisource_benches = [ - MultiSourceBench(os.path.join(multi_source_dir, x)) - for x in os.listdir(multi_source_dir) - if os.path.isdir(os.path.join(multi_source_dir, x)) - ] - else: - multisource_benches = [] - - # main.swift imports - imports = sorted(tests + [msb.name for msb in multisource_benches]) - - # main.swift run functions - def get_run_funcs(filepath): - content = open(filepath).read() - matches = re.findall(r'func run_(.*?)\(', content) - return filter(lambda x: x not in ignored_run_funcs, matches) - - def find_run_funcs(): - swift_files = all_files(perf_dir, '.swift') - return [func for f in swift_files for func in get_run_funcs(f)] - - run_funcs = [(f, f) for f in sorted(find_run_funcs())] - - # Replace originals with files generated from templates - for template_file in template_map: - template_path = os.path.join(script_dir, template_file) - template = template_env.get_template(template_path) - out_file = template_map[template_file] - will_write(out_file) - open(out_file, 'w').write( - template.render(tests=tests, - multisource_benches=multisource_benches, - imports=imports, - run_funcs=run_funcs, - string_tests=string_tests, - other_tests=other_tests) - ) diff --git a/benchmark/scripts/generate_harness/main.swift_template b/benchmark/scripts/generate_harness/main.swift_template deleted file mode 100644 index 889e6306dc8..00000000000 --- a/benchmark/scripts/generate_harness/main.swift_template +++ /dev/null @@ -1,63 +0,0 @@ -//===--- main.swift -------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -//////////////////////////////////////////////////////////////////////////////// -// WARNING: This file is automatically generated from templates and should not -// be directly modified. Instead, make changes to -// scripts/generate_harness/main.swift_template and run -// scripts/generate_harness/generate_harness.py to regenerate this file. -//////////////////////////////////////////////////////////////////////////////// - -// This is just a driver for performance overview tests. -import TestsUtils -import DriverUtils -{% for import in imports %} -import {{ import }} -{% endfor %} - -@inline(__always) -private func addPrecommitTest( - _ name: String, _ function: @escaping (Int) -> () -) { - precommitTests[name] = function -} -@inline(__always) -private func addStringTest( - _ name: String, _ function: @escaping (Int) -> () -) { - stringTests[name] = function -} -@inline(__always) -private func addOtherTest( - _ name: String, _ function: @escaping (Int) -> () -) { - otherTests[name] = function -} - -// The main test suite: precommit tests -{% for run_func in run_funcs %} -addPrecommitTest("{{ run_func[0] }}", run_{{ run_func[1] }}) -{% endfor %} - -// Other tests -{% for test_name in other_tests %} -addOtherTest("{{ test_name }}", run_{{ test_name }}) -{% endfor %} - -// String tests, an extended benchmark suite exercising finer-granularity -// behavior of our Strings. -{% for test_name in string_tests %} -addStringTest("{{ test_name }}", run_{{ test_name }}) -{% endfor %} - -main() - diff --git a/benchmark/utils/main.swift b/benchmark/utils/main.swift index 9859c8f7ef7..c01d504dfba 100644 --- a/benchmark/utils/main.swift +++ b/benchmark/utils/main.swift @@ -1,4 +1,4 @@ -//===--- main.swift -------------------------------------------------------===// +//===--- main.swift -------------------------------------------*- swift -*-===// // // This source file is part of the Swift.org open source project // @@ -11,9 +11,8 @@ //===----------------------------------------------------------------------===// //////////////////////////////////////////////////////////////////////////////// -// WARNING: This file is automatically generated from templates and should not -// be directly modified. Instead, make changes to -// scripts/generate_harness/main.swift_template and run +// WARNING: This file is manually generated from .gyb template and should not +// be directly modified. Instead, make changes to main.swift.gyb and run // scripts/generate_harness/generate_harness.py to regenerate this file. //////////////////////////////////////////////////////////////////////////////// @@ -412,7 +411,6 @@ addPrecommitTest("XorLoop", run_XorLoop) // Other tests addOtherTest("Ackermann", run_Ackermann) -addOtherTest("Fibonacci", run_Fibonacci) addOtherTest("ExistentialTestArrayConditionalShift_ClassValueBuffer1", run_ExistentialTestArrayConditionalShift_ClassValueBuffer1) addOtherTest("ExistentialTestArrayConditionalShift_ClassValueBuffer2", run_ExistentialTestArrayConditionalShift_ClassValueBuffer2) addOtherTest("ExistentialTestArrayConditionalShift_ClassValueBuffer3", run_ExistentialTestArrayConditionalShift_ClassValueBuffer3) @@ -512,20 +510,21 @@ addOtherTest("ExistentialTestTwoMethodCalls_IntValueBuffer1", run_ExistentialTes addOtherTest("ExistentialTestTwoMethodCalls_IntValueBuffer2", run_ExistentialTestTwoMethodCalls_IntValueBuffer2) addOtherTest("ExistentialTestTwoMethodCalls_IntValueBuffer3", run_ExistentialTestTwoMethodCalls_IntValueBuffer3) addOtherTest("ExistentialTestTwoMethodCalls_IntValueBuffer4", run_ExistentialTestTwoMethodCalls_IntValueBuffer4) +addOtherTest("Fibonacci", run_Fibonacci) // String tests, an extended benchmark suite exercising finer-granularity // behavior of our Strings. -addStringTest("StringWalkASCIIScalars", run_StringWalkASCIIScalars) addStringTest("StringWalkASCIICharacters", run_StringWalkASCIICharacters) -addStringTest("StringWalkUnicodeScalars", run_StringWalkUnicodeScalars) -addStringTest("StringWalkUnicodeCharacters", run_StringWalkUnicodeCharacters) -addStringTest("StringWalkMixedScalars", run_StringWalkMixedScalars) -addStringTest("StringWalkMixedCharacters", run_StringWalkMixedCharacters) -addStringTest("StringWalkASCIIScalarsBackwards", run_StringWalkASCIIScalarsBackwards) addStringTest("StringWalkASCIICharactersBackwards", run_StringWalkASCIICharactersBackwards) -addStringTest("StringWalkUnicodeScalarsBackwards", run_StringWalkUnicodeScalarsBackwards) -addStringTest("StringWalkUnicodeCharactersBackwards", run_StringWalkUnicodeCharactersBackwards) -addStringTest("StringWalkMixedScalarsBackwards", run_StringWalkMixedScalarsBackwards) +addStringTest("StringWalkASCIIScalars", run_StringWalkASCIIScalars) +addStringTest("StringWalkASCIIScalarsBackwards", run_StringWalkASCIIScalarsBackwards) +addStringTest("StringWalkMixedCharacters", run_StringWalkMixedCharacters) addStringTest("StringWalkMixedCharactersBackwards", run_StringWalkMixedCharactersBackwards) +addStringTest("StringWalkMixedScalars", run_StringWalkMixedScalars) +addStringTest("StringWalkMixedScalarsBackwards", run_StringWalkMixedScalarsBackwards) +addStringTest("StringWalkUnicodeCharacters", run_StringWalkUnicodeCharacters) +addStringTest("StringWalkUnicodeCharactersBackwards", run_StringWalkUnicodeCharactersBackwards) +addStringTest("StringWalkUnicodeScalars", run_StringWalkUnicodeScalars) +addStringTest("StringWalkUnicodeScalarsBackwards", run_StringWalkUnicodeScalarsBackwards) main() diff --git a/benchmark/utils/main.swift.gyb b/benchmark/utils/main.swift.gyb new file mode 100644 index 00000000000..a71c5be30c3 --- /dev/null +++ b/benchmark/utils/main.swift.gyb @@ -0,0 +1,98 @@ +//===--- main.swift -------------------------------------------*- swift -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +% # Ignore the following warning. This _is_ the correct file to edit. +//////////////////////////////////////////////////////////////////////////////// +// WARNING: This file is manually generated from .gyb template and should not +// be directly modified. Instead, make changes to main.swift.gyb and run +// scripts/generate_harness/generate_harness.py to regenerate this file. +//////////////////////////////////////////////////////////////////////////////// +%{ +from gyb_benchmark_support import ( + tests, + multisource_benches, + all_run_funcs +) + +import re + +# main.swift imports +imports = sorted(tests + [msb.name for msb in multisource_benches]) + +# main.swift run functions + +# The test suites. Currently, "other" and "string" +other_re = [ + "Ackermann", + "Fibonacci", + "ExistentialTest.+" +] + +string_re = [ + "StringWalk.+", +] + +def matches(included_regexes, run_func): + for regexp in included_regexes: + if re.match(regexp, run_func): + return True + return False + +other_tests = filter(lambda x: matches(other_re, x), all_run_funcs) +string_tests = filter(lambda x: matches(string_re, x), all_run_funcs) +ignored_run_funcs = other_tests + string_tests +run_funcs = filter(lambda x: x not in ignored_run_funcs, all_run_funcs) +}% + +// This is just a driver for performance overview tests. +import TestsUtils +import DriverUtils +% for IMPORT in imports: +import ${IMPORT} +% end + +@inline(__always) +private func addPrecommitTest( + _ name: String, _ function: @escaping (Int) -> () +) { + precommitTests[name] = function +} +@inline(__always) +private func addStringTest( + _ name: String, _ function: @escaping (Int) -> () +) { + stringTests[name] = function +} +@inline(__always) +private func addOtherTest( + _ name: String, _ function: @escaping (Int) -> () +) { + otherTests[name] = function +} + +// The main test suite: precommit tests +% for run_func in run_funcs: +addPrecommitTest("${run_func}", run_${run_func}) +% end + +// Other tests +% for test_name in other_tests: +addOtherTest("${test_name }", run_${test_name}) +% end + +// String tests, an extended benchmark suite exercising finer-granularity +// behavior of our Strings. +% for test_name in string_tests: +addStringTest("${test_name}", run_${test_name}) +% end + +main() diff --git a/utils/gyb_benchmark_support.py b/utils/gyb_benchmark_support.py new file mode 100644 index 00000000000..88180efed04 --- /dev/null +++ b/utils/gyb_benchmark_support.py @@ -0,0 +1,58 @@ +# ===--- gyb_benchmark_support.py ---------------------*- coding: utf-8 -*-===// +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See https://swift.org/LICENSE.txt for license information +# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + +import os +import re + + +script_dir = os.path.dirname(os.path.realpath(__file__)) +perf_dir = os.path.realpath(os.path.join(script_dir, '../benchmark')) +single_source_dir = os.path.join(perf_dir, 'single-source') +multi_source_dir = os.path.join(perf_dir, 'multi-source') + +def all_files(directory, extension): # matching: [directory]/**/*[extension] + return [ + os.path.join(root, f) + for root, _, files in os.walk(directory) + for f in files if f.endswith(extension) + ] + + +# CMakeList single-source +test_files = all_files(single_source_dir, '.swift') +tests = sorted(os.path.basename(x).split('.')[0] for x in test_files) + +# CMakeList multi-source +class MultiSourceBench(object): + def __init__(self, path): + self.name = os.path.basename(path) + self.files = [x for x in os.listdir(path) + if x.endswith('.swift')] + +if os.path.isdir(multi_source_dir): + multisource_benches = [ + MultiSourceBench(os.path.join(multi_source_dir, x)) + for x in os.listdir(multi_source_dir) + if os.path.isdir(os.path.join(multi_source_dir, x)) + ] +else: + multisource_benches = [] + + +def get_run_funcs(filepath): + content = open(filepath).read() + return re.findall(r'func run_(.*?)\(', content) + + +def find_run_funcs(): + swift_files = all_files(perf_dir, '.swift') + return sorted([func for f in swift_files for func in get_run_funcs(f)]) + +all_run_funcs = find_run_funcs()