diff --git a/.busted b/.busted deleted file mode 100644 index 37496a5cd..000000000 --- a/.busted +++ /dev/null @@ -1,7 +0,0 @@ -return { - default = { - verbose = true, - ROOT = {"spec/front/unit"}, - lpath = "spec/front/unit/?.lua", - }, -} diff --git a/.ci/after_success.sh b/.ci/after_success.sh index 304b2311c..d51869a61 100755 --- a/.ci/after_success.sh +++ b/.ci/after_success.sh @@ -7,19 +7,11 @@ source "${CI_DIR}/common.sh" set +e if [ -z "${CIRCLE_PULL_REQUEST}" ] && [ "${CIRCLE_BRANCH}" = 'master' ]; then - echo "CIRCLE_NODE_INDEX: ${CIRCLE_NODE_INDEX}" - if [ "${CIRCLE_NODE_INDEX}" = 1 ]; then - echo -e "\\n${ANSI_GREEN}Running make testfront for timings." - make --assume-old=all testfront BUSTED_OVERRIDES="--output=junit -Xoutput junit-test-results.xml" - fi - - if [ "${CIRCLE_NODE_INDEX}" = 0 ]; then - travis_retry make --assume-old=all coverage - pushd install/koreader && { - # see https://github.com/codecov/example-lua - bash <(curl -s https://codecov.io/bash) - } && popd || exit - fi + travis_retry make --assume-old=all coverage + pushd install/koreader && { + # see https://github.com/codecov/example-lua + bash <(curl -s https://codecov.io/bash) + } && popd || exit else echo -e "\\n${ANSI_GREEN}Not on official master branch. Skipping coverage." fi diff --git a/.ci/test.sh b/.ci/test.sh index 43c44de90..de3559e10 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -4,11 +4,6 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null source "${CI_DIR}/common.sh" -pushd install/koreader && { - # the circleci command spits out newlines; we want spaces instead - BUSTED_OVERRIDES="$(circleci tests glob "spec/front/unit/*_spec.lua" | circleci tests split --split-by=timings --timings-type=filename | tr '\n' ' ')" -} && popd || exit - -make testfront --assume-old=all BUSTED_OVERRIDES="${BUSTED_OVERRIDES}" +make testfront --assume-old=all T="-o '${PWD}/test-results.xml'" # vim: sw=4 diff --git a/.circleci/config.yml b/.circleci/config.yml index 7c2c47f74..7b909b3e2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,7 +39,6 @@ jobs: CLICOLOR_FORCE: "1" EMULATE_READER: "1" MAKEFLAGS: "PARALLEL_JOBS=3 OUTPUT_DIR=build INSTALL_DIR=install" - parallelism: 2 steps: # Checkout / fetch. {{{ - checkout @@ -110,7 +109,7 @@ jobs: command: .ci/after_success.sh # By storing the test results CircleCI automatically distributes tests based on execution time. - store_test_results: - path: &TESTS_XML install/koreader/junit-test-results.xml + path: &TESTS_XML test-results.xml # CircleCI doesn't make the test results available as artifacts (October 2017). - store_artifacts: path: *TESTS_XML @@ -125,7 +124,6 @@ jobs: resource_class: small environment: BASH_ENV: "~/.bashrc" - parallelism: 1 steps: - checkout - run: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 940a25c0c..9ac55ed53 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,6 +65,7 @@ jobs: packages=( autoconf automake + bash binutils cmake coreutils diff --git a/.luacov b/.luacov index 62c3c3254..a4d88f804 100644 --- a/.luacov +++ b/.luacov @@ -6,6 +6,11 @@ -- global flag to indicate coverage test LUACOV = true +local lfs = require "lfs" +local kohome = os.getenv("KO_HOME") +local outdir = (kohome or ".") .. "/" +lfs.mkdir(outdir) + return { -- default filename to load for config options if not provided @@ -13,13 +18,13 @@ return { ['configfile'] = '.luacov', -- filename to store stats collected - ['statsfile'] = 'luacov.stats.out', + ['statsfile'] = outdir .. 'luacov.stats.out', -- filename to store report - ['reportfile'] = 'luacov.report.out', + ['reportfile'] = outdir .. 'luacov.report.out', -- Run reporter on completion? (won't work for ticks) - runreport = true, + runreport = false, -- Delete stats file after reporting? deletestats = false, @@ -62,3 +67,5 @@ return { } + +-- vim: ft=lua diff --git a/base b/base index 509f8090d..6df9a585e 160000 --- a/base +++ b/base @@ -1 +1 @@ -Subproject commit 509f8090d74d699a54483cce47ca96ad98baea2b +Subproject commit 6df9a585e3e240c27cd90bd0a11ee7390d7616d9 diff --git a/kodev b/kodev index 80602c2a3..1c3abb2c8 100755 --- a/kodev +++ b/kodev @@ -297,8 +297,8 @@ function parse_options() { exit ${E_OPTERR} fi ARGS=("$@") - # echo "OPTS: ${OPTS[@]} [${#OPTS[@]}]" - # echo "ARGS: ${ARGS[@]} [${#ARGS[@]}]" + # echo "OPTS [${#OPTS[@]}]: $(print_quoted "${OPTS[@]}")" + # echo "ARGS [${#ARGS[@]}]: $(print_quoted "${ARGS[@]}")" } # }}} @@ -651,30 +651,43 @@ $(build_options_help_msg 'BUILD' 'use existing build' '' 'default') } function kodev-test() { + # shellcheck disable=1091 + source "${CURDIR}/base/test-runner/runtests" HELP=" USAGE: $0 ${CMD} - TEST_SUITE: [all|base|front]. Optional: default to all. + TEST_SUITE: [all|base|bench|front]. Optional: default to all. TEST_NAMES: if no TEST_NAMES are given, the full testsuite is run. -OPTIONS: - - -t, --tags=TAGS only run tests with given tags - +${RUNTESTS_HELP} $(build_options_help_msg 'BUILD' 'use existing build' '' 'default') " - parse_options "t:${BUILD_GETOPT_SHORT}" "tags:,${BUILD_GETOPT_LONG}" '*' "$@" - # Handle first argument. - suite='' + parse_options "${RUNTESTS_GETOPT_SHORT}${BUILD_GETOPT_SHORT}" "${RUNTESTS_GETOPT_LONG},${BUILD_GETOPT_LONG}" '*' "$@" + # Forward all options except for most build options. + local targs=(${VERBOSE:+-v}) + set -- "${OPTS[@]}" + while [[ $# -gt 0 ]]; do + case "$1" in + # Those need special handling so an option-like argument is properly forwarded. + --busted | --meson) + targs+=("$1=$2") + shift + ;; + # Other. + *) targs+=("$1") ;; + esac + shift + done set -- "${ARGS[@]}" - if [[ $# -ne 0 ]]; then + if [[ $# -gt 0 ]]; then suite="$1" shift - ARGS=("$@") + else + suite='all' fi - # The rest (custom options included) is forwarded to busted. + targs+=("$@") setup_target 'emulator' - run_make ${NO_BUILD:+--assume-old=all} "test${suite}" BUSTED_OVERRIDES="$(print_quoted "${OPTS[@]}" "${ARGS[@]}")" + run_make ${NO_BUILD:+--assume-old=all} "test${suite}" T="$(print_quoted "${targs[@]}")" } # }}} diff --git a/make/emulator.mk b/make/emulator.mk index 921b99cf4..cbfa61157 100644 --- a/make/emulator.mk +++ b/make/emulator.mk @@ -29,35 +29,39 @@ run-wbuilder: all # Testing & coverage. {{{ -PHONY += coverage coverage-full coverage-run coverage-summary test testbase testfront +PHONY += coverage coverage-full coverage-run coverage-summary test test% -$(INSTALL_DIR)/koreader/.busted: .busted - $(SYMLINK) .busted $@ +$(addprefix test,all base bench front): all test-data + $(RUNTESTS) $(INSTALL_DIR)/koreader $(@:test%=%) $T + +test: testall + +COVERAGE_STATS = luacov.stats.out +COVERAGE_REPORT = luacov.report.out $(INSTALL_DIR)/koreader/.luacov: $(SYMLINK) .luacov $@ -testbase: all test-data $(OUTPUT_DIR)/.busted $(OUTPUT_DIR)/spec/base - cd $(OUTPUT_DIR) && $(BUSTED_LUAJIT) $(or $(BUSTED_OVERRIDES),./spec/base/unit) - -testfront: all test-data $(INSTALL_DIR)/koreader/.busted - # sdr files may have unexpected impact on unit testing - -rm -rf spec/unit/data/*.sdr - cd $(INSTALL_DIR)/koreader && $(BUSTED_LUAJIT) $(BUSTED_OVERRIDES) - -test: testbase testfront - coverage: coverage-summary -coverage-run: all test-data $(INSTALL_DIR)/koreader/.busted $(INSTALL_DIR)/koreader/.luacov - -rm -rf $(INSTALL_DIR)/koreader/luacov.*.out - cd $(INSTALL_DIR)/koreader && $(BUSTED_LUAJIT) --coverage --exclude-tags=nocov +coverage-run: all test-data $(INSTALL_DIR)/koreader/.luacov + rm -f $(addprefix $(INSTALL_DIR)/koreader/,$(COVERAGE_STATS) $(COVERAGE_REPORT)) + # Run tests. + $(RUNTESTS) $(INSTALL_DIR)/koreader front --coverage --tags=!nocov $T + # Aggregate statistics. + cd $(INSTALL_DIR)/koreader && \ + eval "$$($(LUAROCKS_BINARY) path)" && \ + test -r $(COVERAGE_STATS) || \ + ./luajit tools/merge_luacov_stats.lua $(COVERAGE_STATS) spec/run/*/$(COVERAGE_STATS) + # Generate report. + cd $(INSTALL_DIR)/koreader && \ + eval "$$($(LUAROCKS_BINARY) path)" && \ + ./luajit -e 'r = require "luacov.runner"; r.run_report(r.configuration)' /dev/null coverage-full: coverage-run cd $(INSTALL_DIR)/koreader && cat luacov.report.out coverage-summary: coverage-run - # coverage report summary cd $(INSTALL_DIR)/koreader && tail -n \ +$$(($$(grep -nm1 -e "^Summary$$" luacov.report.out|cut -d: -f1)-1)) \ luacov.report.out diff --git a/tools/merge_luacov_stats.lua b/tools/merge_luacov_stats.lua new file mode 100755 index 000000000..5a0ceb1b8 --- /dev/null +++ b/tools/merge_luacov_stats.lua @@ -0,0 +1,24 @@ +#!./luajit + +local stats = require("luacov.stats") +local runner = require("luacov.runner") + +local outfile = nil +local aggregated = {} + +for i, a in ipairs(arg) do + if outfile then + -- print('merging luacov stats: '..(a)) + for name, file_data in pairs(stats.load(a)) do + if aggregated[name] then + runner.update_stats(aggregated[name], file_data) + else + aggregated[name] = file_data + end + end + else + outfile = a + end +end +-- print('saving luacov stats '..outfile) +stats.save(outfile, aggregated)