tests: use new runner

.busted

@@ -1,7 +0,0 @@
-return {
-	default = {
-		verbose = true,
-		ROOT = {"spec/front/unit"},
-		lpath = "spec/front/unit/?.lua",
-	},
-}

.ci/after_success.sh

@@ -7,19 +7,11 @@ source "${CI_DIR}/common.sh"
 set +e
 
 if [ -z "${CIRCLE_PULL_REQUEST}" ] && [ "${CIRCLE_BRANCH}" = 'master' ]; then
-    echo "CIRCLE_NODE_INDEX: ${CIRCLE_NODE_INDEX}"
+    travis_retry make --assume-old=all coverage
-    if [ "${CIRCLE_NODE_INDEX}" = 1 ]; then
+    pushd install/koreader && {
-        echo -e "\\n${ANSI_GREEN}Running make testfront for timings."
+        # see https://github.com/codecov/example-lua
-        make --assume-old=all testfront BUSTED_OVERRIDES="--output=junit -Xoutput junit-test-results.xml"
+        bash <(curl -s https://codecov.io/bash)
-    fi
+    } && popd || exit
-
-    if [ "${CIRCLE_NODE_INDEX}" = 0 ]; then
-        travis_retry make --assume-old=all coverage
-        pushd install/koreader && {
-            # see https://github.com/codecov/example-lua
-            bash <(curl -s https://codecov.io/bash)
-        } && popd || exit
-    fi
 else
     echo -e "\\n${ANSI_GREEN}Not on official master branch. Skipping coverage."
 fi

.ci/test.sh

@@ -4,11 +4,6 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=/dev/null
 source "${CI_DIR}/common.sh"
 
-pushd install/koreader && {
+make testfront --assume-old=all T="-o '${PWD}/test-results.xml'"
-    # the circleci command spits out newlines; we want spaces instead
-    BUSTED_OVERRIDES="$(circleci tests glob "spec/front/unit/*_spec.lua" | circleci tests split --split-by=timings --timings-type=filename | tr '\n' ' ')"
-} && popd || exit
-
-make testfront --assume-old=all BUSTED_OVERRIDES="${BUSTED_OVERRIDES}"
 
 # vim: sw=4

.circleci/config.yml

@@ -39,7 +39,6 @@ jobs:
       CLICOLOR_FORCE: "1"
       EMULATE_READER: "1"
       MAKEFLAGS: "PARALLEL_JOBS=3 OUTPUT_DIR=build INSTALL_DIR=install"
-    parallelism: 2
     steps:
       # Checkout / fetch. {{{
       - checkout
@@ -110,7 +109,7 @@ jobs:
           command: .ci/after_success.sh
       # By storing the test results CircleCI automatically distributes tests based on execution time.
       - store_test_results:
-          path: &TESTS_XML install/koreader/junit-test-results.xml
+          path: &TESTS_XML test-results.xml
       # CircleCI doesn't make the test results available as artifacts (October 2017).
       - store_artifacts:
           path: *TESTS_XML
@@ -125,7 +124,6 @@ jobs:
     resource_class: small
     environment:
       BASH_ENV: "~/.bashrc"
-    parallelism: 1
     steps:
       - checkout
       - run:

.github/workflows/build.yml

@@ -65,6 +65,7 @@ jobs:
           packages=(
             autoconf
             automake
+            bash
             binutils
             cmake
             coreutils

.luacov

@@ -6,6 +6,11 @@
 -- global flag to indicate coverage test
 LUACOV = true
 
+local lfs = require "lfs"
+local kohome = os.getenv("KO_HOME")
+local outdir = (kohome or ".") .. "/"
+lfs.mkdir(outdir)
+
 return {
 
   -- default filename to load for config options if not provided
@@ -13,13 +18,13 @@ return {
   ['configfile'] = '.luacov',
 
   -- filename to store stats collected
-  ['statsfile'] = 'luacov.stats.out',
+  ['statsfile'] = outdir .. 'luacov.stats.out',
 
   -- filename to store report
-  ['reportfile'] = 'luacov.report.out',
+  ['reportfile'] = outdir .. 'luacov.report.out',
 
   -- Run reporter on completion? (won't work for ticks)
-  runreport = true,
+  runreport = false,
 
   -- Delete stats file after reporting?
   deletestats = false,
@@ -62,3 +67,5 @@ return {
 
 
 }
+
+-- vim: ft=lua

kodev

@@ -297,8 +297,8 @@ function parse_options() {
         exit ${E_OPTERR}
     fi
     ARGS=("$@")
-    # echo "OPTS: ${OPTS[@]} [${#OPTS[@]}]"
+    # echo "OPTS [${#OPTS[@]}]: $(print_quoted "${OPTS[@]}")"
-    # echo "ARGS: ${ARGS[@]} [${#ARGS[@]}]"
+    # echo "ARGS [${#ARGS[@]}]: $(print_quoted "${ARGS[@]}")"
 }
 
 # }}}
@@ -651,30 +651,43 @@ $(build_options_help_msg 'BUILD' 'use existing build' '' 'default')
 }
 
 function kodev-test() {
+    # shellcheck disable=1091
+    source "${CURDIR}/base/test-runner/runtests"
     HELP="
 USAGE: $0 ${CMD} <OPTIONS> <TEST_SUITE> <TEST_NAMES>
 
-    TEST_SUITE: [all|base|front]. Optional: default to all.
+    TEST_SUITE: [all|base|bench|front]. Optional: default to all.
     TEST_NAMES: if no TEST_NAMES are given, the full testsuite is run.
 
-OPTIONS:
+${RUNTESTS_HELP}
-
-    -t, --tags=TAGS           only run tests with given tags
-
 $(build_options_help_msg 'BUILD' 'use existing build' '' 'default')
 "
-    parse_options "t:${BUILD_GETOPT_SHORT}" "tags:,${BUILD_GETOPT_LONG}" '*' "$@"
+    parse_options "${RUNTESTS_GETOPT_SHORT}${BUILD_GETOPT_SHORT}" "${RUNTESTS_GETOPT_LONG},${BUILD_GETOPT_LONG}" '*' "$@"
-    # Handle first argument.
+    # Forward all options except for most build options.
-    suite=''
+    local targs=(${VERBOSE:+-v})
+    set -- "${OPTS[@]}"
+    while [[ $# -gt 0 ]]; do
+        case "$1" in
+            # Those need special handling so an option-like argument is properly forwarded.
+            --busted | --meson)
+                targs+=("$1=$2")
+                shift
+                ;;
+            # Other.
+            *) targs+=("$1") ;;
+        esac
+        shift
+    done
     set -- "${ARGS[@]}"
-    if [[ $# -ne 0 ]]; then
+    if [[ $# -gt 0 ]]; then
         suite="$1"
         shift
-        ARGS=("$@")
+    else
+        suite='all'
     fi
-    # The rest (custom options included) is forwarded to busted.
+    targs+=("$@")
     setup_target 'emulator'
-    run_make ${NO_BUILD:+--assume-old=all} "test${suite}" BUSTED_OVERRIDES="$(print_quoted "${OPTS[@]}" "${ARGS[@]}")"
+    run_make ${NO_BUILD:+--assume-old=all} "test${suite}" T="$(print_quoted "${targs[@]}")"
 }
 
 # }}}

make/emulator.mk

@@ -29,35 +29,39 @@ run-wbuilder: all
 
 # Testing & coverage. {{{
 
-PHONY += coverage coverage-full coverage-run coverage-summary test testbase testfront
+PHONY += coverage coverage-full coverage-run coverage-summary test test%
 
-$(INSTALL_DIR)/koreader/.busted: .busted
+$(addprefix test,all base bench front): all test-data
-	$(SYMLINK) .busted $@
+	$(RUNTESTS) $(INSTALL_DIR)/koreader $(@:test%=%) $T
+
+test: testall
+
+COVERAGE_STATS = luacov.stats.out
+COVERAGE_REPORT = luacov.report.out
 
 $(INSTALL_DIR)/koreader/.luacov:
 	$(SYMLINK) .luacov $@
 
-testbase: all test-data $(OUTPUT_DIR)/.busted $(OUTPUT_DIR)/spec/base
-	cd $(OUTPUT_DIR) && $(BUSTED_LUAJIT) $(or $(BUSTED_OVERRIDES),./spec/base/unit)
-
-testfront: all test-data $(INSTALL_DIR)/koreader/.busted
-	# sdr files may have unexpected impact on unit testing
-	-rm -rf spec/unit/data/*.sdr
-	cd $(INSTALL_DIR)/koreader && $(BUSTED_LUAJIT) $(BUSTED_OVERRIDES)
-
-test: testbase testfront
-
 coverage: coverage-summary
 
-coverage-run: all test-data $(INSTALL_DIR)/koreader/.busted $(INSTALL_DIR)/koreader/.luacov
+coverage-run: all test-data $(INSTALL_DIR)/koreader/.luacov
-	-rm -rf $(INSTALL_DIR)/koreader/luacov.*.out
+	rm -f $(addprefix $(INSTALL_DIR)/koreader/,$(COVERAGE_STATS) $(COVERAGE_REPORT))
-	cd $(INSTALL_DIR)/koreader && $(BUSTED_LUAJIT) --coverage --exclude-tags=nocov
+	# Run tests.
+	$(RUNTESTS) $(INSTALL_DIR)/koreader front --coverage --tags=!nocov $T
+	# Aggregate statistics.
+	cd $(INSTALL_DIR)/koreader && \
+	    eval "$$($(LUAROCKS_BINARY) path)" && \
+	    test -r $(COVERAGE_STATS) || \
+	    ./luajit tools/merge_luacov_stats.lua $(COVERAGE_STATS) spec/run/*/$(COVERAGE_STATS)
+	# Generate report.
+	cd $(INSTALL_DIR)/koreader && \
+	    eval "$$($(LUAROCKS_BINARY) path)" && \
+	    ./luajit -e 'r = require "luacov.runner"; r.run_report(r.configuration)' /dev/null
 
 coverage-full: coverage-run
 	cd $(INSTALL_DIR)/koreader && cat luacov.report.out
 
 coverage-summary: coverage-run
-	# coverage report summary
 	cd $(INSTALL_DIR)/koreader && tail -n \
 		+$$(($$(grep -nm1 -e "^Summary$$" luacov.report.out|cut -d: -f1)-1)) \
 		luacov.report.out

tools/merge_luacov_stats.lua

@@ -0,0 +1,24 @@
+#!./luajit
+
+local stats = require("luacov.stats")
+local runner = require("luacov.runner")
+
+local outfile = nil
+local aggregated = {}
+
+for i, a in ipairs(arg) do
+    if outfile then
+        -- print('merging luacov stats: '..(a))
+        for name, file_data in pairs(stats.load(a)) do
+            if aggregated[name] then
+                runner.update_stats(aggregated[name], file_data)
+            else
+                aggregated[name] = file_data
+            end
+        end
+    else
+        outfile = a
+    end
+end
+-- print('saving luacov stats '..outfile)
+stats.save(outfile, aggregated)