Travis: speed up by caching base and running luacheck earlier (#2741)

* Travis: speed up by caching base and running luacheck earlier
* ignore bin and install for git status change detection
* skip coverage except on official master branch. It adds 3 whole minutes and does nothing to prevent regressions
* also cache ~/.luarocks. It evens out but would generally prevent remote timeout shenenigans
* remove base cache dir before caching with verbose remove to see what's going on
* more inclusive shell code quality analysis
* fixed more shellcheck issues
* better shellcheck/shfmt debugging info

.ci/after_success.sh

@@ -33,11 +33,12 @@ if [ "${TRAVIS_PULL_REQUEST}" = false ] && [ "${TRAVIS_BRANCH}" = 'master' ]; th
         commit -a --amend -m 'Automated documentation build from travis-ci.'
     git push -f --quiet origin gh-pages > /dev/null
     echo -e "\n${ANSI_GREEN}Documentation update pushed."
-else
+    popd
-    echo -e "\n${ANSI_GREEN}Not on official master branch, skip documentation update."
-fi
 
-travis_retry make coverage
+    travis_retry make coverage
-pushd koreader-*/koreader
+    pushd koreader-*/koreader
-    luajit "$(which luacov-coveralls)"
+        luajit "$(which luacov-coveralls)"
-popd
+    popd
+else
+    echo -e "\n${ANSI_GREEN}Not on official master branch, skip documentation update and coverage."
+fi

.ci/helper_luarocks.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=/dev/null
+source "${CI_DIR}/common.sh"
+
+rm -rf "${HOME}/.luarocks"
+mkdir "${HOME}/.luarocks"
+cp "${TRAVIS_BUILD_DIR}/install/etc/luarocks/config.lua" "${HOME}/.luarocks/config.lua"
+echo "wrap_bin_scripts = false" >> "$HOME/.luarocks/config.lua"
+travis_retry luarocks --local install luafilesystem
+# for verbose_print module
+travis_retry luarocks --local install ansicolors
+travis_retry luarocks --local install busted 2.0.rc12-1
+#- mv -f $HOME/.luarocks/bin/busted_bootstrap $HOME/.luarocks/bin/busted
+travis_retry luarocks --local install luacov
+# luasec doesn't automatically detect 64-bit libs
+travis_retry luarocks --local install luasec OPENSSL_LIBDIR=/usr/lib/x86_64-linux-gnu
+travis_retry luarocks --local install luacov-coveralls --server=http://rocks.moonscript.org/dev
+travis_retry luarocks --local install luacheck
+travis_retry luarocks --local install lanes  # for parallel luacheck

.ci/helper_shellchecks.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=/dev/null
+source "${CI_DIR}/common.sh"
+
+# shellcheck disable=2016
+mapfile -t shellscript_locations < <( { git grep -lE '^#!(/usr)?/bin/(env )?(bash|sh)' && git submodule --quiet foreach '[ "$path" = "base" ] || git grep -lE "^#!(/usr)?/bin/(env )?(bash|sh)" | sed "s|^|$path/|"' && git ls-files ./*.sh ; } | sort | uniq )
+
+SHELLSCRIPT_ERROR=0
+
+for shellscript in "${shellscript_locations[@]}"; do
+    echo -e "${ANSI_GREEN}Running shellcheck on ${shellscript}"
+    shellcheck "${shellscript}" || SHELLSCRIPT_ERROR=1
+    echo -e "${ANSI_GREEN}Running shfmt on ${shellscript}"
+    [ "$(cat "${shellscript}" )" != "$(shfmt -i 4 "${shellscript}")" ] && echo  -e "${ANSI_RED}Warning: ${shellscript} does not abide by coding style"
+    # @TODO add error handling with something like && shfmt -i 4 "${shellscript}" | diff "${shellscript}"
+done
+
+exit "${SHELLSCRIPT_ERROR}"

.ci/install.sh

@@ -4,6 +4,22 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=/dev/null
 source "${CI_DIR}/common.sh"
 
+# print some useful info
+echo "TRAVIS_BUILD_DIR: ${TRAVIS_BUILD_DIR}"
+echo "pwd: $(pwd)"
+ls
+
+# toss submodules if there are any changes
+if [ "$(git status --ignore-submodules=dirty --porcelain)" ]; then
+    # what changed?
+    git status
+    # purge and reinit submodules
+    git submodule deinit -f .
+    git submodule update --init
+else
+    echo -e "${ANSI_GREEN}Using cached submodules."
+fi
+
 # install our own updated luarocks
 if [ ! -f "${TRAVIS_BUILD_DIR}/install/bin/luarocks" ]; then
     git clone https://github.com/torch/luajit-rocks.git
@@ -12,22 +28,17 @@ if [ ! -f "${TRAVIS_BUILD_DIR}/install/bin/luarocks" ]; then
         cmake . -DWITH_LUAJIT21=ON -DCMAKE_INSTALL_PREFIX="${TRAVIS_BUILD_DIR}/install"
         make install
     popd
+else
+    echo -e "${ANSI_GREEN}Using cached luarocks."
 fi
 
-mkdir "${HOME}/.luarocks"
+if [ ! -d "${HOME}/.luarocks" ] || [ ! -f "${HOME}/.luarocks/$(md5sum < "${CI_DIR}/helper_luarocks.sh")" ] ; then
-cp "${TRAVIS_BUILD_DIR}/install/etc/luarocks/config.lua" "$HOME/.luarocks/config.lua"
+    echo -e "${ANSI_GREEN}Grabbing new .luarocks."
-echo "wrap_bin_scripts = false" >> "$HOME/.luarocks/config.lua"
+    "${CI_DIR}/helper_luarocks.sh"
-travis_retry luarocks --local install luafilesystem
+    touch "${HOME}/.luarocks/$(md5sum < "${CI_DIR}/helper_luarocks.sh")"
-# for verbose_print module
+else
-travis_retry luarocks --local install ansicolors
+    echo -e "${ANSI_GREEN}Using cached .luarocks."
-travis_retry luarocks --local install busted 2.0.rc12-1
+fi
-#- mv -f $HOME/.luarocks/bin/busted_bootstrap $HOME/.luarocks/bin/busted
-travis_retry luarocks --local install luacov
-# luasec doesn't automatically detect 64-bit libs
-travis_retry luarocks --local install luasec OPENSSL_LIBDIR=/usr/lib/x86_64-linux-gnu
-travis_retry luarocks --local install luacov-coveralls --server=http://rocks.moonscript.org/dev
-travis_retry luarocks --local install luacheck
-travis_retry luarocks --local install lanes  # for parallel luacheck
 
 #install our own updated shellcheck
 SHELLCHECK_URL="https://s3.amazonaws.com/travis-blue-public/binaries/ubuntu/14.04/x86_64/shellcheck-0.4.5.tar.bz2"
@@ -35,6 +46,8 @@ if ! command -v shellcheck ; then
     curl -sSL "${SHELLCHECK_URL}" | tar --exclude 'SHA256SUMS' --strip-components=1 -C "${HOME}/bin" -xjf -;
     chmod +x "${HOME}/bin/shellcheck"
     shellcheck --version
+else
+    echo -e "${ANSI_GREEN}Using cached shellcheck."
 fi
 
 # install shfmt
@@ -42,4 +55,6 @@ SHFMT_URL="https://github.com/mvdan/sh/releases/download/v1.2.0/shfmt_v1.2.0_lin
 if ! command -v shfmt ; then
     curl -sSL "${SHFMT_URL}" -o "${HOME}/bin/shfmt"
     chmod +x "${HOME}/bin/shfmt"
+else
+    echo -e "${ANSI_GREEN}Using cached shfmt."
 fi

.ci/script.sh

@@ -4,9 +4,15 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=/dev/null
 source "${CI_DIR}/common.sh"
 
+echo -e "\n${ANSI_GREEN}make fetchthirdparty"
 travis_retry make fetchthirdparty
-find . -type f -name '*.sh' -not -path "./base/*" -not -path "./luajit-rocks/*" -print0 | xargs --null shellcheck
+
-find . -type f -name '*.sh' -not -path "./base/*" -not -path "./luajit-rocks/*" -print0 | xargs shfmt -i 0 -w
+"${CI_DIR}/helper_shellchecks.sh"
-make all
+
-make testfront
+echo -e "\n${ANSI_GREEN}Luacheck results"
 luajit "$(which luacheck)" --no-color -q {reader,setupkoenv,datastorage}.lua frontend plugins
+
+echo -e "\n${ANSI_GREEN}make all"
+make all
+echo -e "\n${ANSI_GREEN}make testfront"
+make testfront

.gitignore

@@ -12,6 +12,8 @@ tags
 test/*
 *.tar
 *.log
+bin/
+install/
 spec/unit/data
 doc/html
 git-rev

.travis.yml

@@ -18,8 +18,14 @@ cache:
   directories:
     - "${HOME}/bin"
     # compiled luarocks binaries
-    - "${HOME}/build/koreader/koreader/install"
+    - "${TRAVIS_BUILD_DIR}/install"
+    # base build
+    - "${TRAVIS_BUILD_DIR}/base"
     - "${HOME}/.ccache"
+    - "${HOME}/.luarocks"
+before_cache:
+  # don't quote like you normally would or it won't expand
+  - rm -frv ${TRAVIS_BUILD_DIR}/base/build/*/cache/*
 
 addons:
   apt:
@@ -28,7 +34,7 @@ addons:
     packages:
       - g++-4.8
       - libsdl1.2-dev
-      # for luasec
+      # luasec dependencies
       - libssl1.0.0
       - nasm
       # OpenSSL likes this (package contains makedepend)

platform/kindle/kotar_cpoint

@@ -8,21 +8,21 @@ export EIPS_NO_SLEEP="true"
 
 # Load our helper functions...
 if [ -f "${KOREADER_DIR}/libkohelper.sh" ] ; then
-	source "${KOREADER_DIR}/libkohelper.sh"
+	# shellcheck source=/dev/null
+	. "${KOREADER_DIR}/libkohelper.sh"
 else
 	echo "Can't source helper functions, aborting!"
 	exit 1
 fi
 
-
 ## First arg is the chekpoint number, and we get one every 200 checkpoints.
 CHECKPOINT_NUM="${1}"
 CHECKPOINT_GRANULARITY="200"
 
 # Use that to build a poor man's progress bar, with dots.
-PROGRESS_AMOUNT="$(( ${CHECKPOINT_NUM} / ${CHECKPOINT_GRANULARITY} ))"
+PROGRESS_AMOUNT="$(( CHECKPOINT_NUM / CHECKPOINT_GRANULARITY ))"
 PROGRESS_STRING="Updating koreader "
-for foo in $( seq 1 ${PROGRESS_AMOUNT} ) ; do
+for _ in $( seq 1 ${PROGRESS_AMOUNT} ) ; do
 	# Append a dot until we hit the needed amount
 	PROGRESS_STRING="${PROGRESS_STRING}."
 done

platform/pocketbook/koreader.app

@@ -7,16 +7,16 @@ KOREADER_DIR=/mnt/ext1/applications/koreader
 # update to new version from OTA directory
 NEWUPDATE="${KOREADER_DIR}/ota/koreader.updated.tar"
 INSTALLED="${KOREADER_DIR}/ota/koreader.installed.tar"
-if [ -f $NEWUPDATE ]; then
+if [ -f ${NEWUPDATE} ]; then
     # TODO: any graphic indication for the updating progress?
-    cd /mnt/ext1/ && tar xf $NEWUPDATE && mv $NEWUPDATE $INSTALLED
+    cd /mnt/ext1/ && tar xf ${NEWUPDATE} && mv ${NEWUPDATE} ${INSTALLED}
 fi
 
 # we're always starting from our working directory
-cd $KOREADER_DIR
+cd ${KOREADER_DIR} || exit
 
 # export load library path for some old firmware
-export LD_LIBRARY_PATH=${KOREADER_DIR}/libs:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=${KOREADER_DIR}/libs:${LD_LIBRARY_PATH}
 
 # export trained OCR data directory
 export TESSDATA_PREFIX="data"
@@ -27,18 +27,21 @@ export STARDICT_DATA_DIR="data/dict"
 # export external font directory
 export EXT_FONT_DIR="/mnt/ext1/fonts"
 
-if [ `echo $@ | wc -c` -eq 1 ]; then
+# shellcheck disable=2000
+if [ "$(echo "$@" | wc -c)" -eq 1 ]; then
     args="/mnt/ext1/"
 else
-    args="$@"
+    args="$*"
 fi
 
 # we keep maximum 500K worth of crash log
-cat crash.log 2> /dev/null | tail -c 500000 > crash.log.new
+if [ -e crash.log ]; then
-mv -f crash.log.new crash.log
+    tail -c 500000 crash.log > crash.log.new
-./reader.lua "$args" >> crash.log 2>&1
+    mv -f crash.log.new crash.log
-
-if pidof reader.lua > /dev/null 2>&1 ; then
-	killall -TERM reader.lua
 fi
 
+./reader.lua "${args}" >> crash.log 2>&1
+
+if pidof reader.lua > /dev/null 2>&1 ; then
+    killall -TERM reader.lua
+fi