build: fix doc generation

also moved MD5 to koreader-base/ffi

.ci/after_success.sh

@@ -1,16 +1,41 @@
 #!/usr/bin/env bash
 
-make coverage
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-cd koreader-*/koreader && luajit $(which luacov-coveralls) -v
+source "${CI_DIR}/common.sh"
 
-# get deploy key for doc repo
+set +e
-openssl aes-256-cbc -K $encrypted_dc71a4fb8382_key -iv $encrypted_dc71a4fb8382_iv \
+
-    -in .ci/koreader_doc.enc -out ~/.ssh/koreader_doc -d
+make coverage
-ssh-add ~/.ssh/koreader_doc
+pushd koreader-*/koreader
-git clone git@github.com:koreader/doc.git
+    luajit $(which luacov-coveralls) -v
-# push doc update
+popd
-make doc
+
-cp -r doc/html/* doc/
+if [ $TRAVIS_BRANCH == 'master' ]; then
-pushd doc
+
-git add .
+    travis_retry luarocks --local install ldoc
-git push origin gh-pages
+    # get deploy key for doc repo
+    openssl aes-256-cbc -k $doc_build_secret -in .ci/koreader_doc.enc -out ~/.ssh/koreader_doc -d
+    chmod 600 ~/.ssh/koreader_doc  # make agent happy
+    eval "$(ssh-agent)" > /dev/null
+    ssh-add ~/.ssh/koreader_doc > /dev/null
+    echo -e "\n${ANSI_GREEN}Check out koreader/doc for update."
+    git clone git@github.com:koreader/doc.git koreader_doc
+
+    # push doc update
+    pushd doc
+        luajit $(which ldoc) . 2> /dev/null
+        if [ ! -d html ]; then
+            echo "Failed to generate documents..."
+            exit 1
+        fi
+    popd
+    cp -r doc/html/* koreader_doc/
+    pushd koreader_doc
+
+    echo -e "\n${ANSI_GREEN}Pusing document update..."
+    git -c user.name="KOReader build bot" -c user.email="non-reply@koreader.rocks" \
+        commit -a --amend -m 'Automated documentation build from travis-ci.'
+    git push -f --quiet origin gh-pages > /dev/null
+    echo -e "\n${ANSI_GREEN}Document update pushed."
+
+fi

.ci/common.sh

@@ -1,3 +1,6 @@
+set -e
+set -o pipefail
+
 ANSI_RED="\033[31;1m"
 ANSI_GREEN="\033[32;1m"
 ANSI_RESET="\033[0m"
@@ -6,6 +9,8 @@ ANSI_CLEAR="\033[0K"
 travis_retry() {
   local result=0
   local count=1
+  set +e
+
   while [ $count -le 3 ]; do
     [ $result -ne 0 ] && {
       echo -e "\n${ANSI_RED}The command \"$@\" failed. Retrying, $count of 3.${ANSI_RESET}\n" >&2
@@ -21,6 +26,7 @@ travis_retry() {
     echo -e "\n${ANSI_RED}The command \"$@\" failed 3 times.${ANSI_RESET}\n" >&2
   }
 
+  set -e
   return $result
 }
 

.ci/install.sh

@@ -15,6 +15,7 @@ mkdir $HOME/.luarocks
 cp ${TRAVIS_BUILD_DIR}/install/etc/luarocks/config.lua $HOME/.luarocks/config.lua
 echo "wrap_bin_scripts = false" >> $HOME/.luarocks/config.lua
 travis_retry luarocks --local install luafilesystem
+# for verbose_print module
 travis_retry luarocks --local install ansicolors
 travis_retry luarocks --local install busted 2.0.rc11-0
 #- travis_retry luarocks --local install busted 1.11.1-1
@@ -25,4 +26,3 @@ travis_retry luarocks --local install luasec OPENSSL_LIBDIR=/usr/lib/x86_64-linu
 travis_retry luarocks --local install luacov-coveralls --server=http://rocks.moonscript.org/dev
 travis_retry luarocks --local install luacheck
 travis_retry luarocks --local install lanes  # for parallel luacheck
-travis_retry luarocks --local install ldoc

.ci/script.sh

@@ -6,5 +6,6 @@ source "${CI_DIR}/common.sh"
 travis_retry make fetchthirdparty
 make all
 travis_retry make testfront
+set +o pipefail
 luajit $(which luacheck) --no-color -q frontend | tee ./luacheck.out
 test $(grep Total ./luacheck.out | awk '{print $2}') -le 63

.gitignore

@@ -33,7 +33,7 @@ i18n
 koreader-android-arm-linux-androideabi
 koreader-kindle-legacy-arm-kindle-linux-gnueabi
 koreader-kindle-arm-linux-gnueabi
-koreader-kobo-arm-linux-gnueabihf
+koreader-kobo-arm-linux-gnueabihf*
 koreader-emulator-i686-w64-mingw32
 koreader-emulator-x86_64-linux-gnu
 koreader-emulator-x86_64-pc-linux-gnu

.travis.yml

@@ -1,6 +1,8 @@
 language: c
 
-sudo: false
+# sudo: false
+sudo: true
+dist: trusty
 
 compiler:
   - gcc

frontend/MD5.lua

@@ -1,246 +0,0 @@
---[[--
-MD5 hash library.
-  ]]
-
-local ffi = require "ffi"
-local bit = require "bit"
-local bxor = bit.bxor
-local bnot = bit.bnot
-local band = bit.band
-local bor = bit.bor
-local rshift = bit.rshift
-local lshift = bit.lshift
-local copy = ffi.copy
-local fill = ffi.fill
-
-ffi.cdef[[
-typedef struct MD5Context {
-  uint32_t buf[4];
-  uint32_t bits[2];
-  unsigned char input[64];
-} MD5_CTX;
-]]
-
-local function byteReverse(buf, len)
-    -- TODO: implement for big-endian architectures?
-end
-
-local function F1(x, y, z) return bxor(z, band(x, bxor(y, z))) end
-local function F2(x, y, z) return F1(z, x, y) end
-local function F3(x, y, z) return bxor(x, y, z) end
-local function F4(x, y, z) return bxor(y, bor(x, bnot(z))) end
-
-local function MD5STEP(f, w, x, y, z, data, s)
-    w = w + f(x, y, z) + data
-    w = bor(lshift(w,s), rshift(w,(32-s)))
-    w = w + x
-
-    return w
-end
-
--- Start MD5 accumulation.  Set bit count to 0 and buffer to mysterious
--- initialization constants.
-local function MD5Init(ctx)
-    ctx.buf[0] = 0x67452301
-    ctx.buf[1] = 0xefcdab89
-    ctx.buf[2] = 0x98badcfe
-    ctx.buf[3] = 0x10325476
-
-    ctx.bits[0] = 0
-    ctx.bits[1] = 0
-end
-
-local function MD5Transform(buf, input)
-    local a = buf[0]
-    local b = buf[1]
-    local c = buf[2]
-    local d = buf[3]
-
-    a = MD5STEP(F1, a, b, c, d, input[0] + 0xd76aa478, 7)
-    d = MD5STEP(F1, d, a, b, c, input[1] + 0xe8c7b756, 12)
-    c = MD5STEP(F1, c, d, a, b, input[2] + 0x242070db, 17)
-    b = MD5STEP(F1, b, c, d, a, input[3] + 0xc1bdceee, 22)
-    a = MD5STEP(F1, a, b, c, d, input[4] + 0xf57c0faf, 7)
-    d = MD5STEP(F1, d, a, b, c, input[5] + 0x4787c62a, 12)
-    c = MD5STEP(F1, c, d, a, b, input[6] + 0xa8304613, 17)
-    b = MD5STEP(F1, b, c, d, a, input[7] + 0xfd469501, 22)
-    a = MD5STEP(F1, a, b, c, d, input[8] + 0x698098d8, 7)
-    d = MD5STEP(F1, d, a, b, c, input[9] + 0x8b44f7af, 12)
-    c = MD5STEP(F1, c, d, a, b, input[10] + 0xffff5bb1, 17)
-    b = MD5STEP(F1, b, c, d, a, input[11] + 0x895cd7be, 22)
-    a = MD5STEP(F1, a, b, c, d, input[12] + 0x6b901122, 7)
-    d = MD5STEP(F1, d, a, b, c, input[13] + 0xfd987193, 12)
-    c = MD5STEP(F1, c, d, a, b, input[14] + 0xa679438e, 17)
-    b = MD5STEP(F1, b, c, d, a, input[15] + 0x49b40821, 22)
-
-    a = MD5STEP(F2, a, b, c, d, input[1] + 0xf61e2562, 5)
-    d = MD5STEP(F2, d, a, b, c, input[6] + 0xc040b340, 9)
-    c = MD5STEP(F2, c, d, a, b, input[11] + 0x265e5a51, 14)
-    b = MD5STEP(F2, b, c, d, a, input[0] + 0xe9b6c7aa, 20)
-    a = MD5STEP(F2, a, b, c, d, input[5] + 0xd62f105d, 5)
-    d = MD5STEP(F2, d, a, b, c, input[10] + 0x02441453, 9)
-    c = MD5STEP(F2, c, d, a, b, input[15] + 0xd8a1e681, 14)
-    b = MD5STEP(F2, b, c, d, a, input[4] + 0xe7d3fbc8, 20)
-    a = MD5STEP(F2, a, b, c, d, input[9] + 0x21e1cde6, 5)
-    d = MD5STEP(F2, d, a, b, c, input[14] + 0xc33707d6, 9)
-    c = MD5STEP(F2, c, d, a, b, input[3] + 0xf4d50d87, 14)
-    b = MD5STEP(F2, b, c, d, a, input[8] + 0x455a14ed, 20)
-    a = MD5STEP(F2, a, b, c, d, input[13] + 0xa9e3e905, 5)
-    d = MD5STEP(F2, d, a, b, c, input[2] + 0xfcefa3f8, 9)
-    c = MD5STEP(F2, c, d, a, b, input[7] + 0x676f02d9, 14)
-    b = MD5STEP(F2, b, c, d, a, input[12] + 0x8d2a4c8a, 20)
-
-    a = MD5STEP(F3, a, b, c, d, input[5] + 0xfffa3942, 4)
-    d = MD5STEP(F3, d, a, b, c, input[8] + 0x8771f681, 11)
-    c = MD5STEP(F3, c, d, a, b, input[11] + 0x6d9d6122, 16)
-    b = MD5STEP(F3, b, c, d, a, input[14] + 0xfde5380c, 23)
-    a = MD5STEP(F3, a, b, c, d, input[1] + 0xa4beea44, 4)
-    d = MD5STEP(F3, d, a, b, c, input[4] + 0x4bdecfa9, 11)
-    c = MD5STEP(F3, c, d, a, b, input[7] + 0xf6bb4b60, 16)
-    b = MD5STEP(F3, b, c, d, a, input[10] + 0xbebfbc70, 23)
-    a = MD5STEP(F3, a, b, c, d, input[13] + 0x289b7ec6, 4)
-    d = MD5STEP(F3, d, a, b, c, input[0] + 0xeaa127fa, 11)
-    c = MD5STEP(F3, c, d, a, b, input[3] + 0xd4ef3085, 16)
-    b = MD5STEP(F3, b, c, d, a, input[6] + 0x04881d05, 23)
-    a = MD5STEP(F3, a, b, c, d, input[9] + 0xd9d4d039, 4)
-    d = MD5STEP(F3, d, a, b, c, input[12] + 0xe6db99e5, 11)
-    c = MD5STEP(F3, c, d, a, b, input[15] + 0x1fa27cf8, 16)
-    b = MD5STEP(F3, b, c, d, a, input[2] + 0xc4ac5665, 23)
-
-    a = MD5STEP(F4, a, b, c, d, input[0] + 0xf4292244, 6)
-    d = MD5STEP(F4, d, a, b, c, input[7] + 0x432aff97, 10)
-    c = MD5STEP(F4, c, d, a, b, input[14] + 0xab9423a7, 15)
-    b = MD5STEP(F4, b, c, d, a, input[5] + 0xfc93a039, 21)
-    a = MD5STEP(F4, a, b, c, d, input[12] + 0x655b59c3, 6)
-    d = MD5STEP(F4, d, a, b, c, input[3] + 0x8f0ccc92, 10)
-    c = MD5STEP(F4, c, d, a, b, input[10] + 0xffeff47d, 15)
-    b = MD5STEP(F4, b, c, d, a, input[1] + 0x85845dd1, 21)
-    a = MD5STEP(F4, a, b, c, d, input[8] + 0x6fa87e4f, 6)
-    d = MD5STEP(F4, d, a, b, c, input[15] + 0xfe2ce6e0, 10)
-    c = MD5STEP(F4, c, d, a, b, input[6] + 0xa3014314, 15)
-    b = MD5STEP(F4, b, c, d, a, input[13] + 0x4e0811a1, 21)
-    a = MD5STEP(F4, a, b, c, d, input[4] + 0xf7537e82, 6)
-    d = MD5STEP(F4, d, a, b, c, input[11] + 0xbd3af235, 10)
-    c = MD5STEP(F4, c, d, a, b, input[2] + 0x2ad7d2bb, 15)
-    b = MD5STEP(F4, b, c, d, a, input[9] + 0xeb86d391, 21)
-
-    buf[0] = band(buf[0] + a, 0xFFFFFFFF)
-    buf[1] = band(buf[1] + b, 0xFFFFFFFF)
-    buf[2] = band(buf[2] + c, 0xFFFFFFFF)
-    buf[3] = band(buf[3] + d, 0xFFFFFFFF)
-end
-
-local function MD5Update(ctx, buf, len)
-    local t
-
-    t = ctx.bits[0]
-    ctx.bits[0] = t + lshift( len, 3)
-    if (ctx.bits[0] < t) then
-        ctx.bits[1] = ctx.bits[1] + 1
-    end
-
-    ctx.bits[1] = ctx.bits[1] + rshift(len, 29)
-
-    t = band(rshift(t, 3), 0x3f)
-
-    if (t > 0) then
-        local p = ffi.cast("unsigned char *", ctx.input + t)
-
-        t = 64 - t
-        if (len < t) then
-            copy(p, buf, len)
-            return
-        end
-
-        copy(p, buf, t)
-        byteReverse(ctx.input, 16)
-        MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input))
-        buf = buf + t
-        len = len - t
-    end
-
-    while (len >= 64) do
-        copy(ctx.input, buf, 64)
-        byteReverse(ctx.input, 16)
-        MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input))
-        buf = buf + 64
-        len = len - 64
-    end
-
-    copy(ctx.input, buf, len)
-end
-
-local function MD5Final(digest, ctx)
-
-    local count
-    local p
-
-    count = band(rshift(ctx.bits[0], 3), 0x3F)
-
-    p = ctx.input + count
-    p[0] = 0x80
-    p = p + 1
-    count = 64 - 1 - count
-
-    if (count < 8) then
-        fill(p, count, 0)
-        byteReverse(ctx.input, 16)
-        MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input))
-        fill(ctx.input, 56, 0)
-    else
-        fill(p, count - 8, 0)
-    end
-
-    byteReverse(ctx.input, 14)
-
-    ffi.cast("uint32_t *", ctx.input)[14] = ctx.bits[0]
-    ffi.cast("uint32_t *", ctx.input)[15] = ctx.bits[1]
-
-    MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input))
-    byteReverse(ffi.cast("unsigned char *",ctx.buf), 4)
-    copy(digest, ctx.buf, 16)
-    fill(ffi.cast("char *", ctx), ffi.sizeof(ctx), 0)
-end
-
-local hex = ffi.new("const char[16]", "0123456789abcdef")
-local function bin2str(output, input, len)
-    if len > 0 then
-        output[0] = hex[rshift(input[0], 4)]
-        output[1] = hex[band(input[0], 0xF)]
-        return bin2str(output+2, input+1, len-1)
-    end
-end
-
-local md5 = {}
-
---- Create a new md5 hashing instance.
----- @return md5 instance
-function md5:new()
-    self.ctx = ffi.new("MD5_CTX")
-    MD5Init(self.ctx)
-end
-
---- Feed content to md5 hashing instance.
----- @param luastr Lua string
-function md5:update(luastr)
-    MD5Update(self.ctx, ffi.cast("const char*", luastr), #luastr)
-end
-
---- Calcualte md5 sum.
----- @param luastr Lua string
----- @return md5 sum in Lua string
-function md5:sum(luastr)
-    local buf = ffi.new("char[33]")
-    local hash = ffi.new("uint8_t[16]")
-    if luastr then
-        md5:new()
-        md5:update(luastr)
-    end
-    MD5Final(hash, self.ctx)
-
-    bin2str(buf, hash, ffi.sizeof(hash))
-
-    return ffi.string(buf)
-end
-
-return md5

frontend/cache.lua

@@ -1,7 +1,7 @@
 --[[
 A global LRU cache
 ]]--
-local md5 = require("MD5")
+local md5 = require("ffi/MD5")
 local lfs = require("libs/libkoreader-lfs")
 local DataStorage = require("datastorage")
 local DEBUG = require("dbg")
@@ -123,7 +123,7 @@ function Cache:check(key, ItemClass)
         end
         return self.cache[key]
     elseif ItemClass then
-        local cached = self.cached[md5:sum(key)]
+        local cached = self.cached[md5.sum(key)]
         if cached then
             local item = ItemClass:new{}
             local ok, msg = pcall(item.load, item, cached)
@@ -153,14 +153,14 @@ function Cache:serialize()
         cached_size = cached_size + (lfs.attributes(file, "size") or 0)
     end
     table.sort(sorted_caches, function(v1,v2) return v1.time > v2.time end)
-    -- serialize the most recently used cache
+    -- only serialize the most recently used cache
     local cache_size = 0
     for _, key in ipairs(self.cache_order) do
         local cache_item = self.cache[key]
         -- only dump cache item that requests serialization explicitly
         if cache_item.persistent and cache_item.dump then
             DEBUG("dump cache item", key)
-            cache_size = cache_item:dump(cache_path..md5:sum(key)) or 0
+            cache_size = cache_item:dump(cache_path..md5.sum(key)) or 0
             if cache_size > 0 then break end
         end
     end

frontend/document/document.lua

@@ -109,23 +109,23 @@ end
 -- 1048576, 4194304, 16777216, 67108864, 268435456 or 1073741824, appending data
 -- by highlighting in koreader may change the digest value.
 function Document:fastDigest()
-    local md5 = require("MD5")
+    local md5 = require("ffi/MD5")
     local lshift = bit.lshift
     local file = io.open(self.file, 'rb')
     if file then
         local step, size = 1024, 1024
-        md5:new()
+        local m = md5.new()
         for i = -1, 10 do
             file:seek("set", lshift(step, 2*i))
             local sample = file:read(size)
             if sample then
-                md5:update(sample)
+                m:update(sample)
             else
                 break
             end
         end
         file:close()
-        return md5:sum()
+        return m:sum()
     end
 end
 

kodev

@@ -42,7 +42,7 @@ OPTIONS:
 TARGET:
 ${SUPPORTED_TARGETS}"
 
-    while [[ $1 == '--'* ]]; do
+    while [[ $1 == '-'* ]]; do
         PARAM=`echo $1 | awk -F= '{print $1}'`
         VALUE=`echo $1 | awk -F= '{print $2}'`
         case $PARAM in
@@ -59,7 +59,7 @@ ${SUPPORTED_TARGETS}"
                 exit 1
                 ;;
         esac
-        shift
+        shift 1
     done
 
     case $1 in
@@ -215,7 +215,7 @@ OPTIONS:
     --no-build          run reader without rebuilding
     --disable-touch     use this if you want to simulate keyboard only devices
     "
-    while [[ $1 == '--'* ]]; do
+    while [[ $1 == '-'* ]]; do
         PARAM=`echo $1 | awk -F= '{print $1}'`
         VALUE=`echo $1 | awk -F= '{print $2}'`
         case $PARAM in
@@ -274,7 +274,7 @@ OPTIONS:
 
     --tags=TAGS     only run tests with given tags
     "
-    while [[ $1 == '--'* ]]; do
+    while [[ $1 == '-'* ]]; do
         PARAM=`echo $1 | awk -F= '{print $1}'`
         VALUE=`echo $1 | awk -F= '{print $2}'`
         case $PARAM in
@@ -308,7 +308,7 @@ OPTIONS:
     if [ ! -z $2 ]; then
         test_path="${test_path}/$2"
     fi
-    busted ${opts} -o verbose_print --exclude-tags=notest ${test_path}
+    busted --lua=./luajit ${opts} -o ./spec/$1/unit/verbose_print --exclude-tags=notest ${test_path}
 
     popd
 }

spec/unit/kosync_spec.lua

@@ -1,7 +1,7 @@
 require("commonrequire")
 local UIManager = require("ui/uimanager")
 local DEBUG = require("dbg")
-local md5 = require("MD5")
+local md5 = require("ffi/MD5")
 --DEBUG:turnOn()
 
 local service = [[
@@ -70,7 +70,7 @@ describe("KOSync modules #notest #nocov", function()
         req.headers['x-auth-key'] = args.userkey
     end
     -- password should be hashed before submitting to server
-    local username, password = "koreader", md5:sum("koreader")
+    local username, password = "koreader", md5.sum("koreader")
     -- fake progress data
     local doc, percentage, progress, device =
         "41cce710f34e5ec21315e19c99821415", -- fast digest of the document

spec/unit/md5_spec.lua

@@ -1,19 +0,0 @@
-require("commonrequire")
-
-local md5 = require("MD5")
-
-describe("MD5 module", function()
-    it("should calculate correct MD5 hashes", function()
-        assert.is_equal(md5:sum(""), "d41d8cd98f00b204e9800998ecf8427e")
-        assert.is_equal(md5:sum("\0"), "93b885adfe0da089cdf634904fd59f71")
-        assert.is_equal(md5:sum("0123456789abcdefX"), "1b05aba914a8b12315c7ee52b42f3d35")
-    end)
-    it("should calculate MD5 sum by updating", function()
-        md5:new()
-        md5:update("0123456789")
-        md5:update("abcdefghij")
-        local md5sum = md5:sum()
-        assert.is_equal(md5sum, md5:sum("0123456789abcdefghij"))
-    end)
-end)
-