evernote: ReadHistory integration and text file output (#2498)

frontend/docsettings.lua

@@ -17,6 +17,7 @@ end
 
 -- Sidecar directory is the file without _last_ suffix.
 function DocSettings:getSidecarDir(doc_path)
+    if doc_path == nil or doc_path == '' then return '' end
     local file_without_suffix = doc_path:match("(.*)%.")
     if file_without_suffix then
         return file_without_suffix..".sdr"
@@ -26,6 +27,11 @@ function DocSettings:getSidecarDir(doc_path)
     return doc_path..".sdr"
 end
 
+function DocSettings:getSidecarFile(doc_path)
+    if doc_path == nil or doc_path == '' then return '' end
+    return self:getSidecarDir(doc_path) .. "/" .. doc_path:match(".*%.(.+)") .. ".lua"
+end
+
 function DocSettings:hasSidecarDir(doc_path)
     -- We may be called with items from FileManager, which may not be a document
     if lfs.attributes(doc_path, "mode") == "directory" then
@@ -39,24 +45,30 @@ function DocSettings:getHistoryPath(fullpath)
 end
 
 function DocSettings:getPathFromHistory(hist_name)
-    if hist_name == nil or hist_name == '' then return nil end
+    if hist_name == nil or hist_name == '' then return '' end
     -- 1. select everything included in brackets
     local s = string.match(hist_name,"%b[]")
-    if s == nil or s == '' then return nil end
+    if s == nil or s == '' then return '' end
     -- 2. crop the bracket-sign from both sides
     -- 3. and finally replace decorative signs '#' to dir-char '/'
     return string.gsub(string.sub(s,2,-3),"#","/")
 end
 
 function DocSettings:getNameFromHistory(hist_name)
-    if hist_name == nil or hist_name == '' then return nil end
+    if hist_name == nil or hist_name == '' then return '' end
     local s = string.match(hist_name, "%b[]")
-    if s == nil or s == '' then return nil end
+    if s == nil or s == '' then return '' end
     -- at first, search for path length
     -- and return the rest of string without 4 last characters (".lua")
     return string.sub(hist_name, string.len(s)+2, -5)
 end
 
+function DocSettings:ensureSidecar(sidecar)
+    if lfs.attributes(sidecar, "mode") ~= "directory" then
+        lfs.mkdir(sidecar)
+    end
+end
+
 function DocSettings:open(docfile)
     -- TODO(zijiehe): Remove history_path, use only sidecar.
     local new = {}
@@ -65,16 +77,13 @@ function DocSettings:open(docfile)
 
     local sidecar = self:getSidecarDir(docfile)
     new.sidecar = sidecar
-    if lfs.attributes(sidecar, "mode") ~= "directory" then
-        lfs.mkdir(sidecar)
-    end
+    DocSettings:ensureSidecar(sidecar)
     -- If there is a file which has a same name as the sidecar directory, or
     -- the file system is read-only, we should not waste time to read it.
     if lfs.attributes(sidecar, "mode") == "directory" then
         -- New sidecar file name is metadata.{file last suffix}.lua. So we
         -- can handle two files with only different suffixes.
-        new.sidecar_file = sidecar.."/metadata."..
-                           docfile:match(".*%.(.+)")..".lua"
+        new.sidecar_file = self:getSidecarFile(docfile)
         new.legacy_sidecar_file = sidecar.."/"..
                                   docfile:match("([^%/]+%..+)")..".lua"
     end
@@ -136,6 +145,7 @@ function DocSettings:flush()
     -- If we can write to sidecar_file, we do not need to write to history_file
     -- anymore.
     local serials = { self.sidecar_file, self.history_file }
+    self:ensureSidecar(self.sidecar)
     local s_out = dump(self.data)
     os.setlocale('C', 'numeric')
     for _, f in pairs(serials) do
@@ -174,6 +184,7 @@ function DocSettings:purge()
     if lfs.attributes(self.sidecar, "mode") == "directory" then
         purgeDir(self.sidecar)
     end
+    self.data = {}
 end
 
 return DocSettings

frontend/util.lua

@@ -252,4 +252,18 @@ function util.replaceSlashChar(str)
     return str:gsub('%/','_')
 end
 
+-- Split a file into its path and name
+function util.splitFilePathName(file)
+    if file == nil or file == "" then return "", "" end
+    if string.find(file, "/") == nil then return "", file end
+    return string.gsub(file, "(.*/)(.*)", "%1"), string.gsub(file, ".*/", "")
+end
+
+-- Split a file name into its pure file name and suffix
+function util.splitFileNameSuffix(file)
+    if file == nil or file == "" then return "", "" end
+    if string.find(file, "%.") == nil then return file, "" end
+    return string.gsub(file, "(.*)%.(.*)", "%1"), string.gsub(file, ".*%.", "")
+end
+
 return util

plugins/evernote.koplugin/clip.lua

@@ -1,6 +1,8 @@
 local DocumentRegistry = require("document/documentregistry")
 local DocSettings = require("docsettings")
+local ReadHistory = require("readhistory")
 local md5 = require("ffi/MD5")
+local util = require("util")
 
 local MyClipping = {
     my_clippings = "/mnt/us/documents/My Clippings.txt",
@@ -254,25 +256,37 @@ function MyClipping:parseHighlight(highlights, book)
     table.sort(book, function(v1, v2) return v1[1].page < v2[1].page end)
 end
 
-function MyClipping:parseHistory()
-    local clippings = {}
-    for f in lfs.dir(self.history_dir) do
-        local path = self.history_dir.."/"..f
-        if lfs.attributes(path, "mode") == "file" and path:find(".+%.lua$") then
-            local ok, stored = pcall(dofile, path)
+function MyClipping:parseHistoryFile(clippings, history_file, doc_file)
+    if lfs.attributes(history_file, "mode") ~= "file"
+    or not history_file:find(".+%.lua$") then
+        return
+    end
+    if lfs.attributes(doc_file, "mode") ~= "file" then return end
+    local ok, stored = pcall(dofile, history_file)
     if ok and stored.highlight then
-                local _, _, docname = path:find("%[.*%](.*)%.lua$")
-                local title, author = self:getTitle(docname)
-                local docpath = DocSettings:getPathFromHistory(f)
-                local name = DocSettings:getNameFromHistory(f)
+        local _, docname = util.splitFilePathName(doc_file)
+        local title, author = self:getTitle(util.splitFileNameSuffix(docname))
         clippings[title] = {
-                    file = docpath .. "/" .. name,
+            file = doc_file,
             title = title,
             author = author,
         }
         self:parseHighlight(stored.highlight, clippings[title])
     end
 end
+
+function MyClipping:parseHistory()
+    local clippings = {}
+    for f in lfs.dir(self.history_dir) do
+        self:parseHistoryFile(clippings,
+                              self.history_dir .. "/" .. f,
+                              DocSettings:getPathFromHistory(f) .. "/" ..
+                              DocSettings:getNameFromHistory(f))
+    end
+    for _, item in ipairs(ReadHistory.hist) do
+        self:parseHistoryFile(clippings,
+                              DocSettings:getSidecarFile(item.file),
+                              item.file)
     end
 
     return clippings

plugins/evernote.koplugin/main.lua

@@ -13,6 +13,7 @@ local T = require("ffi/util").template
 local _ = require("gettext")
 local slt2 = require('slt2')
 local MyClipping = require("clip")
+local realpath = require("ffi/util").realpath
 
 local EvernoteExporter = InputContainer:new{
     name = "evernote",
@@ -35,6 +36,11 @@ function EvernoteExporter:init()
     self.evernote_token = settings.token
     self.notebook_guid = settings.notebook
     self.html_export = settings.html_export or false
+    if self.html_export then
+        self.txt_export = false
+    else
+        self.txt_export = settings.txt_export or false
+    end
 
     self.parser = MyClipping:new{
         my_clippings = "/mnt/us/documents/My Clippings.txt",
@@ -50,7 +56,7 @@ function EvernoteExporter:isDocless()
 end
 
 function EvernoteExporter:readyToExport()
-    return self.evernote_token ~= nil or self.html_export ~= false
+    return self.evernote_token ~= nil or self.html_export ~= false or self.txt_export ~= false
 end
 
 function EvernoteExporter:migrateClippings()
@@ -139,9 +145,28 @@ function EvernoteExporter:addToMainMenu(tab_item_table)
                 checked_func = function() return self.html_export end,
                 callback = function()
                     self.html_export = not self.html_export
+                    if self.html_export then self.txt_export = false end
                     self:saveSettings()
                 end
             },
+            {
+                text = _("Export to local clipping text file"),
+                checked_func = function() return self.txt_export end,
+                callback = function()
+                    self.txt_export = not self.txt_export
+                    if self.txt_export then self.html_export = false end
+                    self:saveSettings()
+                end
+            },
+            {
+                text = _("Purge history records"),
+                callback = function()
+                    self.config:purge()
+                    UIManager:show(InfoMessage:new{
+                        text = _("History records are purged.\nAll notes will be exported again next time.\nSuggest to remove existing KOReaderClipping.txt to avoid a duplication."),
+                    })
+                end
+            }
         }
     })
 end
@@ -254,6 +279,7 @@ function EvernoteExporter:saveSettings()
         token = self.evernote_token,
         notebook = self.notebook_guid,
         html_export = self.html_export,
+        txt_export = self.txt_export,
     }
     G_reader_settings:saveSetting("evernote", settings)
 end
@@ -319,13 +345,19 @@ end
 
 function EvernoteExporter:exportClippings(clippings)
     local client = nil
-    local exported_stamp = "html"
-    if not self.html_export then
+    local exported_stamp
+    if not self.html_export and not self.txt_export then
         client = require("EvernoteClient"):new{
             domain = self.evernote_domain,
             authToken = self.evernote_token,
         }
         exported_stamp = self.notebook_guid
+    elseif self.html_export then
+        exported_stamp= "html"
+    elseif self.txt_export then
+        exported_stamp = "txt"
+    else
+        assert("an exported_stamp is expected for a new export type")
     end
 
     local export_count, error_count = 0, 0
@@ -339,11 +371,11 @@ function EvernoteExporter:exportClippings(clippings)
         if booknotes.exported[exported_stamp] ~= true then
             local ok, err
             if self.html_export then
-                ok, err = pcall(self.exportBooknotesToHTML, self,
-                        title, booknotes)
+                ok, err = pcall(self.exportBooknotesToHTML, self, title, booknotes)
+            elseif self.txt_export then
+                ok, err = pcall(self.exportBooknotesToTXT, self, title, booknotes)
             else
-                ok, err = pcall(self.exportBooknotesToEvernote, self,
-                        client, title, booknotes)
+                ok, err = pcall(self.exportBooknotesToEvernote, self, client, title, booknotes)
             end
             -- error reporting
             if not ok and err and err:find("Transport not open") then
@@ -385,6 +417,9 @@ function EvernoteExporter:exportClippings(clippings)
             )
         end
     end
+    if self.html_export or self.txt_export then
+        msg = msg .. T(_("\nNotes can be found in %1/."), realpath(self.clipping_dir))
+    end
     UIManager:show(InfoMessage:new{ text = msg })
 end
 
@@ -427,4 +462,30 @@ function EvernoteExporter:exportBooknotesToHTML(title, booknotes)
     end
 end
 
+function EvernoteExporter:exportBooknotesToTXT(title, booknotes)
+    local file = io.open(self.clipping_dir .. "/KOReaderClipping.txt", "a")
+    if file then
+        file:write(title .. "\n")
+        for _ignore1, chapter in ipairs(booknotes) do
+            if chapter.title then
+                file:write(" - " .. chapter.title .. "\n\n")
+            end
+            for _ignore2, clipping in ipairs(chapter) do
+                file:write(T(_("   -- Page: %1, added on %2\n\n"),
+                             clipping.page, os.date("%c", clipping.time)))
+                if clipping.text then
+                    file:write(clipping.text)
+                end
+                if clipping.image then
+                    file:write(_("<An image>"))
+                end
+                file:write("\n==========\n")
+            end
+        end
+
+        file:write("\n")
+        file:close()
+    end
+end
+
 return EvernoteExporter

spec/unit/util_spec.lua

@@ -189,6 +189,37 @@ describe("util module", function()
         })
     end)
 
-
-
+    it("should split file path and name", function()
+        local test = function(full, path, name)
+            local p, n = util.splitFilePathName(full)
+            assert.are_same(p, path)
+            assert.are_same(n, name)
+        end
+        test("/a/b/c.txt", "/a/b/", "c.txt")
+        test("/a/b////c.txt", "/a/b////", "c.txt")
+        test("/a/b/", "/a/b/", "")
+        test("c.txt", "", "c.txt")
+        test("", "", "")
+        test(nil, "", "")
+        test("a/b", "a/", "b")
+        test("/b", "/", "b")
+        assert.are_same(util.splitFilePathName("/a/b/c.txt"), "/a/b/")
+    end)
+
+    it("should split file name and suffix", function()
+        local test = function(full, name, suffix)
+            local n, s = util.splitFileNameSuffix(full)
+            assert.are_same(n, name)
+            assert.are_same(s, suffix)
+        end
+        test("a.txt", "a", "txt")
+        test("/a/b.txt", "/a/b", "txt")
+        test("a", "a", "")
+        test("/a/b", "/a/b", "")
+        test("/a/", "/a/", "")
+        test("/a/.txt", "/a/", "txt")
+        test(nil, "", "")
+        test("", "", "")
+        assert.are_same(util.splitFileNameSuffix("a.txt"), "a")
+    end)
 end)