[plugin] Exporter: add Readwise.io support (#8548)

This extends exporter.koplugin with support for [Readwise.io](https://readwise.io), a highlight/notes aggregation service.

[Readwise API documentation](https://readwise.io/api_deets)

This additionally improves the highlight exporter's ability to get the correct title and author of a document, by checking actual metadata instead of inferring from filename. It also includes a modification to the plugin's highlight parsing logic to separate the highlight contents in `.text` from the notes in `.note`. This change actually fixes an existing bug in the HTML export template note.tpl, which has been missing notes because of the lack of the `.note` field.

plugins/exporter.koplugin/ReadwiseClient.lua

@@ -0,0 +1,70 @@
+local http = require("socket.http")
+local json = require("json")
+local logger = require("logger")
+local ltn12 = require("ltn12")
+local socket = require("socket")
+local socketutil = require("socketutil")
+
+local ReadwiseClient =  {
+    auth_token = ""
+}
+
+function ReadwiseClient:new(o)
+    o = o or {}
+    self.__index = self
+    setmetatable(o, self)
+    return o
+end
+
+function ReadwiseClient:_makeRequest(endpoint, method, request_body)
+    local sink = {}
+    local request_body_json = json.encode(request_body)
+    local source = ltn12.source.string(request_body_json)
+    socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT)
+    local request = {
+        url     = "https://readwise.io/api/v2/" .. endpoint,
+        method  = method,
+        sink    = ltn12.sink.table(sink),
+        source  = source,
+        headers = {
+            ["Content-Length"] = #request_body_json,
+            ["Content-Type"] = "application/json",
+            ["Authorization"] = "Token " .. self.auth_token
+        },
+    }
+    local code, _, status = socket.skip(1, http.request(request))
+    socketutil:reset_timeout()
+
+    if code ~= 200 then
+        logger.warn("ReadwiseClient: HTTP response code <> 200. Response status: ", status)
+        error("ReadwiseClient: HTTP response code <> 200.")
+    end
+
+    local response = json.decode(sink[1])
+
+    return response
+end
+
+function ReadwiseClient:createHighlights(booknotes)
+    local highlights = {}
+    for _, chapter in ipairs(booknotes) do
+        for _, clipping in ipairs(chapter) do
+            local highlight = {
+                text = clipping.text,
+                title = booknotes.title,
+                author = booknotes.author ~= "" and booknotes.author or nil, -- optional author
+                source_type = "koreader",
+                category = "books",
+                note = clipping.note,
+                location = clipping.page,
+                location_type = "page",
+                highlighted_at = os.date("!%Y-%m-%dT%TZ", clipping.time),
+            }
+            table.insert(highlights, highlight)
+        end
+    end
+    local result = self:_makeRequest("highlights", "POST", { highlights = highlights })
+    logger.dbg("ReadwiseClient createHighlights result", result)
+end
+
+return ReadwiseClient

plugins/exporter.koplugin/clip.lua

@@ -4,6 +4,8 @@ local ReadHistory = require("readhistory")
 local logger = require("logger")
 local md5 = require("ffi/sha2").md5
 local util = require("util")
+local _ = require("gettext")
+local T = require("ffi/util").template
 
 local MyClipping = {
     my_clippings = "/mnt/us/documents/My Clippings.txt",
@@ -98,10 +100,18 @@ local extensions = {
     [".doc"] = true,
 }
 
+-- first attempt to parse from document metadata
 -- remove file extensions added by former KOReader
 -- extract author name in "Title(Author)" format
 -- extract author name in "Title - Author" format
-function MyClipping:getTitle(line)
+function MyClipping:getTitle(line, path)
+    if path then
+        local props = self:getProps(path)
+        if props and props.title ~= "" then
+            return props.title, props.authors or props.author
+        end
+    end
+
     line = line:match("^%s*(.-)%s*$") or ""
     if extensions[line:sub(-4):lower()] then
         line = line:sub(1, -5)
@@ -228,6 +238,15 @@ end
 
 function MyClipping:parseHighlight(highlights, bookmarks, book)
     --DEBUG("book", book.file)
+
+    -- create a translated pattern that matches bookmark auto-text
+    -- see ReaderBookmark:getBookmarkAutoText and ReaderBookmark:getBookmarkPageString
+    --- @todo Remove this once we get rid of auto-text or improve the data model.
+    local pattern = "^" .. T(_("Page %1 %2 @ %3"),
+                               "%[?%d*%]?%d+",
+                               "(.*)",
+                               "%d%d%d%d%-%d%d%-%d%d %d%d:%d%d:%d%d") .. "$"
+
     for page, items in pairs(highlights) do
         for _, item in ipairs(items) do
             local clipping = {}
@@ -238,8 +257,11 @@ function MyClipping:parseHighlight(highlights, bookmarks, book)
             clipping.chapter = item.chapter
             for _, bookmark in pairs(bookmarks) do
                 if bookmark.datetime == item.datetime and bookmark.text then
-                    local tmp = string.gsub(bookmark.text, "Page %d+ ", "")
+                    local bookmark_quote = bookmark.text:match(pattern)
-                    clipping.text = string.gsub(tmp, " @ %d%d%d%d%-%d%d%-%d%d %d%d:%d%d:%d%d", "")
+                    if bookmark_quote ~= clipping.text and bookmark.text ~= clipping.text then
+                        -- use modified quoted text or entire bookmark text if it's not a match
+                        clipping.note = bookmark_quote or bookmark.text
+                    end
                 end
             end
             if item.text == "" and item.pos0 and item.pos1 and
@@ -282,7 +304,7 @@ function MyClipping:parseHistoryFile(clippings, history_file, doc_file)
             return
         end
         local _, docname = util.splitFilePathName(doc_file)
-        local title, author = self:getTitle(util.splitFileNameSuffix(docname))
+        local title, author = self:getTitle(util.splitFileNameSuffix(docname), doc_file)
         clippings[title] = {
             file = doc_file,
             title = title,
@@ -309,11 +331,31 @@ function MyClipping:parseHistory()
     return clippings
 end
 
+function MyClipping:getProps(file)
+    local document = DocumentRegistry:openDocument(file)
+    local book_props = nil
+    if document then
+        local loaded = true
+        if document.loadDocument then -- CreDocument
+            if not document:loadDocument(false) then -- load only metadata
+                -- failed loading, calling other methods would segfault
+                loaded = false
+            end
+        end
+        if loaded then
+            book_props = document:getProps()
+        end
+        document:close()
+    end
+
+    return book_props
+end
+
 function MyClipping:parseCurrentDoc(view)
     local clippings = {}
     local path = view.document.file
     local _, _, docname = path:find(".*/(.*)")
-    local title, author = self:getTitle(docname)
+    local title, author = self:getTitle(docname, path)
     clippings[title] = {
         file = view.document.file,
         title = title,

plugins/exporter.koplugin/main.lua

@@ -4,11 +4,13 @@ local InfoMessage = require("ui/widget/infomessage")
 local NetworkMgr = require("ui/network/manager")
 local DataStorage = require("datastorage")
 local DocSettings = require("docsettings")
+local InputDialog = require("ui/widget/inputdialog")
 local UIManager = require("ui/uimanager")
 local logger = require("logger")
 local util = require("ffi/util")
 local Device = require("device")
 local JoplinClient = require("JoplinClient")
+local ReadwiseClient = require("ReadwiseClient")
 local T = util.template
 local _ = require("gettext")
 local N_ = _.ngettext
@@ -49,10 +51,12 @@ function Exporter:init()
     self.joplin_port = settings.joplin_port or 41185
     self.joplin_token = settings.joplin_token -- or your token
     self.joplin_notebook_guid = settings.joplin_notebook_guid or nil
+    self.readwise_token = settings.readwise_token or nil
     self.html_export = settings.html_export or false
     self.joplin_export = settings.joplin_export or false
     self.txt_export = settings.txt_export or false
     self.json_export = settings.json_export or false
+    self.readwise_export = settings.readwise_export or false
     --- @todo Is this if block necessary? Nowhere in the code they are assigned both true.
     -- Do they check against external modifications to settings file?
 
@@ -60,11 +64,14 @@ function Exporter:init()
         self.txt_export = false
         self.joplin_export = false
         self.json_export = false
+        self.readwise_export = false
     elseif self.txt_export then
         self.joplin_export = false
         self.json_export = false
+        self.readwise_export = false
     elseif self.json_export then
         self.joplin_export = false
+        self.readwise_export = false
     end
 
     self.parser = MyClipping:new{
@@ -86,7 +93,8 @@ function Exporter:readyToExport()
     return self.html_export ~= false or
             self.txt_export ~= false or
             self.json_export ~= false or
-            self.joplin_export ~= false
+            self.joplin_export ~= false or
+            self.readwise_export ~= false
 end
 
 function Exporter:migrateClippings()
@@ -106,7 +114,6 @@ function Exporter:addToMainMenu(menu_items)
             {
                 text = _("Joplin") ,
                 checked_func = function() return self.joplin_export end,
-                separator = true,
                 sub_item_table ={
                     {
                         text = _("Set Joplin IP and Port"),
@@ -161,16 +168,10 @@ function Exporter:addToMainMenu(menu_items)
                         text = _("Set authorization token"),
                         keep_menu_open = true,
                         callback = function()
-                            local MultiInputDialog = require("ui/widget/multiinputdialog")
                             local auth_dialog
-                            auth_dialog = MultiInputDialog:new{
+                            auth_dialog = InputDialog:new{
                                 title = _("Set authorization token for Joplin"),
-                                fields = {
+                                input = self.joplin_token,
-                                    {
-                                        text = self.joplin_token,
-                                        input_type = "string"
-                                    }
-                                },
                                 buttons = {
                                     {
                                         {
@@ -182,8 +183,7 @@ function Exporter:addToMainMenu(menu_items)
                                         {
                                             text = _("Set token"),
                                             callback = function()
-                                                local auth_field = auth_dialog:getFields()
+                                                self.joplin_token = auth_dialog:getInputText()
-                                                self.joplin_token = auth_field[1]
                                                 self:saveSettings()
                                                 UIManager:close(auth_dialog)
                                             end
@@ -204,6 +204,7 @@ function Exporter:addToMainMenu(menu_items)
                                 self.html_export = false
                                 self.txt_export = false
                                 self.json_export = false
+                                self.readwise_export = false
                             end
                             self:saveSettings()
                         end
@@ -213,7 +214,7 @@ function Exporter:addToMainMenu(menu_items)
                         keep_menu_open = true,
                         callback = function()
                             UIManager:show(InfoMessage:new{
-                                text = T(_([[You can enter your auth token on your computer by saving an empty token. Then quit KOReader, edit the evernote.joplin_token field in %1/settings.reader.lua after creating a backup, and restart KOReader once you're done.
+                                text = T(_([[You can enter your auth token on your computer by saving an empty token. Then quit KOReader, edit the exporter.joplin_token field in %1/settings.reader.lua after creating a backup, and restart KOReader once you're done.
 
 To export to Joplin, you must forward the IP and port used by this plugin to the localhost:port on which Joplin is listening. This can be done with socat or a similar program. For example:
 
@@ -221,7 +222,71 @@ For Windows: netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenpo
 
 For Linux: $socat tcp-listen:41185,reuseaddr,fork tcp:localhost:41184
 
-For more information, please visit https://github.com/koreader/koreader/wiki/Evernote-export.]])
+For more information, please visit https://github.com/koreader/koreader/wiki/Highlight-export.]])
+                            , BD.dirpath(DataStorage:getDataDir()))
+                            })
+                        end
+                    }
+                }
+            },
+            {
+                text = _("Readwise") ,
+                checked_func = function() return self.readwise_export end,
+                separator = true,
+                sub_item_table ={
+                    {
+                        text = _("Set authorization token"),
+                        keep_menu_open = true,
+                        callback = function()
+                            local auth_dialog
+                            auth_dialog = InputDialog:new{
+                                title = _("Set authorization token for Readwise"),
+                                input = self.readwise_token,
+                                buttons = {
+                                    {
+                                        {
+                                            text = _("Cancel"),
+                                            callback = function()
+                                                UIManager:close(auth_dialog)
+                                            end
+                                        },
+                                        {
+                                            text = _("Set token"),
+                                            callback = function()
+                                                self.readwise_token = auth_dialog:getInputText()
+                                                self:saveSettings()
+                                                UIManager:close(auth_dialog)
+                                            end
+                                        }
+                                    }
+                                }
+                            }
+                            UIManager:show(auth_dialog)
+                            auth_dialog:onShowKeyboard()
+                        end
+                    },
+                    {
+                        text = _("Export to Readwise"),
+                        checked_func = function() return self.readwise_export end,
+                        callback = function()
+                            self.readwise_export = not self.readwise_export
+                            if self.readwise_export then
+                                self.html_export = false
+                                self.txt_export = false
+                                self.json_export = false
+                                self.joplin_export = false
+                            end
+                            self:saveSettings()
+                        end
+                    },
+                    {
+                        text = _("Help"),
+                        keep_menu_open = true,
+                        callback = function()
+                            UIManager:show(InfoMessage:new{
+                                text = T(_([[You can enter your auth token on your computer by saving an empty token. Then quit KOReader, edit the exporter.readwise_token field in %1/settings.reader.lua after creating a backup, and restart KOReader once you're done.
+
+For more information, please visit https://github.com/koreader/koreader/wiki/Highlight-export.]])
                             , BD.dirpath(DataStorage:getDataDir()))
                             })
                         end
@@ -270,6 +335,7 @@ For more information, please visit https://github.com/koreader/koreader/wiki/Eve
                         self.txt_export = false
                         self.html_export = false
                         self.joplin_export = false
+                        self.readwise_export = false
                     end
                     self:saveSettings()
                 end
@@ -283,6 +349,7 @@ For more information, please visit https://github.com/koreader/koreader/wiki/Eve
                         self.txt_export = false
                         self.json_export = false
                         self.joplin_export = false
+                        self.readwise_export = false
                     end
                     self:saveSettings()
                 end
@@ -296,6 +363,7 @@ For more information, please visit https://github.com/koreader/koreader/wiki/Eve
                         self.html_export = false
                         self.json_export = false
                         self.joplin_export = false
+                        self.readwise_export = false
                     end
                     self:saveSettings()
                 end,
@@ -325,7 +393,9 @@ function Exporter:saveSettings()
         joplin_port = self.joplin_port,
         joplin_token = self.joplin_token,
         joplin_notebook_guid = self.joplin_notebook_guid,
-        joplin_export = self.joplin_export
+        joplin_export = self.joplin_export,
+        readwise_token = self.readwise_token,
+        readwise_export = self.readwise_export
     }
     G_reader_settings:saveSetting("exporter", settings)
 end
@@ -413,6 +483,7 @@ end
 function Exporter:exportClippings(clippings)
     local exported_stamp
     local joplin_client
+    local readwise_client
     if self.html_export then
         exported_stamp= "html"
     elseif self.json_export then
@@ -433,6 +504,11 @@ function Exporter:exportClippings(clippings)
             self.joplin_notebook_guid = joplin_client:createNotebook(self.notebook_name)
             self:saveSettings()
         end
+    elseif self.readwise_export then
+        exported_stamp = "readwise"
+        readwise_client = ReadwiseClient:new{
+            auth_token = self.readwise_token
+        }
     else
         assert("an exported_stamp is expected for a new export type")
     end
@@ -456,6 +532,8 @@ function Exporter:exportClippings(clippings)
                 ok, err = pcall(self.exportBooknotesToJSON, self, title, booknotes)
             elseif self.joplin_export then
                 ok, err = pcall(self.exportBooknotesToJoplin, self, joplin_client, title, booknotes)
+            elseif self.readwise_export then
+                ok, err = pcall(self.exportBooknotesToReadwise, self, readwise_client, title, booknotes)
             end
             -- Error reporting
             if not ok and err and err:find("Transport not open") then
@@ -539,6 +617,9 @@ function Exporter:exportBooknotesToTXT(title, booknotes)
                 if clipping.text then
                     file:write(clipping.text)
                 end
+                if clipping.note then
+                    file:write("\n---\n" .. clipping.note)
+                end
                 if clipping.image then
                     file:write(_("<An image>"))
                 end
@@ -565,7 +646,11 @@ function Exporter:exportBooknotesToJoplin(client, title, booknotes)
 
         for _, clipping in ipairs(chapter) do
             note = note .. os.date("%Y-%m-%d %H:%M:%S \n", clipping.time)
-            note = note .. clipping.text .. "\n * * *\n"
+            note = note .. clipping.text
+            if clipping.note then
+                note = note .. "\n---\n" .. clipping.note
+            end
+            note = note .. "\n * * *\n"
         end
     end
 
@@ -577,4 +662,8 @@ function Exporter:exportBooknotesToJoplin(client, title, booknotes)
 
 end
 
+function Exporter:exportBooknotesToReadwise(client, title, booknotes)
+    client:createHighlights(booknotes)
+end
+
 return Exporter