averello/koreader-mirror
1
0
Fork 0
mirror of https://github.com/koreader/koreader.git synced 2025-12-13 20:36:53 +01:00
Code Issues Packages Projects Releases Wiki Activity

[plugin] Exporter: add Readwise.io support (#8548)

Browse Source 
This extends exporter.koplugin with support for [Readwise.io](https://readwise.io), a highlight/notes aggregation service.

[Readwise API documentation](https://readwise.io/api_deets)

This additionally improves the highlight exporter's ability to get the correct title and author of a document, by checking actual metadata instead of inferring from filename. It also includes a modification to the plugin's highlight parsing logic to separate the highlight contents in `.text` from the notes in `.note`. This change actually fixes an existing bug in the HTML export template note.tpl, which has been missing notes because of the lack of the `.note` field.
This commit is contained in:
Dylan Garrett
2021-12-25 01:24:48 -08:00
committed by GitHub
parent a9229ec3aa
commit 3cf3c97e87
3 changed files with 222 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
plugins/exporter.koplugin/ReadwiseClient.lua Normal file
View File

@@ -0,0 +1,70 @@
local http = require("socket.http")
local json = require("json")
local logger = require("logger")
local ltn12 = require("ltn12")
local socket = require("socket")
local socketutil = require("socketutil")
local ReadwiseClient = {
auth_token = ""
}
function ReadwiseClient:new(o)
o = o or {}
self.__index = self
setmetatable(o, self)
return o
end
function ReadwiseClient:_makeRequest(endpoint, method, request_body)
local sink = {}
local request_body_json = json.encode(request_body)
local source = ltn12.source.string(request_body_json)
socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT)
local request = {
url = "https://readwise.io/api/v2/" .. endpoint,
method = method,
sink = ltn12.sink.table(sink),
source = source,
headers = {
["Content-Length"] = #request_body_json,
["Content-Type"] = "application/json",
["Authorization"] = "Token " .. self.auth_token
},
}
local code, _, status = socket.skip(1, http.request(request))
socketutil:reset_timeout()
if code ~= 200 then
logger.warn("ReadwiseClient: HTTP response code <> 200. Response status: ", status)
error("ReadwiseClient: HTTP response code <> 200.")
end
local response = json.decode(sink[1])
return response
end
function ReadwiseClient:createHighlights(booknotes)
local highlights = {}
for _, chapter in ipairs(booknotes) do
for _, clipping in ipairs(chapter) do
local highlight = {
text = clipping.text,
title = booknotes.title,
author = booknotes.author ~= "" and booknotes.author or nil, -- optional author
source_type = "koreader",
category = "books",
note = clipping.note,
location = clipping.page,
location_type = "page",
highlighted_at = os.date("!%Y-%m-%dT%TZ", clipping.time),
}
table.insert(highlights, highlight)
end
end
local result = self:_makeRequest("highlights", "POST", { highlights = highlights })
logger.dbg("ReadwiseClient createHighlights result", result)
end
return ReadwiseClient

52
plugins/exporter.koplugin/clip.lua
View File

@@ -4,6 +4,8 @@ local ReadHistory = require("readhistory")
local logger = require("logger")
local md5 = require("ffi/sha2").md5
local util = require("util")
local _ = require("gettext")
local T = require("ffi/util").template
local MyClipping = {
my_clippings = "/mnt/us/documents/My Clippings.txt",
@@ -98,10 +100,18 @@ local extensions = {
[".doc"] = true,
}
-- first attempt to parse from document metadata
-- remove file extensions added by former KOReader
-- extract author name in "Title(Author)" format
-- extract author name in "Title - Author" format
function MyClipping:getTitle(line)
function MyClipping:getTitle(line, path)
if path then
local props = self:getProps(path)
if props and props.title ~= "" then
return props.title, props.authors or props.author
end
end
line = line:match("^%s*(.-)%s*$") or ""
if extensions[line:sub(-4):lower()] then
line = line:sub(1, -5)
@@ -228,6 +238,15 @@ end
function MyClipping:parseHighlight(highlights, bookmarks, book)
--DEBUG("book", book.file)
-- create a translated pattern that matches bookmark auto-text
-- see ReaderBookmark:getBookmarkAutoText and ReaderBookmark:getBookmarkPageString
--- @todo Remove this once we get rid of auto-text or improve the data model.
local pattern = "^" .. T(_("Page %1 %2 @ %3"),
"%[?%d*%]?%d+",
"(.*)",
"%d%d%d%d%-%d%d%-%d%d %d%d:%d%d:%d%d") .. "$"
for page, items in pairs(highlights) do
for _, item in ipairs(items) do
local clipping = {}
@@ -238,8 +257,11 @@ function MyClipping:parseHighlight(highlights, bookmarks, book)
clipping.chapter = item.chapter
for _, bookmark in pairs(bookmarks) do
if bookmark.datetime == item.datetime and bookmark.text then
local tmp = string.gsub(bookmark.text, "Page %d+ ", "")
clipping.text = string.gsub(tmp, " @ %d%d%d%d%-%d%d%-%d%d %d%d:%d%d:%d%d", "")
local bookmark_quote = bookmark.text:match(pattern)
if bookmark_quote ~= clipping.text and bookmark.text ~= clipping.text then
-- use modified quoted text or entire bookmark text if it's not a match
clipping.note = bookmark_quote or bookmark.text
end
end
end
if item.text == "" and item.pos0 and item.pos1 and
@@ -282,7 +304,7 @@ function MyClipping:parseHistoryFile(clippings, history_file, doc_file)
return
end
local _, docname = util.splitFilePathName(doc_file)
local title, author = self:getTitle(util.splitFileNameSuffix(docname))
local title, author = self:getTitle(util.splitFileNameSuffix(docname), doc_file)
clippings[title] = {
file = doc_file,
title = title,
@@ -309,11 +331,31 @@ function MyClipping:parseHistory()
return clippings
end
function MyClipping:getProps(file)
local document = DocumentRegistry:openDocument(file)
local book_props = nil
if document then
local loaded = true
if document.loadDocument then -- CreDocument
if not document:loadDocument(false) then -- load only metadata
-- failed loading, calling other methods would segfault
loaded = false
end
end
if loaded then
book_props = document:getProps()
end
document:close()
end
return book_props
end
function MyClipping:parseCurrentDoc(view)
local clippings = {}
local path = view.document.file
local _, _, docname = path:find(".*/(.*)")
local title, author = self:getTitle(docname)
local title, author = self:getTitle(docname, path)
clippings[title] = {
file = view.document.file,
title = title,

121
plugins/exporter.koplugin/main.lua
View File

@@ -4,11 +4,13 @@ local InfoMessage = require("ui/widget/infomessage")
local NetworkMgr = require("ui/network/manager")
local DataStorage = require("datastorage")
local DocSettings = require("docsettings")
local InputDialog = require("ui/widget/inputdialog")
local UIManager = require("ui/uimanager")
local logger = require("logger")
local util = require("ffi/util")
local Device = require("device")
local JoplinClient = require("JoplinClient")
local ReadwiseClient = require("ReadwiseClient")
local T = util.template
local _ = require("gettext")
local N_ = _.ngettext
@@ -49,10 +51,12 @@ function Exporter:init()
self.joplin_port = settings.joplin_port or 41185
self.joplin_token = settings.joplin_token -- or your token
self.joplin_notebook_guid = settings.joplin_notebook_guid or nil
self.readwise_token = settings.readwise_token or nil
self.html_export = settings.html_export or false
self.joplin_export = settings.joplin_export or false
self.txt_export = settings.txt_export or false
self.json_export = settings.json_export or false
self.readwise_export = settings.readwise_export or false
--- @todo Is this if block necessary? Nowhere in the code they are assigned both true.
-- Do they check against external modifications to settings file?
@@ -60,11 +64,14 @@ function Exporter:init()
self.txt_export = false
self.joplin_export = false
self.json_export = false
self.readwise_export = false
elseif self.txt_export then
self.joplin_export = false
self.json_export = false
self.readwise_export = false
elseif self.json_export then
self.joplin_export = false
self.readwise_export = false
end
self.parser = MyClipping:new{
@@ -86,7 +93,8 @@ function Exporter:readyToExport()
return self.html_export ~= false or
self.txt_export ~= false or
self.json_export ~= false or
self.joplin_export ~= false
self.joplin_export ~= false or
self.readwise_export ~= false
end
function Exporter:migrateClippings()
@@ -106,7 +114,6 @@ function Exporter:addToMainMenu(menu_items)
{
text = _("Joplin") ,
checked_func = function() return self.joplin_export end,
separator = true,
sub_item_table ={
{
text = _("Set Joplin IP and Port"),
@@ -161,16 +168,10 @@ function Exporter:addToMainMenu(menu_items)
text = _("Set authorization token"),
keep_menu_open = true,
callback = function()
local MultiInputDialog = require("ui/widget/multiinputdialog")
local auth_dialog
auth_dialog = MultiInputDialog:new{
auth_dialog = InputDialog:new{
title = _("Set authorization token for Joplin"),
fields = {
{
text = self.joplin_token,
input_type = "string"
}
},
input = self.joplin_token,
buttons = {
{
{
@@ -182,8 +183,7 @@ function Exporter:addToMainMenu(menu_items)
{
text = _("Set token"),
callback = function()
local auth_field = auth_dialog:getFields()
self.joplin_token = auth_field[1]
self.joplin_token = auth_dialog:getInputText()
self:saveSettings()
UIManager:close(auth_dialog)
end
@@ -204,6 +204,7 @@ function Exporter:addToMainMenu(menu_items)
self.html_export = false
self.txt_export = false
self.json_export = false
self.readwise_export = false
end
self:saveSettings()
end
@@ -213,7 +214,7 @@ function Exporter:addToMainMenu(menu_items)
keep_menu_open = true,
callback = function()
UIManager:show(InfoMessage:new{
text = T(_([[You can enter your auth token on your computer by saving an empty token. Then quit KOReader, edit the evernote.joplin_token field in %1/settings.reader.lua after creating a backup, and restart KOReader once you're done.
text = T(_([[You can enter your auth token on your computer by saving an empty token. Then quit KOReader, edit the exporter.joplin_token field in %1/settings.reader.lua after creating a backup, and restart KOReader once you're done.
To export to Joplin, you must forward the IP and port used by this plugin to the localhost:port on which Joplin is listening. This can be done with socat or a similar program. For example:
@@ -221,7 +222,71 @@ For Windows: netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenpo
For Linux: $socat tcp-listen:41185,reuseaddr,fork tcp:localhost:41184
For more information, please visit https://github.com/koreader/koreader/wiki/Evernote-export.]])
For more information, please visit https://github.com/koreader/koreader/wiki/Highlight-export.]])
, BD.dirpath(DataStorage:getDataDir()))
})
end
}
}
},
{
text = _("Readwise") ,
checked_func = function() return self.readwise_export end,
separator = true,
sub_item_table ={
{
text = _("Set authorization token"),
keep_menu_open = true,
callback = function()
local auth_dialog
auth_dialog = InputDialog:new{
title = _("Set authorization token for Readwise"),
input = self.readwise_token,
buttons = {
{
{
text = _("Cancel"),
callback = function()
UIManager:close(auth_dialog)
end
},
{
text = _("Set token"),
callback = function()
self.readwise_token = auth_dialog:getInputText()
self:saveSettings()
UIManager:close(auth_dialog)
end
}
}
}
}
UIManager:show(auth_dialog)
auth_dialog:onShowKeyboard()
end
},
{
text = _("Export to Readwise"),
checked_func = function() return self.readwise_export end,
callback = function()
self.readwise_export = not self.readwise_export
if self.readwise_export then
self.html_export = false
self.txt_export = false
self.json_export = false
self.joplin_export = false
end
self:saveSettings()
end
},
{
text = _("Help"),
keep_menu_open = true,
callback = function()
UIManager:show(InfoMessage:new{
text = T(_([[You can enter your auth token on your computer by saving an empty token. Then quit KOReader, edit the exporter.readwise_token field in %1/settings.reader.lua after creating a backup, and restart KOReader once you're done.
For more information, please visit https://github.com/koreader/koreader/wiki/Highlight-export.]])
, BD.dirpath(DataStorage:getDataDir()))
})
end
@@ -270,6 +335,7 @@ For more information, please visit https://github.com/koreader/koreader/wiki/Eve
self.txt_export = false
self.html_export = false
self.joplin_export = false
self.readwise_export = false
end
self:saveSettings()
end
@@ -283,6 +349,7 @@ For more information, please visit https://github.com/koreader/koreader/wiki/Eve
self.txt_export = false
self.json_export = false
self.joplin_export = false
self.readwise_export = false
end
self:saveSettings()
end
@@ -296,6 +363,7 @@ For more information, please visit https://github.com/koreader/koreader/wiki/Eve
self.html_export = false
self.json_export = false
self.joplin_export = false
self.readwise_export = false
end
self:saveSettings()
end,
@@ -325,7 +393,9 @@ function Exporter:saveSettings()
joplin_port = self.joplin_port,
joplin_token = self.joplin_token,
joplin_notebook_guid = self.joplin_notebook_guid,
joplin_export = self.joplin_export
joplin_export = self.joplin_export,
readwise_token = self.readwise_token,
readwise_export = self.readwise_export
}
G_reader_settings:saveSetting("exporter", settings)
end
@@ -413,6 +483,7 @@ end
function Exporter:exportClippings(clippings)
local exported_stamp
local joplin_client
local readwise_client
if self.html_export then
exported_stamp= "html"
elseif self.json_export then
@@ -433,6 +504,11 @@ function Exporter:exportClippings(clippings)
self.joplin_notebook_guid = joplin_client:createNotebook(self.notebook_name)
self:saveSettings()
end
elseif self.readwise_export then
exported_stamp = "readwise"
readwise_client = ReadwiseClient:new{
auth_token = self.readwise_token
}
else
assert("an exported_stamp is expected for a new export type")
end
@@ -456,6 +532,8 @@ function Exporter:exportClippings(clippings)
ok, err = pcall(self.exportBooknotesToJSON, self, title, booknotes)
elseif self.joplin_export then
ok, err = pcall(self.exportBooknotesToJoplin, self, joplin_client, title, booknotes)
elseif self.readwise_export then
ok, err = pcall(self.exportBooknotesToReadwise, self, readwise_client, title, booknotes)
end
-- Error reporting
if not ok and err and err:find("Transport not open") then
@@ -539,6 +617,9 @@ function Exporter:exportBooknotesToTXT(title, booknotes)
if clipping.text then
file:write(clipping.text)
end
if clipping.note then
file:write("\n---\n" .. clipping.note)
end
if clipping.image then
file:write(_("<An image>"))
end
@@ -565,7 +646,11 @@ function Exporter:exportBooknotesToJoplin(client, title, booknotes)
for _, clipping in ipairs(chapter) do
note = note .. os.date("%Y-%m-%d %H:%M:%S \n", clipping.time)
note = note .. clipping.text .. "\n * * *\n"
note = note .. clipping.text
if clipping.note then
note = note .. "\n---\n" .. clipping.note
end
note = note .. "\n * * *\n"
end
end
@@ -577,4 +662,8 @@ function Exporter:exportBooknotesToJoplin(client, title, booknotes)
end
function Exporter:exportBooknotesToReadwise(client, title, booknotes)
client:createHighlights(booknotes)
end
return Exporter