averello/koreader-mirror
1
0
Fork 0
mirror of https://github.com/koreader/koreader.git synced 2025-12-13 20:36:53 +01:00
Code Issues Packages Projects Releases Wiki Activity

[plugin] NewsDownloader: add HTTP basic authentication (#14303)

Browse Source 
Closes <https://github.com/koreader/koreader/issues/13300>.
This commit is contained in:
Frans de Jonge
2025-09-14 15:50:49 +02:00
committed by GitHub
parent 8da7774905
commit 105694e117
4 changed files with 89 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
plugins/newsdownloader.koplugin/epubdownloadbackend.lua
View File

@@ -156,8 +156,7 @@ local function build_cookies(cookies)
return s
end
-- Get URL content
local function getUrlContent(url, cookies, timeout, maxtime, add_to_cache)
local function getUrlContent(url, cookies, timeout, maxtime, add_to_cache, extra_headers)
logger.dbg("getUrlContent(", url, ",", cookies, ", ", timeout, ",", maxtime, ",", add_to_cache, ")")
if not timeout then timeout = 10 end
@@ -169,9 +168,15 @@ local function getUrlContent(url, cookies, timeout, maxtime, add_to_cache)
url = url,
method = "GET",
sink = maxtime and socketutil.table_sink(sink) or ltn12.sink.table(sink),
headers = {
["cookie"] = build_cookies(cookies)
}
headers = (function()
local h = { ["cookie"] = build_cookies(cookies) }
if extra_headers then
for k, v in pairs(extra_headers) do
h[k] = v
end
end
return h
end)()
}
logger.dbg("request:", request)
local code, headers, status = socket.skip(1, http.request(request))
@@ -258,9 +263,9 @@ function EpubDownloadBackend:getConnectionCookies(url, credentials)
return cookies
end
function EpubDownloadBackend:getResponseAsString(url, cookies, add_to_cache)
function EpubDownloadBackend:getResponseAsString(url, cookies, add_to_cache, extra_headers)
logger.dbg("EpubDownloadBackend:getResponseAsString(", url, ")")
local success, content = getUrlContent(url, cookies, nil, nil, add_to_cache)
local success, content = getUrlContent(url, cookies, nil, nil, add_to_cache, extra_headers)
if (success) then
return content
else
@@ -276,21 +281,21 @@ function EpubDownloadBackend:resetTrapWidget()
self.trap_widget = nil
end
function EpubDownloadBackend:loadPage(url, cookies)
function EpubDownloadBackend:loadPage(url, cookies, extra_headers)
local completed, success, content
if self.trap_widget then -- if previously set with EpubDownloadBackend:setTrapWidget()
local Trapper = require("ui/trapper")
local timeout, maxtime = 30, 60
-- We use dismissableRunInSubprocess with complex return values:
completed, success, content = Trapper:dismissableRunInSubprocess(function()
return getUrlContent(url, cookies, timeout, maxtime)
return getUrlContent(url, cookies, timeout, maxtime, nil, extra_headers)
end, self.trap_widget)
if not completed then
error(self.dismissed_error_code) -- "Interrupted by user"
end
else
local timeout, maxtime = 10, 60
success, content = getUrlContent(url, cookies, timeout, maxtime)
success, content = getUrlContent(url, cookies, timeout, maxtime, nil, extra_headers)
end
logger.dbg("success:", success, "type(content):", type(content), "content:", type(content) == "string" and content:sub(1, 500), "...")
if not success then

33
plugins/newsdownloader.koplugin/feed_view.lua
View File

@@ -7,7 +7,10 @@ local FeedView = {
DOWNLOAD_FULL_ARTICLE = "download_full_article",
INCLUDE_IMAGES = "include_images",
ENABLE_FILTER = "enable_filter",
FILTER_ELEMENT = "filter_element"
FILTER_ELEMENT = "filter_element",
-- HTTP Basic Auth (optional)
HTTP_AUTH_USERNAME = "http_auth_username",
HTTP_AUTH_PASSWORD = "http_auth_password",
}
function FeedView:getList(feed_config, callback, edit_feed_attribute_callback, delete_feed_callback)
@@ -67,6 +70,9 @@ function FeedView:getItem(id, feed, edit_feed_callback, delete_feed_callback)
local include_images = feed.include_images ~= false
local enable_filter = feed.enable_filter ~= false
local filter_element = feed.filter_element
local http_auth = feed.http_auth or { username = nil, password = nil }
local http_auth_username = http_auth.username
local http_auth_password_set = type(http_auth.password) == "string" and #http_auth.password > 0
local vc = {
{
@@ -136,6 +142,31 @@ function FeedView:getItem(id, feed, edit_feed_callback, delete_feed_callback)
)
end
},
--- HTTP Basic auth fields (optional)
"---",
{
_("HTTP auth username"),
http_auth_username or "",
callback = function()
edit_feed_callback(
id,
FeedView.HTTP_AUTH_USERNAME,
http_auth_username
)
end
},
{
_("HTTP auth password"),
http_auth_password_set and "••••••" or "",
callback = function()
-- Do not prefill the password; let the user type a new value.
edit_feed_callback(
id,
FeedView.HTTP_AUTH_PASSWORD,
""
)
end
},
}
-- We don't always display this. For instance: if a feed

47
plugins/newsdownloader.koplugin/main.lua
View File

@@ -15,6 +15,7 @@ local Persist = require("persist")
local WidgetContainer = require("ui/widget/container/widgetcontainer")
local dateparser = require("lib.dateparser")
local http = require("socket.http")
local mime = require("mime")
local lfs = require("libs/libkoreader-lfs")
local ltn12 = require("ltn12")
local logger = require("logger")
@@ -59,7 +60,8 @@ local function getEmptyFeed()
download_full_article = false,
include_images = true,
enable_filter = false,
filter_element = ""
filter_element = "",
http_auth = { username = nil, password = nil },
}
end
@@ -300,6 +302,7 @@ function NewsDownloader:loadConfigAndProcessFeeds(touchmenu_instance)
local enable_filter = feed.enable_filter or feed.enable_filter == nil
local filter_element = feed.filter_element or feed.filter_element == nil
local credentials = feed.credentials
local http_auth = feed.http_auth
-- Check if the two required attributes are set.
if url and limit then
feed_message = T(_("Processing %1/%2:\n%3"), idx, total_feed_entries, BD.url(url))
@@ -308,6 +311,7 @@ function NewsDownloader:loadConfigAndProcessFeeds(touchmenu_instance)
self:processFeedSource(
url,
credentials,
http_auth,
tonumber(limit),
unsupported_feeds_urls,
download_full_article,
@@ -386,18 +390,23 @@ function NewsDownloader:loadConfigAndProcessFeedsWithUI(touchmenu_instance)
end)
end
function NewsDownloader:processFeedSource(url, credentials, limit, unsupported_feeds_urls, download_full_article, include_images, message, enable_filter, filter_element)
function NewsDownloader:processFeedSource(url, credentials, http_auth, limit, unsupported_feeds_urls, download_full_article, include_images, message, enable_filter, filter_element)
-- Check if we have a cached response first
local cache = DownloadBackend:getCache()
local cached_response = cache:check(url)
local ok, error, response
local cookies = nil
local extra_headers = nil
if credentials ~= nil then
logger.dbg("Auth Cookies from ", credentials.url)
cookies = DownloadBackend:getConnectionCookies(credentials.url, credentials.auth)
end
if http_auth and http_auth.username and http_auth.password then
extra_headers = { ["Authorization"] = "Basic " .. mime.b64((http_auth.username or "") .. ":" .. (http_auth.password or "")) }
end
if cached_response then
logger.dbg("NewsDownloader: Checking cache validity for:", url)
local headers_cached = cached_response.headers
@@ -440,6 +449,9 @@ function NewsDownloader:processFeedSource(url, credentials, limit, unsupported_f
if cookies then
headers["Cookie"] = cookies
end
if extra_headers and extra_headers["Authorization"] then
headers["Authorization"] = extra_headers["Authorization"]
end
local code, response_headers = socket.skip(1, http.request{
url = url,
headers = headers,
@@ -466,7 +478,7 @@ function NewsDownloader:processFeedSource(url, credentials, limit, unsupported_f
if not response then
ok, response = pcall(function()
return DownloadBackend:getResponseAsString(url, cookies, true)
return DownloadBackend:getResponseAsString(url, cookies, true, extra_headers)
end)
end
@@ -534,6 +546,7 @@ function NewsDownloader:processFeedSource(url, credentials, limit, unsupported_f
FEED_TYPE_ATOM,
feeds,
cookies,
http_auth,
limit,
download_full_article,
include_images,
@@ -548,6 +561,7 @@ function NewsDownloader:processFeedSource(url, credentials, limit, unsupported_f
FEED_TYPE_RSS,
feeds,
cookies,
http_auth,
limit,
download_full_article,
include_images,
@@ -598,7 +612,7 @@ function NewsDownloader:deserializeXMLString(xml_str)
return xmlhandler.root
end
function NewsDownloader:processFeed(feed_type, feeds, cookies, limit, download_full_article, include_images, message, enable_filter, filter_element)
function NewsDownloader:processFeed(feed_type, feeds, cookies, http_auth, limit, download_full_article, include_images, message, enable_filter, filter_element)
local feed_title
local feed_item
local total_items
@@ -666,6 +680,7 @@ function NewsDownloader:processFeed(feed_type, feeds, cookies, limit, download_f
self:downloadFeed(
feed,
cookies,
http_auth,
feed_output_dir,
include_images,
article_message,
@@ -709,7 +724,7 @@ local function getTitleWithDate(feed)
return title
end
function NewsDownloader:downloadFeed(feed, cookies, feed_output_dir, include_images, message, enable_filter, filter_element)
function NewsDownloader:downloadFeed(feed, cookies, http_auth, feed_output_dir, include_images, message, enable_filter, filter_element)
local title_with_date = getTitleWithDate(feed)
local news_file_path = ("%s%s%s"):format(feed_output_dir,
title_with_date,
@@ -722,7 +737,11 @@ function NewsDownloader:downloadFeed(feed, cookies, feed_output_dir, include_ima
logger.dbg("NewsDownloader: News file will be stored to :", news_file_path)
local article_message = T(_("%1\n%2"), message, title_with_date)
local link = self.getFeedLink(feed.link)
local html = DownloadBackend:loadPage(link, cookies)
local extra_headers = nil
if http_auth and http_auth.username and http_auth.password then
extra_headers = { ["Authorization"] = "Basic " .. mime.b64((http_auth.username or "") .. ":" .. (http_auth.password or "")) }
end
local html = DownloadBackend:loadPage(link, cookies, extra_headers)
DownloadBackend:createEpub(news_file_path, html, link, include_images, article_message, enable_filter, filter_element)
end
end
@@ -909,7 +928,9 @@ function NewsDownloader:editFeedAttribute(id, key, value)
-- attribute will need and displays the corresponding dialog.
if key == FeedView.URL
or key == FeedView.LIMIT
or key == FeedView.FILTER_ELEMENT then
or key == FeedView.FILTER_ELEMENT
or key == FeedView.HTTP_AUTH_USERNAME
or key == FeedView.HTTP_AUTH_PASSWORD then
local title
local input_type
@@ -926,6 +947,12 @@ function NewsDownloader:editFeedAttribute(id, key, value)
title = _("Edit filter element.")
description = _("Filter based on the given CSS selector. E.g.: name_of_css.element.class")
input_type = "string"
elseif key == FeedView.HTTP_AUTH_USERNAME then
title = _("HTTP auth username")
input_type = "string"
elseif key == FeedView.HTTP_AUTH_PASSWORD then
title = _("HTTP auth password")
input_type = "string"
else
return false
end
@@ -1112,6 +1139,12 @@ function NewsDownloader:updateFeedConfig(id, key, value)
}
)
end
elseif key == FeedView.HTTP_AUTH_USERNAME then
feed.http_auth = feed.http_auth or { username = "", password = "" }
feed.http_auth.username = value or ""
elseif key == FeedView.HTTP_AUTH_PASSWORD then
feed.http_auth = feed.http_auth or { username = "", password = "" }
feed.http_auth.password = value or ""
end
end
-- Now we insert the updated (or newly created) feed into the

4
spec/unit/newsdownloader_spec.lua
View File

@@ -187,7 +187,7 @@ describe("NewsDownloader module", function()
processed = true
end
NewsDownloader:processFeed("rss", feeds, nil, 1, false, false, "Testing", true, nil)
NewsDownloader:processFeed("rss", feeds, nil, nil, 1, false, false, "Testing", true, nil)
assert.is_true(processed)
@@ -230,7 +230,7 @@ describe("NewsDownloader module", function()
processed = true
end
NewsDownloader:processFeed("atom", feeds, nil, 1, false, false, "Testing", true, nil)
NewsDownloader:processFeed("atom", feeds, nil, nil, 1, false, false, "Testing", true, nil)
assert.is_true(processed)