From 105694e1178ca9e13778367d268865879d264531 Mon Sep 17 00:00:00 2001 From: Frans de Jonge Date: Sun, 14 Sep 2025 15:50:49 +0200 Subject: [PATCH] [plugin] NewsDownloader: add HTTP basic authentication (#14303) Closes . --- .../epubdownloadbackend.lua | 25 ++++++---- plugins/newsdownloader.koplugin/feed_view.lua | 33 ++++++++++++- plugins/newsdownloader.koplugin/main.lua | 47 ++++++++++++++++--- spec/unit/newsdownloader_spec.lua | 4 +- 4 files changed, 89 insertions(+), 20 deletions(-) diff --git a/plugins/newsdownloader.koplugin/epubdownloadbackend.lua b/plugins/newsdownloader.koplugin/epubdownloadbackend.lua index f3b714b43..0d3ddb617 100644 --- a/plugins/newsdownloader.koplugin/epubdownloadbackend.lua +++ b/plugins/newsdownloader.koplugin/epubdownloadbackend.lua @@ -156,8 +156,7 @@ local function build_cookies(cookies) return s end --- Get URL content -local function getUrlContent(url, cookies, timeout, maxtime, add_to_cache) +local function getUrlContent(url, cookies, timeout, maxtime, add_to_cache, extra_headers) logger.dbg("getUrlContent(", url, ",", cookies, ", ", timeout, ",", maxtime, ",", add_to_cache, ")") if not timeout then timeout = 10 end @@ -169,9 +168,15 @@ local function getUrlContent(url, cookies, timeout, maxtime, add_to_cache) url = url, method = "GET", sink = maxtime and socketutil.table_sink(sink) or ltn12.sink.table(sink), - headers = { - ["cookie"] = build_cookies(cookies) - } + headers = (function() + local h = { ["cookie"] = build_cookies(cookies) } + if extra_headers then + for k, v in pairs(extra_headers) do + h[k] = v + end + end + return h + end)() } logger.dbg("request:", request) local code, headers, status = socket.skip(1, http.request(request)) @@ -258,9 +263,9 @@ function EpubDownloadBackend:getConnectionCookies(url, credentials) return cookies end -function EpubDownloadBackend:getResponseAsString(url, cookies, add_to_cache) +function EpubDownloadBackend:getResponseAsString(url, cookies, add_to_cache, extra_headers) logger.dbg("EpubDownloadBackend:getResponseAsString(", url, ")") - local success, content = getUrlContent(url, cookies, nil, nil, add_to_cache) + local success, content = getUrlContent(url, cookies, nil, nil, add_to_cache, extra_headers) if (success) then return content else @@ -276,21 +281,21 @@ function EpubDownloadBackend:resetTrapWidget() self.trap_widget = nil end -function EpubDownloadBackend:loadPage(url, cookies) +function EpubDownloadBackend:loadPage(url, cookies, extra_headers) local completed, success, content if self.trap_widget then -- if previously set with EpubDownloadBackend:setTrapWidget() local Trapper = require("ui/trapper") local timeout, maxtime = 30, 60 -- We use dismissableRunInSubprocess with complex return values: completed, success, content = Trapper:dismissableRunInSubprocess(function() - return getUrlContent(url, cookies, timeout, maxtime) + return getUrlContent(url, cookies, timeout, maxtime, nil, extra_headers) end, self.trap_widget) if not completed then error(self.dismissed_error_code) -- "Interrupted by user" end else local timeout, maxtime = 10, 60 - success, content = getUrlContent(url, cookies, timeout, maxtime) + success, content = getUrlContent(url, cookies, timeout, maxtime, nil, extra_headers) end logger.dbg("success:", success, "type(content):", type(content), "content:", type(content) == "string" and content:sub(1, 500), "...") if not success then diff --git a/plugins/newsdownloader.koplugin/feed_view.lua b/plugins/newsdownloader.koplugin/feed_view.lua index 29f57046f..63374dba2 100644 --- a/plugins/newsdownloader.koplugin/feed_view.lua +++ b/plugins/newsdownloader.koplugin/feed_view.lua @@ -7,7 +7,10 @@ local FeedView = { DOWNLOAD_FULL_ARTICLE = "download_full_article", INCLUDE_IMAGES = "include_images", ENABLE_FILTER = "enable_filter", - FILTER_ELEMENT = "filter_element" + FILTER_ELEMENT = "filter_element", + -- HTTP Basic Auth (optional) + HTTP_AUTH_USERNAME = "http_auth_username", + HTTP_AUTH_PASSWORD = "http_auth_password", } function FeedView:getList(feed_config, callback, edit_feed_attribute_callback, delete_feed_callback) @@ -67,6 +70,9 @@ function FeedView:getItem(id, feed, edit_feed_callback, delete_feed_callback) local include_images = feed.include_images ~= false local enable_filter = feed.enable_filter ~= false local filter_element = feed.filter_element + local http_auth = feed.http_auth or { username = nil, password = nil } + local http_auth_username = http_auth.username + local http_auth_password_set = type(http_auth.password) == "string" and #http_auth.password > 0 local vc = { { @@ -136,6 +142,31 @@ function FeedView:getItem(id, feed, edit_feed_callback, delete_feed_callback) ) end }, + --- HTTP Basic auth fields (optional) + "---", + { + _("HTTP auth username"), + http_auth_username or "", + callback = function() + edit_feed_callback( + id, + FeedView.HTTP_AUTH_USERNAME, + http_auth_username + ) + end + }, + { + _("HTTP auth password"), + http_auth_password_set and "••••••" or "", + callback = function() + -- Do not prefill the password; let the user type a new value. + edit_feed_callback( + id, + FeedView.HTTP_AUTH_PASSWORD, + "" + ) + end + }, } -- We don't always display this. For instance: if a feed diff --git a/plugins/newsdownloader.koplugin/main.lua b/plugins/newsdownloader.koplugin/main.lua index 313d02015..a9bd75164 100644 --- a/plugins/newsdownloader.koplugin/main.lua +++ b/plugins/newsdownloader.koplugin/main.lua @@ -15,6 +15,7 @@ local Persist = require("persist") local WidgetContainer = require("ui/widget/container/widgetcontainer") local dateparser = require("lib.dateparser") local http = require("socket.http") +local mime = require("mime") local lfs = require("libs/libkoreader-lfs") local ltn12 = require("ltn12") local logger = require("logger") @@ -59,7 +60,8 @@ local function getEmptyFeed() download_full_article = false, include_images = true, enable_filter = false, - filter_element = "" + filter_element = "", + http_auth = { username = nil, password = nil }, } end @@ -300,6 +302,7 @@ function NewsDownloader:loadConfigAndProcessFeeds(touchmenu_instance) local enable_filter = feed.enable_filter or feed.enable_filter == nil local filter_element = feed.filter_element or feed.filter_element == nil local credentials = feed.credentials + local http_auth = feed.http_auth -- Check if the two required attributes are set. if url and limit then feed_message = T(_("Processing %1/%2:\n%3"), idx, total_feed_entries, BD.url(url)) @@ -308,6 +311,7 @@ function NewsDownloader:loadConfigAndProcessFeeds(touchmenu_instance) self:processFeedSource( url, credentials, + http_auth, tonumber(limit), unsupported_feeds_urls, download_full_article, @@ -386,18 +390,23 @@ function NewsDownloader:loadConfigAndProcessFeedsWithUI(touchmenu_instance) end) end -function NewsDownloader:processFeedSource(url, credentials, limit, unsupported_feeds_urls, download_full_article, include_images, message, enable_filter, filter_element) +function NewsDownloader:processFeedSource(url, credentials, http_auth, limit, unsupported_feeds_urls, download_full_article, include_images, message, enable_filter, filter_element) -- Check if we have a cached response first local cache = DownloadBackend:getCache() local cached_response = cache:check(url) local ok, error, response local cookies = nil + local extra_headers = nil if credentials ~= nil then logger.dbg("Auth Cookies from ", credentials.url) cookies = DownloadBackend:getConnectionCookies(credentials.url, credentials.auth) end + if http_auth and http_auth.username and http_auth.password then + extra_headers = { ["Authorization"] = "Basic " .. mime.b64((http_auth.username or "") .. ":" .. (http_auth.password or "")) } + end + if cached_response then logger.dbg("NewsDownloader: Checking cache validity for:", url) local headers_cached = cached_response.headers @@ -440,6 +449,9 @@ function NewsDownloader:processFeedSource(url, credentials, limit, unsupported_f if cookies then headers["Cookie"] = cookies end + if extra_headers and extra_headers["Authorization"] then + headers["Authorization"] = extra_headers["Authorization"] + end local code, response_headers = socket.skip(1, http.request{ url = url, headers = headers, @@ -466,7 +478,7 @@ function NewsDownloader:processFeedSource(url, credentials, limit, unsupported_f if not response then ok, response = pcall(function() - return DownloadBackend:getResponseAsString(url, cookies, true) + return DownloadBackend:getResponseAsString(url, cookies, true, extra_headers) end) end @@ -534,6 +546,7 @@ function NewsDownloader:processFeedSource(url, credentials, limit, unsupported_f FEED_TYPE_ATOM, feeds, cookies, + http_auth, limit, download_full_article, include_images, @@ -548,6 +561,7 @@ function NewsDownloader:processFeedSource(url, credentials, limit, unsupported_f FEED_TYPE_RSS, feeds, cookies, + http_auth, limit, download_full_article, include_images, @@ -598,7 +612,7 @@ function NewsDownloader:deserializeXMLString(xml_str) return xmlhandler.root end -function NewsDownloader:processFeed(feed_type, feeds, cookies, limit, download_full_article, include_images, message, enable_filter, filter_element) +function NewsDownloader:processFeed(feed_type, feeds, cookies, http_auth, limit, download_full_article, include_images, message, enable_filter, filter_element) local feed_title local feed_item local total_items @@ -666,6 +680,7 @@ function NewsDownloader:processFeed(feed_type, feeds, cookies, limit, download_f self:downloadFeed( feed, cookies, + http_auth, feed_output_dir, include_images, article_message, @@ -709,7 +724,7 @@ local function getTitleWithDate(feed) return title end -function NewsDownloader:downloadFeed(feed, cookies, feed_output_dir, include_images, message, enable_filter, filter_element) +function NewsDownloader:downloadFeed(feed, cookies, http_auth, feed_output_dir, include_images, message, enable_filter, filter_element) local title_with_date = getTitleWithDate(feed) local news_file_path = ("%s%s%s"):format(feed_output_dir, title_with_date, @@ -722,7 +737,11 @@ function NewsDownloader:downloadFeed(feed, cookies, feed_output_dir, include_ima logger.dbg("NewsDownloader: News file will be stored to :", news_file_path) local article_message = T(_("%1\n%2"), message, title_with_date) local link = self.getFeedLink(feed.link) - local html = DownloadBackend:loadPage(link, cookies) + local extra_headers = nil + if http_auth and http_auth.username and http_auth.password then + extra_headers = { ["Authorization"] = "Basic " .. mime.b64((http_auth.username or "") .. ":" .. (http_auth.password or "")) } + end + local html = DownloadBackend:loadPage(link, cookies, extra_headers) DownloadBackend:createEpub(news_file_path, html, link, include_images, article_message, enable_filter, filter_element) end end @@ -909,7 +928,9 @@ function NewsDownloader:editFeedAttribute(id, key, value) -- attribute will need and displays the corresponding dialog. if key == FeedView.URL or key == FeedView.LIMIT - or key == FeedView.FILTER_ELEMENT then + or key == FeedView.FILTER_ELEMENT + or key == FeedView.HTTP_AUTH_USERNAME + or key == FeedView.HTTP_AUTH_PASSWORD then local title local input_type @@ -926,6 +947,12 @@ function NewsDownloader:editFeedAttribute(id, key, value) title = _("Edit filter element.") description = _("Filter based on the given CSS selector. E.g.: name_of_css.element.class") input_type = "string" + elseif key == FeedView.HTTP_AUTH_USERNAME then + title = _("HTTP auth username") + input_type = "string" + elseif key == FeedView.HTTP_AUTH_PASSWORD then + title = _("HTTP auth password") + input_type = "string" else return false end @@ -1112,6 +1139,12 @@ function NewsDownloader:updateFeedConfig(id, key, value) } ) end + elseif key == FeedView.HTTP_AUTH_USERNAME then + feed.http_auth = feed.http_auth or { username = "", password = "" } + feed.http_auth.username = value or "" + elseif key == FeedView.HTTP_AUTH_PASSWORD then + feed.http_auth = feed.http_auth or { username = "", password = "" } + feed.http_auth.password = value or "" end end -- Now we insert the updated (or newly created) feed into the diff --git a/spec/unit/newsdownloader_spec.lua b/spec/unit/newsdownloader_spec.lua index d98f2b4a0..d6b340526 100644 --- a/spec/unit/newsdownloader_spec.lua +++ b/spec/unit/newsdownloader_spec.lua @@ -187,7 +187,7 @@ describe("NewsDownloader module", function() processed = true end - NewsDownloader:processFeed("rss", feeds, nil, 1, false, false, "Testing", true, nil) + NewsDownloader:processFeed("rss", feeds, nil, nil, 1, false, false, "Testing", true, nil) assert.is_true(processed) @@ -230,7 +230,7 @@ describe("NewsDownloader module", function() processed = true end - NewsDownloader:processFeed("atom", feeds, nil, 1, false, false, "Testing", true, nil) + NewsDownloader:processFeed("atom", feeds, nil, nil, 1, false, false, "Testing", true, nil) assert.is_true(processed)