[plugin] NewsDownloader: add HTTP basic authentication (#14303)

Closes <https://github.com/koreader/koreader/issues/13300>.

plugins/newsdownloader.koplugin/epubdownloadbackend.lua

@@ -156,8 +156,7 @@ local function build_cookies(cookies)
     return s
 end
 
--- Get URL content
+local function getUrlContent(url, cookies, timeout, maxtime, add_to_cache, extra_headers)
-local function getUrlContent(url, cookies, timeout, maxtime, add_to_cache)
     logger.dbg("getUrlContent(", url, ",", cookies, ", ", timeout, ",", maxtime, ",", add_to_cache, ")")
 
     if not timeout then timeout = 10 end
@@ -169,9 +168,15 @@ local function getUrlContent(url, cookies, timeout, maxtime, add_to_cache)
         url     = url,
         method  = "GET",
         sink    = maxtime and socketutil.table_sink(sink) or ltn12.sink.table(sink),
-        headers = {
+        headers = (function()
-            ["cookie"] = build_cookies(cookies)
+            local h = { ["cookie"] = build_cookies(cookies) }
-        }
+            if extra_headers then
+                for k, v in pairs(extra_headers) do
+                    h[k] = v
+                end
+            end
+            return h
+        end)()
     }
     logger.dbg("request:", request)
     local code, headers, status = socket.skip(1, http.request(request))
@@ -258,9 +263,9 @@ function EpubDownloadBackend:getConnectionCookies(url, credentials)
     return cookies
 end
 
-function EpubDownloadBackend:getResponseAsString(url, cookies, add_to_cache)
+function EpubDownloadBackend:getResponseAsString(url, cookies, add_to_cache, extra_headers)
     logger.dbg("EpubDownloadBackend:getResponseAsString(", url, ")")
-    local success, content = getUrlContent(url, cookies, nil, nil, add_to_cache)
+    local success, content = getUrlContent(url, cookies, nil, nil, add_to_cache, extra_headers)
     if (success) then
         return content
     else
@@ -276,21 +281,21 @@ function EpubDownloadBackend:resetTrapWidget()
     self.trap_widget = nil
 end
 
-function EpubDownloadBackend:loadPage(url, cookies)
+function EpubDownloadBackend:loadPage(url, cookies, extra_headers)
     local completed, success, content
     if self.trap_widget then -- if previously set with EpubDownloadBackend:setTrapWidget()
         local Trapper = require("ui/trapper")
         local timeout, maxtime = 30, 60
         -- We use dismissableRunInSubprocess with complex return values:
         completed, success, content = Trapper:dismissableRunInSubprocess(function()
-            return getUrlContent(url, cookies, timeout, maxtime)
+            return getUrlContent(url, cookies, timeout, maxtime, nil, extra_headers)
         end, self.trap_widget)
         if not completed then
             error(self.dismissed_error_code) -- "Interrupted by user"
         end
     else
         local timeout, maxtime = 10, 60
-        success, content = getUrlContent(url, cookies, timeout, maxtime)
+        success, content = getUrlContent(url, cookies, timeout, maxtime, nil, extra_headers)
     end
     logger.dbg("success:", success, "type(content):", type(content), "content:", type(content) == "string" and content:sub(1, 500), "...")
     if not success then

plugins/newsdownloader.koplugin/feed_view.lua

@@ -7,7 +7,10 @@ local FeedView = {
     DOWNLOAD_FULL_ARTICLE = "download_full_article",
     INCLUDE_IMAGES = "include_images",
     ENABLE_FILTER = "enable_filter",
-    FILTER_ELEMENT = "filter_element"
+    FILTER_ELEMENT = "filter_element",
+    -- HTTP Basic Auth (optional)
+    HTTP_AUTH_USERNAME = "http_auth_username",
+    HTTP_AUTH_PASSWORD = "http_auth_password",
 }
 
 function FeedView:getList(feed_config, callback, edit_feed_attribute_callback, delete_feed_callback)
@@ -67,6 +70,9 @@ function FeedView:getItem(id, feed, edit_feed_callback, delete_feed_callback)
     local include_images = feed.include_images ~= false
     local enable_filter = feed.enable_filter ~= false
     local filter_element = feed.filter_element
+    local http_auth = feed.http_auth or { username = nil, password = nil }
+    local http_auth_username = http_auth.username
+    local http_auth_password_set = type(http_auth.password) == "string" and #http_auth.password > 0
 
     local vc = {
         {
@@ -136,6 +142,31 @@ function FeedView:getItem(id, feed, edit_feed_callback, delete_feed_callback)
                 )
             end
         },
+        --- HTTP Basic auth fields (optional)
+        "---",
+        {
+            _("HTTP auth username"),
+            http_auth_username or "",
+            callback = function()
+                edit_feed_callback(
+                    id,
+                    FeedView.HTTP_AUTH_USERNAME,
+                    http_auth_username
+                )
+            end
+        },
+        {
+            _("HTTP auth password"),
+            http_auth_password_set and "••••••" or "",
+            callback = function()
+                -- Do not prefill the password; let the user type a new value.
+                edit_feed_callback(
+                    id,
+                    FeedView.HTTP_AUTH_PASSWORD,
+                    ""
+                )
+            end
+        },
     }
 
     -- We don't always display this. For instance: if a feed

plugins/newsdownloader.koplugin/main.lua

@@ -15,6 +15,7 @@ local Persist = require("persist")
 local WidgetContainer = require("ui/widget/container/widgetcontainer")
 local dateparser = require("lib.dateparser")
 local http = require("socket.http")
+local mime = require("mime")
 local lfs = require("libs/libkoreader-lfs")
 local ltn12 = require("ltn12")
 local logger = require("logger")
@@ -59,7 +60,8 @@ local function getEmptyFeed()
         download_full_article = false,
         include_images = true,
         enable_filter = false,
-        filter_element = ""
+        filter_element = "",
+        http_auth = { username = nil, password = nil },
     }
 end
 
@@ -300,6 +302,7 @@ function NewsDownloader:loadConfigAndProcessFeeds(touchmenu_instance)
         local enable_filter = feed.enable_filter or feed.enable_filter == nil
         local filter_element = feed.filter_element or feed.filter_element == nil
         local credentials = feed.credentials
+        local http_auth = feed.http_auth
         -- Check if the two required attributes are set.
         if url and limit then
             feed_message = T(_("Processing %1/%2:\n%3"), idx, total_feed_entries, BD.url(url))
@@ -308,6 +311,7 @@ function NewsDownloader:loadConfigAndProcessFeeds(touchmenu_instance)
             self:processFeedSource(
                 url,
                 credentials,
+                http_auth,
                 tonumber(limit),
                 unsupported_feeds_urls,
                 download_full_article,
@@ -386,18 +390,23 @@ function NewsDownloader:loadConfigAndProcessFeedsWithUI(touchmenu_instance)
     end)
 end
 
-function NewsDownloader:processFeedSource(url, credentials, limit, unsupported_feeds_urls, download_full_article, include_images, message, enable_filter, filter_element)
+function NewsDownloader:processFeedSource(url, credentials, http_auth, limit, unsupported_feeds_urls, download_full_article, include_images, message, enable_filter, filter_element)
     -- Check if we have a cached response first
     local cache = DownloadBackend:getCache()
     local cached_response = cache:check(url)
     local ok, error, response
 
     local cookies = nil
+    local extra_headers = nil
     if credentials ~= nil then
         logger.dbg("Auth Cookies from ", credentials.url)
         cookies = DownloadBackend:getConnectionCookies(credentials.url, credentials.auth)
     end
 
+    if http_auth and http_auth.username and http_auth.password then
+        extra_headers = { ["Authorization"] = "Basic " .. mime.b64((http_auth.username or "") .. ":" .. (http_auth.password or "")) }
+    end
+
     if cached_response then
         logger.dbg("NewsDownloader: Checking cache validity for:", url)
         local headers_cached = cached_response.headers
@@ -440,6 +449,9 @@ function NewsDownloader:processFeedSource(url, credentials, limit, unsupported_f
                 if cookies then
                     headers["Cookie"] = cookies
                 end
+                if extra_headers and extra_headers["Authorization"] then
+                    headers["Authorization"] = extra_headers["Authorization"]
+                end
                 local code, response_headers = socket.skip(1, http.request{
                     url = url,
                     headers = headers,
@@ -466,7 +478,7 @@ function NewsDownloader:processFeedSource(url, credentials, limit, unsupported_f
 
     if not response then
         ok, response = pcall(function()
-            return DownloadBackend:getResponseAsString(url, cookies, true)
+            return DownloadBackend:getResponseAsString(url, cookies, true, extra_headers)
         end)
     end
 
@@ -534,6 +546,7 @@ function NewsDownloader:processFeedSource(url, credentials, limit, unsupported_f
                     FEED_TYPE_ATOM,
                     feeds,
                     cookies,
+                    http_auth,
                     limit,
                     download_full_article,
                     include_images,
@@ -548,6 +561,7 @@ function NewsDownloader:processFeedSource(url, credentials, limit, unsupported_f
                     FEED_TYPE_RSS,
                     feeds,
                     cookies,
+                    http_auth,
                     limit,
                     download_full_article,
                     include_images,
@@ -598,7 +612,7 @@ function NewsDownloader:deserializeXMLString(xml_str)
     return xmlhandler.root
 end
 
-function NewsDownloader:processFeed(feed_type, feeds, cookies, limit, download_full_article, include_images, message, enable_filter, filter_element)
+function NewsDownloader:processFeed(feed_type, feeds, cookies, http_auth, limit, download_full_article, include_images, message, enable_filter, filter_element)
     local feed_title
     local feed_item
     local total_items
@@ -666,6 +680,7 @@ function NewsDownloader:processFeed(feed_type, feeds, cookies, limit, download_f
             self:downloadFeed(
                 feed,
                 cookies,
+                http_auth,
                 feed_output_dir,
                 include_images,
                 article_message,
@@ -709,7 +724,7 @@ local function getTitleWithDate(feed)
     return title
 end
 
-function NewsDownloader:downloadFeed(feed, cookies, feed_output_dir, include_images, message, enable_filter, filter_element)
+function NewsDownloader:downloadFeed(feed, cookies, http_auth, feed_output_dir, include_images, message, enable_filter, filter_element)
     local title_with_date = getTitleWithDate(feed)
     local news_file_path = ("%s%s%s"):format(feed_output_dir,
                                              title_with_date,
@@ -722,7 +737,11 @@ function NewsDownloader:downloadFeed(feed, cookies, feed_output_dir, include_ima
         logger.dbg("NewsDownloader: News file will be stored to :", news_file_path)
         local article_message = T(_("%1\n%2"), message, title_with_date)
         local link = self.getFeedLink(feed.link)
-        local html = DownloadBackend:loadPage(link, cookies)
+        local extra_headers = nil
+        if http_auth and http_auth.username and http_auth.password then
+            extra_headers = { ["Authorization"] = "Basic " .. mime.b64((http_auth.username or "") .. ":" .. (http_auth.password or "")) }
+        end
+        local html = DownloadBackend:loadPage(link, cookies, extra_headers)
         DownloadBackend:createEpub(news_file_path, html, link, include_images, article_message, enable_filter, filter_element)
     end
 end
@@ -909,7 +928,9 @@ function NewsDownloader:editFeedAttribute(id, key, value)
     -- attribute will need and displays the corresponding dialog.
     if key == FeedView.URL
         or key == FeedView.LIMIT
-        or key == FeedView.FILTER_ELEMENT then
+        or key == FeedView.FILTER_ELEMENT
+        or key == FeedView.HTTP_AUTH_USERNAME
+        or key == FeedView.HTTP_AUTH_PASSWORD then
 
         local title
         local input_type
@@ -926,6 +947,12 @@ function NewsDownloader:editFeedAttribute(id, key, value)
             title = _("Edit filter element.")
             description = _("Filter based on the given CSS selector. E.g.: name_of_css.element.class")
             input_type = "string"
+        elseif key == FeedView.HTTP_AUTH_USERNAME then
+            title = _("HTTP auth username")
+            input_type = "string"
+        elseif key == FeedView.HTTP_AUTH_PASSWORD then
+            title = _("HTTP auth password")
+            input_type = "string"
         else
             return false
         end
@@ -1112,6 +1139,12 @@ function NewsDownloader:updateFeedConfig(id, key, value)
                         }
                     )
                 end
+            elseif key == FeedView.HTTP_AUTH_USERNAME then
+                feed.http_auth = feed.http_auth or { username = "", password = "" }
+                feed.http_auth.username = value or ""
+            elseif key == FeedView.HTTP_AUTH_PASSWORD then
+                feed.http_auth = feed.http_auth or { username = "", password = "" }
+                feed.http_auth.password = value or ""
             end
         end
         -- Now we insert the updated (or newly created) feed into the

spec/unit/newsdownloader_spec.lua

@@ -187,7 +187,7 @@ describe("NewsDownloader module", function()
                 processed = true
             end
 
-            NewsDownloader:processFeed("rss", feeds, nil, 1, false, false, "Testing", true, nil)
+            NewsDownloader:processFeed("rss", feeds, nil, nil, 1, false, false, "Testing", true, nil)
 
             assert.is_true(processed)
 
@@ -230,7 +230,7 @@ describe("NewsDownloader module", function()
                 processed = true
             end
 
-            NewsDownloader:processFeed("atom", feeds, nil, 1, false, false, "Testing", true, nil)
+            NewsDownloader:processFeed("atom", feeds, nil, nil, 1, false, false, "Testing", true, nil)
 
             assert.is_true(processed)