Pass User-Agent to all web requests

Author: benoit74

Dockerfile

@@ -43,6 +43,7 @@ COPY --from=zimui /src/dist /src/zimui
 
 ENV MINDTOUCH_ZIMUI_DIST=/src/zimui \
     MINDTOUCH_OUTPUT=/output \
-    MINDTOUCH_TMP=/tmp
+    MINDTOUCH_TMP=/tmp\
+    MINDTOUCH_CONTACT_INFO=https://www.kiwix.org
 
 CMD ["mindtouch2zim", "--help"]

scraper/src/mindtouch2zim/asset.py

@@ -8,13 +8,13 @@
 from pif import get_public_ip
 from PIL import Image
 from requests.exceptions import RequestException
-from zimscraperlib.download import stream_file
 from zimscraperlib.image.optimization import optimize_webp
 from zimscraperlib.image.presets import WebpMedium
 from zimscraperlib.rewriting.url_rewriting import HttpUrl, ZimPath
 from zimscraperlib.zim import Creator
 
 from mindtouch2zim.constants import KNOWN_BAD_ASSETS_REGEX, logger, web_session
+from mindtouch2zim.download import stream_file
 from mindtouch2zim.errors import KnownBadAssetFailedError
 from mindtouch2zim.utils import backoff_hdlr
 
@@ -241,20 +241,22 @@ def get_asset_content(
     ) -> BytesIO:
         """Download of a given asset, optimize if needed, or download from S3 cache"""
 
-        if not always_fetch_online:
-            header_data = self._get_header_data_for(asset_url)
-            if header_data.content_type:
-                mime_type = header_data.content_type.split(";")[0].strip()
-                if mime_type in SUPPORTED_IMAGE_MIME_TYPES:
-                    return self._get_image_content(
-                        asset_path=asset_path,
-                        asset_url=asset_url,
-                        header_data=header_data,
-                    )
-                else:
-                    logger.debug(f"Not optimizing, unsupported mime type: {mime_type}")
-
         try:
+            if not always_fetch_online:
+                header_data = self._get_header_data_for(asset_url)
+                if header_data.content_type:
+                    mime_type = header_data.content_type.split(";")[0].strip()
+                    if mime_type in SUPPORTED_IMAGE_MIME_TYPES:
+                        return self._get_image_content(
+                            asset_path=asset_path,
+                            asset_url=asset_url,
+                            header_data=header_data,
+                        )
+                    else:
+                        logger.debug(
+                            f"Not optimizing, unsupported mime type: {mime_type}"
+                        )
+
             return self._download_from_online(asset_url=asset_url)
         except RequestException as exc:
             # check if the failing download match known bad assets regex early, and if

scraper/src/mindtouch2zim/constants.py

@@ -22,3 +22,6 @@
 logger = getLogger(NAME, level=logging.DEBUG, log_format=DEFAULT_FORMAT_WITH_THREADS)
 
 web_session = get_session()
+
+# info passed in User-Agent header of web requests
+CONTACT_INFO = "https://www.kiwix.org"

scraper/src/mindtouch2zim/download.py

@@ -0,0 +1,49 @@
+import pathlib
+from typing import IO
+
+import requests
+import requests.structures
+import urllib3
+import zimscraperlib.__about__
+import zimscraperlib.constants
+from zimscraperlib.download import stream_file as stream_file_orig
+
+from mindtouch2zim.constants import CONTACT_INFO, NAME, VERSION
+
+
+def get_user_agent() -> str:
+    return (
+        f"{NAME}/{VERSION} ({CONTACT_INFO}) "
+        f"{zimscraperlib.constants.NAME}/{zimscraperlib.__about__.__version__} "
+        f"requests/{requests.__version__} "
+        f"urllib3/{urllib3._version.__version__}"
+    )
+
+
+def stream_file(
+    url: str,
+    fpath: pathlib.Path | None = None,
+    byte_stream: IO[bytes] | None = None,
+    block_size: int | None = 1024,
+    proxies: dict[str, str] | None = None,
+    max_retries: int | None = 5,
+    headers: dict[str, str] | None = None,
+    session: requests.Session | None = None,
+    *,
+    only_first_block: bool | None = False,
+) -> tuple[int, requests.structures.CaseInsensitiveDict[str]]:
+    if headers:
+        headers["User-Agent"] = get_user_agent()
+    else:
+        headers = {"User-Agent": get_user_agent()}
+    return stream_file_orig(
+        url=url,
+        fpath=fpath,
+        byte_stream=byte_stream,
+        block_size=block_size,
+        proxies=proxies,
+        max_retries=max_retries,
+        headers=headers,
+        session=session,
+        only_first_block=only_first_block,
+    )

scraper/src/mindtouch2zim/entrypoint.py

@@ -242,6 +242,12 @@ def main(tmpdir: str) -> None:
         dest="bad_assets_threshold",
     )
 
+    parser.add_argument(
+        "--contact-info",
+        help="Contact information to pass in User-Agent headers",
+        default=os.getenv("MINDTOUCH_CONTACT_INFO", "https://www.kiwix.org"),
+    )
+
     args = parser.parse_args()
 
     logger.setLevel(level=logging.DEBUG if args.debug else logging.INFO)
@@ -281,6 +287,7 @@ def main(tmpdir: str) -> None:
             assets_workers=args.assets_workers,
             bad_assets_regex=args.bad_assets_regex,
             bad_assets_threshold=args.bad_assets_threshold,
+            contact_info=args.contact_info,
         ).run()
     except SystemExit:
         logger.error("Generation failed, exiting")

scraper/src/mindtouch2zim/processor.py

@@ -13,7 +13,6 @@
 from requests import RequestException
 from requests.exceptions import HTTPError
 from schedule import every, run_pending
-from zimscraperlib.download import stream_file
 from zimscraperlib.image import convert_image, resize_image
 from zimscraperlib.image.conversion import convert_svg2png
 from zimscraperlib.image.probing import format_for
@@ -47,6 +46,7 @@
     logger,
     web_session,
 )
+from mindtouch2zim.download import get_user_agent, stream_file
 from mindtouch2zim.errors import NoIllustrationFoundError
 from mindtouch2zim.html import get_text
 from mindtouch2zim.html_rewriting import HtmlUrlsRewriter
@@ -146,6 +146,7 @@ def __init__(
         bad_assets_regex: str | None,
         bad_assets_threshold: int,
         assets_workers: int,
+        contact_info: str,
         *,
         overwrite_existing_zim: bool,
     ) -> None:
@@ -176,6 +177,8 @@ def __init__(
         self.asset_executor = Parallel(
             n_jobs=assets_workers, return_as="generator_unordered", backend="threading"
         )
+        mindtouch2zim.constants.CONTACT_INFO = contact_info
+        logger.debug(f"User-Agent: { get_user_agent()}")
 
         self.stats_items_done = 0
         # we add 1 more items to process so that progress is not 100% at the beginning

scraper/tests-integration/test_client.py

@@ -14,6 +14,7 @@
     MindtouchClient,
     MindtouchHome,
 )
+from mindtouch2zim.download import get_user_agent
 
 
 @pytest.fixture(scope="module")
@@ -119,7 +120,11 @@ def test_get_home_image_url(home: MindtouchHome):
 def test_get_home_image_size(home: MindtouchHome, home_png_size: int):
     """Ensures image url is retrievable"""
     dst = io.BytesIO()
-    stream_file(home.welcome_image_url, byte_stream=dst)
+    stream_file(
+        home.welcome_image_url,
+        byte_stream=dst,
+        headers={"User-Agent": get_user_agent()},
+    )
     assert format_for(dst, from_suffix=False) == "PNG"
     assert len(dst.getvalue()) == home_png_size