From 1cb57d0be475893e021ed44b8286f2d4f1ef47b4 Mon Sep 17 00:00:00 2001 From: Ryszard Knop Date: Fri, 31 Jan 2025 22:35:37 +0100 Subject: [PATCH] Sync settings and CLI arguments Most CLI arguments can be now saved into the configuration profile JSON. This also cleans up passing around some, but not all args, into various classes and methods, instead of just passing all of settings. --- itch_dl/cli.py | 51 ++++++++++++++++++++++--------------------- itch_dl/config.py | 33 ++++++++++++++++++++++++++-- itch_dl/downloader.py | 18 +++++---------- 3 files changed, 63 insertions(+), 39 deletions(-) diff --git a/itch_dl/cli.py b/itch_dl/cli.py index bc18b39..d8ddffe 100644 --- a/itch_dl/cli.py +++ b/itch_dl/cli.py @@ -17,43 +17,42 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Bulk download stuff from Itch.io.") parser.add_argument("url_or_path", help="itch.io URL or path to a game jam entries.json file") - parser.add_argument("--api-key", metavar="key", default=None, - help="itch.io API key - https://itch.io/user/settings/api-keys") parser.add_argument("--profile", metavar="profile", default=None, help="configuration profile to load") - parser.add_argument("--urls-only", action="store_true", - help="print scraped game URLs without downloading them") - parser.add_argument("--download-to", metavar="path", - help="directory to save results into (default: current dir)") - parser.add_argument("--parallel", metavar="parallel", type=int, default=1, - help="how many threads to use for downloading games (default: 1)") + + # These args must match config.py -> Settings class. Make sure all defaults here + # evaluate to False, or apply_args_on_settings will override profile settings. + parser.add_argument("--api-key", metavar="key", default=None, + help="itch.io API key - https://itch.io/user/settings/api-keys") + parser.add_argument("--user-agent", metavar="agent", default=None, + help="user agent to use when sending HTTP requests") + parser.add_argument("--download-to", metavar="path", default=None, + help="directory to save results into (default: current working dir)") parser.add_argument("--mirror-web", action="store_true", help="try to fetch assets on game sites") + parser.add_argument("--urls-only", action="store_true", + help="print scraped game URLs without downloading them") + parser.add_argument("--parallel", metavar="parallel", type=int, default=None, + help="how many threads to use for downloading games (default: 1)") parser.add_argument("--filter-files-glob", metavar="glob", default=None, help="filter downloaded files with a shell-style glob/fnmatch (unmatched files are skipped)") parser.add_argument("--filter-files-regex", metavar="regex", default=None, help="filter downloaded files with a Python regex (unmatched files are skipped)") parser.add_argument("--verbose", action="store_true", help="print verbose logs") + return parser.parse_args() # fmt: on -def apply_args_on_settings(args: argparse.Namespace, settings: Settings): - """Apply settings overrides from provided command line arguments, if set.""" - for key in ("api_key", "filter_files_glob", "filter_files_regex"): - value = getattr(args, key) - if value: - setattr(settings, key, value) - - def run() -> int: args = parse_args() if args.verbose: logging.getLogger().setLevel(logging.DEBUG) - settings = load_config(profile=args.profile) - apply_args_on_settings(args, settings) + settings: Settings = load_config(args, profile=args.profile) + if settings.verbose: + logging.getLogger().setLevel(logging.DEBUG) if not settings.api_key: exit( @@ -61,6 +60,9 @@ def run() -> int: "See https://github.com/DragoonAethis/itch-dl/wiki/API-Keys for more info." ) + url_or_path = args.url_or_path + del args # Do not use `args` beyond this point. + # Check API key validity: client = ItchApiClient(settings.api_key, settings.user_agent) profile_req = client.get("/profile") @@ -70,25 +72,24 @@ def run() -> int: "See https://github.com/DragoonAethis/itch-dl/wiki/API-Keys for more info." ) - jobs = get_jobs_for_url_or_path(args.url_or_path, settings) + jobs = get_jobs_for_url_or_path(url_or_path, settings) jobs = list(set(jobs)) # Deduplicate, just in case... logging.info(f"Found {len(jobs)} URL(s).") if len(jobs) == 0: exit("No URLs to download.") - if args.urls_only: + if settings.urls_only: for job in jobs: print(job) return 0 - download_to = os.getcwd() - if args.download_to is not None: - download_to = os.path.normpath(args.download_to) - os.makedirs(download_to, exist_ok=True) + # If the download dir is not set, use the current working dir: + settings.download_to = os.path.normpath(settings.download_to or os.getcwd()) + os.makedirs(settings.download_to, exist_ok=True) # Grab all the download keys (there's no way to fetch them per title...): keys = get_download_keys(client) - return drive_downloads(jobs, download_to, args.mirror_web, settings, keys, parallel=args.parallel) + return drive_downloads(jobs, settings, keys) diff --git a/itch_dl/config.py b/itch_dl/config.py index 0f52aa2..e75416b 100644 --- a/itch_dl/config.py +++ b/itch_dl/config.py @@ -2,6 +2,7 @@ import os import json import logging import platform +import argparse from typing import Optional import requests @@ -9,6 +10,18 @@ from pydantic import BaseModel from . import __version__ +OVERRIDABLE_SETTINGS = ( + "api_key", + "user_agent", + "download_to", + "mirror_web", + "urls_only", + "parallel", + "filter_files_glob", + "filter_files_regex", + "verbose", +) + class Settings(BaseModel): """Available settings for itch-dl. Make sure all of them @@ -17,9 +30,16 @@ class Settings(BaseModel): api_key: Optional[str] = None user_agent: str = f"python-requests/{requests.__version__} itch-dl/{__version__}" + download_to: Optional[str] = None + mirror_web: bool = False + urls_only: bool = False + parallel: int = 1 + filter_files_glob: Optional[str] = None filter_files_regex: Optional[str] = None + verbose: bool = False + def create_and_get_config_path() -> str: """Returns the configuration directory in the appropriate @@ -37,7 +57,7 @@ def create_and_get_config_path() -> str: return os.path.join(base_path, "itch-dl") -def load_config(profile: Optional[str] = None) -> Settings: +def load_config(args: argparse.Namespace, profile: Optional[str] = None) -> Settings: """Loads the configuration from the file system if it exists, the returns a Settings object.""" config_path = create_and_get_config_path() @@ -58,4 +78,13 @@ def load_config(profile: Optional[str] = None) -> Settings: config_data.update(profile_data) - return Settings(**config_data) + # All settings from the base file: + settings = Settings(**config_data) + + # Apply overrides from CLI args: + for key in OVERRIDABLE_SETTINGS: + value = getattr(args, key) + if value: + setattr(settings, key, value) + + return settings diff --git a/itch_dl/downloader.py b/itch_dl/downloader.py index 087e3f2..63f5b5c 100644 --- a/itch_dl/downloader.py +++ b/itch_dl/downloader.py @@ -62,10 +62,7 @@ class GameMetadata(TypedDict, total=False): class GameDownloader: - def __init__(self, download_to: str, mirror_web: bool, settings: Settings, keys: Dict[int, str]): - self.download_to = download_to - self.mirror_web = mirror_web - + def __init__(self, settings: Settings, keys: Dict[int, str]): self.settings = settings self.download_keys = keys self.client = ItchApiClient(settings.api_key, settings.user_agent) @@ -258,7 +255,7 @@ class GameDownloader: author, game = match["author"], match["game"] - download_path = os.path.join(self.download_to, author, game) + download_path = os.path.join(self.settings.download_to, author, game) os.makedirs(download_path, exist_ok=True) paths: Dict[str, str] = {k: os.path.join(download_path, v) for k, v in TARGET_PATHS.items()} @@ -372,7 +369,7 @@ class GameDownloader: logging.warning(f"Game {title} has external download URLs: {external_urls}") # TODO: Mirror JS/CSS assets - if self.mirror_web: + if self.settings.mirror_web: os.makedirs(paths["screenshots"], exist_ok=True) for screenshot in metadata["screenshots"]: if not screenshot: @@ -406,20 +403,17 @@ class GameDownloader: def drive_downloads( jobs: List[str], - download_to: str, - mirror_web: bool, settings: Settings, keys: Dict[int, str], - parallel: int = 1, ): - downloader = GameDownloader(download_to, mirror_web, settings, keys) + downloader = GameDownloader(settings, keys) tqdm_args = { "desc": "Games", "unit": "game", } - if parallel > 1: - results = thread_map(downloader.download, jobs, max_workers=parallel, **tqdm_args) + if settings.parallel > 1: + results = thread_map(downloader.download, jobs, max_workers=settings.parallel, **tqdm_args) else: results = [downloader.download(job) for job in tqdm(jobs, **tqdm_args)]