diff --git a/itch_dl/cli.py b/itch_dl/cli.py index bc18b39..d8ddffe 100644 --- a/itch_dl/cli.py +++ b/itch_dl/cli.py @@ -17,43 +17,42 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Bulk download stuff from Itch.io.") parser.add_argument("url_or_path", help="itch.io URL or path to a game jam entries.json file") - parser.add_argument("--api-key", metavar="key", default=None, - help="itch.io API key - https://itch.io/user/settings/api-keys") parser.add_argument("--profile", metavar="profile", default=None, help="configuration profile to load") - parser.add_argument("--urls-only", action="store_true", - help="print scraped game URLs without downloading them") - parser.add_argument("--download-to", metavar="path", - help="directory to save results into (default: current dir)") - parser.add_argument("--parallel", metavar="parallel", type=int, default=1, - help="how many threads to use for downloading games (default: 1)") + + # These args must match config.py -> Settings class. Make sure all defaults here + # evaluate to False, or apply_args_on_settings will override profile settings. + parser.add_argument("--api-key", metavar="key", default=None, + help="itch.io API key - https://itch.io/user/settings/api-keys") + parser.add_argument("--user-agent", metavar="agent", default=None, + help="user agent to use when sending HTTP requests") + parser.add_argument("--download-to", metavar="path", default=None, + help="directory to save results into (default: current working dir)") parser.add_argument("--mirror-web", action="store_true", help="try to fetch assets on game sites") + parser.add_argument("--urls-only", action="store_true", + help="print scraped game URLs without downloading them") + parser.add_argument("--parallel", metavar="parallel", type=int, default=None, + help="how many threads to use for downloading games (default: 1)") parser.add_argument("--filter-files-glob", metavar="glob", default=None, help="filter downloaded files with a shell-style glob/fnmatch (unmatched files are skipped)") parser.add_argument("--filter-files-regex", metavar="regex", default=None, help="filter downloaded files with a Python regex (unmatched files are skipped)") parser.add_argument("--verbose", action="store_true", help="print verbose logs") + return parser.parse_args() # fmt: on -def apply_args_on_settings(args: argparse.Namespace, settings: Settings): - """Apply settings overrides from provided command line arguments, if set.""" - for key in ("api_key", "filter_files_glob", "filter_files_regex"): - value = getattr(args, key) - if value: - setattr(settings, key, value) - - def run() -> int: args = parse_args() if args.verbose: logging.getLogger().setLevel(logging.DEBUG) - settings = load_config(profile=args.profile) - apply_args_on_settings(args, settings) + settings: Settings = load_config(args, profile=args.profile) + if settings.verbose: + logging.getLogger().setLevel(logging.DEBUG) if not settings.api_key: exit( @@ -61,6 +60,9 @@ def run() -> int: "See https://github.com/DragoonAethis/itch-dl/wiki/API-Keys for more info." ) + url_or_path = args.url_or_path + del args # Do not use `args` beyond this point. + # Check API key validity: client = ItchApiClient(settings.api_key, settings.user_agent) profile_req = client.get("/profile") @@ -70,25 +72,24 @@ def run() -> int: "See https://github.com/DragoonAethis/itch-dl/wiki/API-Keys for more info." ) - jobs = get_jobs_for_url_or_path(args.url_or_path, settings) + jobs = get_jobs_for_url_or_path(url_or_path, settings) jobs = list(set(jobs)) # Deduplicate, just in case... logging.info(f"Found {len(jobs)} URL(s).") if len(jobs) == 0: exit("No URLs to download.") - if args.urls_only: + if settings.urls_only: for job in jobs: print(job) return 0 - download_to = os.getcwd() - if args.download_to is not None: - download_to = os.path.normpath(args.download_to) - os.makedirs(download_to, exist_ok=True) + # If the download dir is not set, use the current working dir: + settings.download_to = os.path.normpath(settings.download_to or os.getcwd()) + os.makedirs(settings.download_to, exist_ok=True) # Grab all the download keys (there's no way to fetch them per title...): keys = get_download_keys(client) - return drive_downloads(jobs, download_to, args.mirror_web, settings, keys, parallel=args.parallel) + return drive_downloads(jobs, settings, keys) diff --git a/itch_dl/config.py b/itch_dl/config.py index 0f52aa2..e75416b 100644 --- a/itch_dl/config.py +++ b/itch_dl/config.py @@ -2,6 +2,7 @@ import os import json import logging import platform +import argparse from typing import Optional import requests @@ -9,6 +10,18 @@ from pydantic import BaseModel from . import __version__ +OVERRIDABLE_SETTINGS = ( + "api_key", + "user_agent", + "download_to", + "mirror_web", + "urls_only", + "parallel", + "filter_files_glob", + "filter_files_regex", + "verbose", +) + class Settings(BaseModel): """Available settings for itch-dl. Make sure all of them @@ -17,9 +30,16 @@ class Settings(BaseModel): api_key: Optional[str] = None user_agent: str = f"python-requests/{requests.__version__} itch-dl/{__version__}" + download_to: Optional[str] = None + mirror_web: bool = False + urls_only: bool = False + parallel: int = 1 + filter_files_glob: Optional[str] = None filter_files_regex: Optional[str] = None + verbose: bool = False + def create_and_get_config_path() -> str: """Returns the configuration directory in the appropriate @@ -37,7 +57,7 @@ def create_and_get_config_path() -> str: return os.path.join(base_path, "itch-dl") -def load_config(profile: Optional[str] = None) -> Settings: +def load_config(args: argparse.Namespace, profile: Optional[str] = None) -> Settings: """Loads the configuration from the file system if it exists, the returns a Settings object.""" config_path = create_and_get_config_path() @@ -58,4 +78,13 @@ def load_config(profile: Optional[str] = None) -> Settings: config_data.update(profile_data) - return Settings(**config_data) + # All settings from the base file: + settings = Settings(**config_data) + + # Apply overrides from CLI args: + for key in OVERRIDABLE_SETTINGS: + value = getattr(args, key) + if value: + setattr(settings, key, value) + + return settings diff --git a/itch_dl/downloader.py b/itch_dl/downloader.py index 087e3f2..63f5b5c 100644 --- a/itch_dl/downloader.py +++ b/itch_dl/downloader.py @@ -62,10 +62,7 @@ class GameMetadata(TypedDict, total=False): class GameDownloader: - def __init__(self, download_to: str, mirror_web: bool, settings: Settings, keys: Dict[int, str]): - self.download_to = download_to - self.mirror_web = mirror_web - + def __init__(self, settings: Settings, keys: Dict[int, str]): self.settings = settings self.download_keys = keys self.client = ItchApiClient(settings.api_key, settings.user_agent) @@ -258,7 +255,7 @@ class GameDownloader: author, game = match["author"], match["game"] - download_path = os.path.join(self.download_to, author, game) + download_path = os.path.join(self.settings.download_to, author, game) os.makedirs(download_path, exist_ok=True) paths: Dict[str, str] = {k: os.path.join(download_path, v) for k, v in TARGET_PATHS.items()} @@ -372,7 +369,7 @@ class GameDownloader: logging.warning(f"Game {title} has external download URLs: {external_urls}") # TODO: Mirror JS/CSS assets - if self.mirror_web: + if self.settings.mirror_web: os.makedirs(paths["screenshots"], exist_ok=True) for screenshot in metadata["screenshots"]: if not screenshot: @@ -406,20 +403,17 @@ class GameDownloader: def drive_downloads( jobs: List[str], - download_to: str, - mirror_web: bool, settings: Settings, keys: Dict[int, str], - parallel: int = 1, ): - downloader = GameDownloader(download_to, mirror_web, settings, keys) + downloader = GameDownloader(settings, keys) tqdm_args = { "desc": "Games", "unit": "game", } - if parallel > 1: - results = thread_map(downloader.download, jobs, max_workers=parallel, **tqdm_args) + if settings.parallel > 1: + results = thread_map(downloader.download, jobs, max_workers=settings.parallel, **tqdm_args) else: results = [downloader.download(job) for job in tqdm(jobs, **tqdm_args)]