Sync settings and CLI arguments

Most CLI arguments can be now saved into the configuration profile JSON.
This also cleans up passing around some, but not all args, into various
classes and methods, instead of just passing all of settings.

itch_dl/cli.py

@@ -17,43 +17,42 @@ def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Bulk download stuff from Itch.io.")
     parser.add_argument("url_or_path",
                         help="itch.io URL or path to a game jam entries.json file")
-    parser.add_argument("--api-key", metavar="key", default=None,
-                        help="itch.io API key - https://itch.io/user/settings/api-keys")
     parser.add_argument("--profile", metavar="profile", default=None,
                         help="configuration profile to load")
-    parser.add_argument("--urls-only", action="store_true",
+
-                        help="print scraped game URLs without downloading them")
+    # These args must match config.py -> Settings class. Make sure all defaults here
-    parser.add_argument("--download-to", metavar="path",
+    # evaluate to False, or apply_args_on_settings will override profile settings.
-                        help="directory to save results into (default: current dir)")
+    parser.add_argument("--api-key", metavar="key", default=None,
-    parser.add_argument("--parallel", metavar="parallel", type=int, default=1,
+                        help="itch.io API key - https://itch.io/user/settings/api-keys")
-                        help="how many threads to use for downloading games (default: 1)")
+    parser.add_argument("--user-agent", metavar="agent", default=None,
+                        help="user agent to use when sending HTTP requests")
+    parser.add_argument("--download-to", metavar="path", default=None,
+                        help="directory to save results into (default: current working dir)")
     parser.add_argument("--mirror-web", action="store_true",
                         help="try to fetch assets on game sites")
+    parser.add_argument("--urls-only", action="store_true",
+                        help="print scraped game URLs without downloading them")
+    parser.add_argument("--parallel", metavar="parallel", type=int, default=None,
+                        help="how many threads to use for downloading games (default: 1)")
     parser.add_argument("--filter-files-glob", metavar="glob", default=None,
                         help="filter downloaded files with a shell-style glob/fnmatch (unmatched files are skipped)")
     parser.add_argument("--filter-files-regex", metavar="regex", default=None,
                         help="filter downloaded files with a Python regex (unmatched files are skipped)")
     parser.add_argument("--verbose", action="store_true",
                         help="print verbose logs")
+
     return parser.parse_args()
     # fmt: on
 
 
-def apply_args_on_settings(args: argparse.Namespace, settings: Settings):
-    """Apply settings overrides from provided command line arguments, if set."""
-    for key in ("api_key", "filter_files_glob", "filter_files_regex"):
-        value = getattr(args, key)
-        if value:
-            setattr(settings, key, value)
-
-
 def run() -> int:
     args = parse_args()
     if args.verbose:
         logging.getLogger().setLevel(logging.DEBUG)
 
-    settings = load_config(profile=args.profile)
+    settings: Settings = load_config(args, profile=args.profile)
-    apply_args_on_settings(args, settings)
+    if settings.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
 
     if not settings.api_key:
         exit(
@@ -61,6 +60,9 @@ def run() -> int:
             "See https://github.com/DragoonAethis/itch-dl/wiki/API-Keys for more info."
         )
 
+    url_or_path = args.url_or_path
+    del args  # Do not use `args` beyond this point.
+
     # Check API key validity:
     client = ItchApiClient(settings.api_key, settings.user_agent)
     profile_req = client.get("/profile")
@@ -70,25 +72,24 @@ def run() -> int:
             "See https://github.com/DragoonAethis/itch-dl/wiki/API-Keys for more info."
         )
 
-    jobs = get_jobs_for_url_or_path(args.url_or_path, settings)
+    jobs = get_jobs_for_url_or_path(url_or_path, settings)
     jobs = list(set(jobs))  # Deduplicate, just in case...
     logging.info(f"Found {len(jobs)} URL(s).")
 
     if len(jobs) == 0:
         exit("No URLs to download.")
 
-    if args.urls_only:
+    if settings.urls_only:
         for job in jobs:
             print(job)
 
         return 0
 
-    download_to = os.getcwd()
+    # If the download dir is not set, use the current working dir:
-    if args.download_to is not None:
+    settings.download_to = os.path.normpath(settings.download_to or os.getcwd())
-        download_to = os.path.normpath(args.download_to)
+    os.makedirs(settings.download_to, exist_ok=True)
-        os.makedirs(download_to, exist_ok=True)
 
     # Grab all the download keys (there's no way to fetch them per title...):
     keys = get_download_keys(client)
 
-    return drive_downloads(jobs, download_to, args.mirror_web, settings, keys, parallel=args.parallel)
+    return drive_downloads(jobs, settings, keys)

itch_dl/config.py

@@ -2,6 +2,7 @@ import os
 import json
 import logging
 import platform
+import argparse
 from typing import Optional
 
 import requests
@@ -9,6 +10,18 @@ from pydantic import BaseModel
 
 from . import __version__
 
+OVERRIDABLE_SETTINGS = (
+    "api_key",
+    "user_agent",
+    "download_to",
+    "mirror_web",
+    "urls_only",
+    "parallel",
+    "filter_files_glob",
+    "filter_files_regex",
+    "verbose",
+)
+
 
 class Settings(BaseModel):
     """Available settings for itch-dl. Make sure all of them
@@ -17,9 +30,16 @@ class Settings(BaseModel):
     api_key: Optional[str] = None
     user_agent: str = f"python-requests/{requests.__version__} itch-dl/{__version__}"
 
+    download_to: Optional[str] = None
+    mirror_web: bool = False
+    urls_only: bool = False
+    parallel: int = 1
+
     filter_files_glob: Optional[str] = None
     filter_files_regex: Optional[str] = None
 
+    verbose: bool = False
+
 
 def create_and_get_config_path() -> str:
     """Returns the configuration directory in the appropriate
@@ -37,7 +57,7 @@ def create_and_get_config_path() -> str:
     return os.path.join(base_path, "itch-dl")
 
 
-def load_config(profile: Optional[str] = None) -> Settings:
+def load_config(args: argparse.Namespace, profile: Optional[str] = None) -> Settings:
     """Loads the configuration from the file system if it exists,
     the returns a Settings object."""
     config_path = create_and_get_config_path()
@@ -58,4 +78,13 @@ def load_config(profile: Optional[str] = None) -> Settings:
 
         config_data.update(profile_data)
 
-    return Settings(**config_data)
+    # All settings from the base file:
+    settings = Settings(**config_data)
+
+    # Apply overrides from CLI args:
+    for key in OVERRIDABLE_SETTINGS:
+        value = getattr(args, key)
+        if value:
+            setattr(settings, key, value)
+
+    return settings

itch_dl/downloader.py

@@ -62,10 +62,7 @@ class GameMetadata(TypedDict, total=False):
 
 
 class GameDownloader:
-    def __init__(self, download_to: str, mirror_web: bool, settings: Settings, keys: Dict[int, str]):
+    def __init__(self, settings: Settings, keys: Dict[int, str]):
-        self.download_to = download_to
-        self.mirror_web = mirror_web
-
         self.settings = settings
         self.download_keys = keys
         self.client = ItchApiClient(settings.api_key, settings.user_agent)
@@ -258,7 +255,7 @@ class GameDownloader:
 
         author, game = match["author"], match["game"]
 
-        download_path = os.path.join(self.download_to, author, game)
+        download_path = os.path.join(self.settings.download_to, author, game)
         os.makedirs(download_path, exist_ok=True)
 
         paths: Dict[str, str] = {k: os.path.join(download_path, v) for k, v in TARGET_PATHS.items()}
@@ -372,7 +369,7 @@ class GameDownloader:
             logging.warning(f"Game {title} has external download URLs: {external_urls}")
 
         # TODO: Mirror JS/CSS assets
-        if self.mirror_web:
+        if self.settings.mirror_web:
             os.makedirs(paths["screenshots"], exist_ok=True)
             for screenshot in metadata["screenshots"]:
                 if not screenshot:
@@ -406,20 +403,17 @@ class GameDownloader:
 
 def drive_downloads(
     jobs: List[str],
-    download_to: str,
-    mirror_web: bool,
     settings: Settings,
     keys: Dict[int, str],
-    parallel: int = 1,
 ):
-    downloader = GameDownloader(download_to, mirror_web, settings, keys)
+    downloader = GameDownloader(settings, keys)
     tqdm_args = {
         "desc": "Games",
         "unit": "game",
     }
 
-    if parallel > 1:
+    if settings.parallel > 1:
-        results = thread_map(downloader.download, jobs, max_workers=parallel, **tqdm_args)
+        results = thread_map(downloader.download, jobs, max_workers=settings.parallel, **tqdm_args)
     else:
         results = [downloader.download(job) for job in tqdm(jobs, **tqdm_args)]