Mirrors/itch-dl
2
0
Fork 0
mirror of https://github.com/DragoonAethis/itch-dl.git synced 2025-02-01 22:32:37 +01:00
Code Issues Packages Projects Releases Wiki Activity

Sync settings and CLI arguments

Browse Source 
Most CLI arguments can be now saved into the configuration profile JSON.
This also cleans up passing around some, but not all args, into various
classes and methods, instead of just passing all of settings.
This commit is contained in:
Ryszard Knop 2025-01-31 22:35:37 +01:00
parent 5a9cbf675f
commit 1cb57d0be4
3 changed files with 63 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
itch_dl/cli.py
View File

@ -17,43 +17,42 @@ def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Bulk download stuff from Itch.io.")
parser.add_argument("url_or_path",
help="itch.io URL or path to a game jam entries.json file")
parser.add_argument("--api-key", metavar="key", default=None,
help="itch.io API key - https://itch.io/user/settings/api-keys")
parser.add_argument("--profile", metavar="profile", default=None,
help="configuration profile to load")
parser.add_argument("--urls-only", action="store_true",
help="print scraped game URLs without downloading them")
parser.add_argument("--download-to", metavar="path",
help="directory to save results into (default: current dir)")
parser.add_argument("--parallel", metavar="parallel", type=int, default=1,
help="how many threads to use for downloading games (default: 1)")
# These args must match config.py -> Settings class. Make sure all defaults here
# evaluate to False, or apply_args_on_settings will override profile settings.
parser.add_argument("--api-key", metavar="key", default=None,
help="itch.io API key - https://itch.io/user/settings/api-keys")
parser.add_argument("--user-agent", metavar="agent", default=None,
help="user agent to use when sending HTTP requests")
parser.add_argument("--download-to", metavar="path", default=None,
help="directory to save results into (default: current working dir)")
parser.add_argument("--mirror-web", action="store_true",
help="try to fetch assets on game sites")
parser.add_argument("--urls-only", action="store_true",
help="print scraped game URLs without downloading them")
parser.add_argument("--parallel", metavar="parallel", type=int, default=None,
help="how many threads to use for downloading games (default: 1)")
parser.add_argument("--filter-files-glob", metavar="glob", default=None,
help="filter downloaded files with a shell-style glob/fnmatch (unmatched files are skipped)")
parser.add_argument("--filter-files-regex", metavar="regex", default=None,
help="filter downloaded files with a Python regex (unmatched files are skipped)")
parser.add_argument("--verbose", action="store_true",
help="print verbose logs")
return parser.parse_args()
# fmt: on
def apply_args_on_settings(args: argparse.Namespace, settings: Settings):
"""Apply settings overrides from provided command line arguments, if set."""
for key in ("api_key", "filter_files_glob", "filter_files_regex"):
value = getattr(args, key)
if value:
setattr(settings, key, value)
def run() -> int:
args = parse_args()
if args.verbose:
logging.getLogger().setLevel(logging.DEBUG)
settings = load_config(profile=args.profile)
apply_args_on_settings(args, settings)
settings: Settings = load_config(args, profile=args.profile)
if settings.verbose:
logging.getLogger().setLevel(logging.DEBUG)
if not settings.api_key:
exit(
@ -61,6 +60,9 @@ def run() -> int:
"See https://github.com/DragoonAethis/itch-dl/wiki/API-Keys for more info."
)
url_or_path = args.url_or_path
del args # Do not use `args` beyond this point.
# Check API key validity:
client = ItchApiClient(settings.api_key, settings.user_agent)
profile_req = client.get("/profile")
@ -70,25 +72,24 @@ def run() -> int:
"See https://github.com/DragoonAethis/itch-dl/wiki/API-Keys for more info."
)
jobs = get_jobs_for_url_or_path(args.url_or_path, settings)
jobs = get_jobs_for_url_or_path(url_or_path, settings)
jobs = list(set(jobs)) # Deduplicate, just in case...
logging.info(f"Found {len(jobs)} URL(s).")
if len(jobs) == 0:
exit("No URLs to download.")
if args.urls_only:
if settings.urls_only:
for job in jobs:
print(job)
return 0
download_to = os.getcwd()
if args.download_to is not None:
download_to = os.path.normpath(args.download_to)
os.makedirs(download_to, exist_ok=True)
# If the download dir is not set, use the current working dir:
settings.download_to = os.path.normpath(settings.download_to or os.getcwd())
os.makedirs(settings.download_to, exist_ok=True)
# Grab all the download keys (there's no way to fetch them per title...):
keys = get_download_keys(client)
return drive_downloads(jobs, download_to, args.mirror_web, settings, keys, parallel=args.parallel)
return drive_downloads(jobs, settings, keys)

33
itch_dl/config.py
View File

@ -2,6 +2,7 @@ import os
import json
import logging
import platform
import argparse
from typing import Optional
import requests
@ -9,6 +10,18 @@ from pydantic import BaseModel
from . import __version__
OVERRIDABLE_SETTINGS = (
"api_key",
"user_agent",
"download_to",
"mirror_web",
"urls_only",
"parallel",
"filter_files_glob",
"filter_files_regex",
"verbose",
)
class Settings(BaseModel):
"""Available settings for itch-dl. Make sure all of them
@ -17,9 +30,16 @@ class Settings(BaseModel):
api_key: Optional[str] = None
user_agent: str = f"python-requests/{requests.__version__} itch-dl/{__version__}"
download_to: Optional[str] = None
mirror_web: bool = False
urls_only: bool = False
parallel: int = 1
filter_files_glob: Optional[str] = None
filter_files_regex: Optional[str] = None
verbose: bool = False
def create_and_get_config_path() -> str:
"""Returns the configuration directory in the appropriate
@ -37,7 +57,7 @@ def create_and_get_config_path() -> str:
return os.path.join(base_path, "itch-dl")
def load_config(profile: Optional[str] = None) -> Settings:
def load_config(args: argparse.Namespace, profile: Optional[str] = None) -> Settings:
"""Loads the configuration from the file system if it exists,
the returns a Settings object."""
config_path = create_and_get_config_path()
@ -58,4 +78,13 @@ def load_config(profile: Optional[str] = None) -> Settings:
config_data.update(profile_data)
return Settings(**config_data)
# All settings from the base file:
settings = Settings(**config_data)
# Apply overrides from CLI args:
for key in OVERRIDABLE_SETTINGS:
value = getattr(args, key)
if value:
setattr(settings, key, value)
return settings

18
itch_dl/downloader.py
View File

@ -62,10 +62,7 @@ class GameMetadata(TypedDict, total=False):
class GameDownloader:
def __init__(self, download_to: str, mirror_web: bool, settings: Settings, keys: Dict[int, str]):
self.download_to = download_to
self.mirror_web = mirror_web
def __init__(self, settings: Settings, keys: Dict[int, str]):
self.settings = settings
self.download_keys = keys
self.client = ItchApiClient(settings.api_key, settings.user_agent)
@ -258,7 +255,7 @@ class GameDownloader:
author, game = match["author"], match["game"]
download_path = os.path.join(self.download_to, author, game)
download_path = os.path.join(self.settings.download_to, author, game)
os.makedirs(download_path, exist_ok=True)
paths: Dict[str, str] = {k: os.path.join(download_path, v) for k, v in TARGET_PATHS.items()}
@ -372,7 +369,7 @@ class GameDownloader:
logging.warning(f"Game {title} has external download URLs: {external_urls}")
# TODO: Mirror JS/CSS assets
if self.mirror_web:
if self.settings.mirror_web:
os.makedirs(paths["screenshots"], exist_ok=True)
for screenshot in metadata["screenshots"]:
if not screenshot:
@ -406,20 +403,17 @@ class GameDownloader:
def drive_downloads(
jobs: List[str],
download_to: str,
mirror_web: bool,
settings: Settings,
keys: Dict[int, str],
parallel: int = 1,
):
downloader = GameDownloader(download_to, mirror_web, settings, keys)
downloader = GameDownloader(settings, keys)
tqdm_args = {
"desc": "Games",
"unit": "game",
}
if parallel > 1:
results = thread_map(downloader.download, jobs, max_workers=parallel, **tqdm_args)
if settings.parallel > 1:
results = thread_map(downloader.download, jobs, max_workers=settings.parallel, **tqdm_args)
else:
results = [downloader.download(job) for job in tqdm(jobs, **tqdm_args)]