From 4542057654181fa4ef3a8bab721452c9ee71a8b9 Mon Sep 17 00:00:00 2001 From: Ryszard Knop Date: Sun, 12 Jun 2022 19:28:31 +0200 Subject: [PATCH] Add a Settings system Allows permanently configuring itch-dl with an API key and other things in the future. Adds a new dependency, Pydantic, to validate the config. --- itch_dl/api.py | 3 ++- itch_dl/cli.py | 41 +++++++++++++++++++++++++++-------------- itch_dl/downloader.py | 9 +++++---- itch_dl/handlers.py | 5 +++-- pyproject.toml | 3 ++- 5 files changed, 39 insertions(+), 22 deletions(-) diff --git a/itch_dl/api.py b/itch_dl/api.py index 21be0f2..1fd8005 100644 --- a/itch_dl/api.py +++ b/itch_dl/api.py @@ -8,11 +8,12 @@ from .consts import ITCH_API class ItchApiClient: - def __init__(self, api_key: str, base_url: Optional[str] = None): + def __init__(self, api_key: str, user_agent: str, base_url: Optional[str] = None): self.base_url = base_url or ITCH_API self.api_key = api_key self.requests = Session() + self.requests.headers['User-Agent'] = user_agent retry_strategy = Retry( total=5, diff --git a/itch_dl/cli.py b/itch_dl/cli.py index 20de8bc..140cc76 100644 --- a/itch_dl/cli.py +++ b/itch_dl/cli.py @@ -4,6 +4,7 @@ import argparse from .handlers import get_jobs_for_url_or_path from .downloader import drive_downloads +from .config import Settings, load_config from .keys import get_download_keys from .api import ItchApiClient @@ -11,12 +12,14 @@ logging.basicConfig() logging.getLogger().setLevel(logging.INFO) -def parse_args(): +def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Bulk download stuff from Itch.io.") parser.add_argument("url_or_path", help="itch.io URL or path to a game jam entries.json file") - parser.add_argument("--api-key", metavar="key", required=True, + parser.add_argument("--api-key", metavar="key", default=None, help="itch.io API key - https://itch.io/user/settings/api-keys") + parser.add_argument("--profile", metavar="profile", default=None, + help="configuration profile to load") parser.add_argument("--urls-only", action="store_true", help="print scraped game URLs without downloading them") parser.add_argument("--download-to", metavar="path", @@ -30,18 +33,36 @@ def parse_args(): return parser.parse_args() +def apply_args_on_settings(args: argparse.Namespace, settings: Settings): + if args.api_key: + settings.api_key = args.api_key + + def run() -> int: args = parse_args() if args.verbose: logging.getLogger().setLevel(logging.DEBUG) - jobs = get_jobs_for_url_or_path(args.url_or_path, args.api_key) + settings = load_config(profile=args.profile) + apply_args_on_settings(args, settings) + + if not settings.api_key: + exit("You did not provide an API key which itch-dl requires.\n" + "See https://github.com/DragoonAethis/itch-dl/wiki/API-Keys for more info.") + + # Check API key validity: + client = ItchApiClient(settings.api_key, settings.user_agent) + profile_req = client.get("/profile") + if not profile_req.ok: + exit(f"Provided API key appears to be invalid: {profile_req.text}\n" + "See https://github.com/DragoonAethis/itch-dl/wiki/API-Keys for more info.") + + jobs = get_jobs_for_url_or_path(args.url_or_path, settings) jobs = list(set(jobs)) # Deduplicate, just in case... logging.info(f"Found {len(jobs)} URL(s).") if len(jobs) == 0: - print("No URLs to download.") - return 1 + exit("No URLs to download.") if args.urls_only: for job in jobs: @@ -54,15 +75,7 @@ def run() -> int: download_to = os.path.normpath(args.download_to) os.makedirs(download_to, exist_ok=True) - client = ItchApiClient(args.api_key) - - # Check API key validity: - profile_req = client.get("/profile") - if not profile_req.ok: - print(f"Provided API key appears to be invalid: {profile_req.text}") - exit(1) - # Grab all the download keys (there's no way to fetch them per title...): keys = get_download_keys(client) - return drive_downloads(jobs, download_to, args.mirror_web, args.api_key, keys, parallel=args.parallel) + return drive_downloads(jobs, download_to, args.mirror_web, settings, keys, parallel=args.parallel) diff --git a/itch_dl/downloader.py b/itch_dl/downloader.py index a5c62ae..6f41c8f 100644 --- a/itch_dl/downloader.py +++ b/itch_dl/downloader.py @@ -14,6 +14,7 @@ from tqdm.contrib.concurrent import thread_map from .api import ItchApiClient from .utils import ItchDownloadError, get_int_after_marker_in_json from .consts import ITCH_GAME_URL_REGEX +from .config import Settings from .infobox import parse_infobox, InfoboxMetadata TARGET_PATHS = { @@ -58,12 +59,12 @@ class GameMetadata(TypedDict, total=False): class GameDownloader: - def __init__(self, download_to: str, mirror_web: bool, api_key: str, keys: Dict[int, str]): + def __init__(self, download_to: str, mirror_web: bool, settings: Settings, keys: Dict[int, str]): self.download_to = download_to self.mirror_web = mirror_web self.download_keys = keys - self.client = ItchApiClient(api_key) + self.client = ItchApiClient(settings.api_key, settings.user_agent) @staticmethod def get_rating_json(site) -> Optional[dict]: @@ -337,11 +338,11 @@ def drive_downloads( jobs: List[str], download_to: str, mirror_web: bool, - api_key: str, + settings: Settings, keys: Dict[int, str], parallel: int = 1 ): - downloader = GameDownloader(download_to, mirror_web, api_key, keys) + downloader = GameDownloader(download_to, mirror_web, settings, keys) tqdm_args = { "desc": "Games", "unit": "game", diff --git a/itch_dl/handlers.py b/itch_dl/handlers.py index f1d57d2..b17abb5 100644 --- a/itch_dl/handlers.py +++ b/itch_dl/handlers.py @@ -9,6 +9,7 @@ from bs4 import BeautifulSoup from .api import ItchApiClient from .utils import ItchDownloadError, get_int_after_marker_in_json from .consts import ITCH_BASE, ITCH_URL, ITCH_BROWSER_TYPES +from .config import Settings def get_jobs_for_game_jam_json(game_jam_json: dict) -> List[str]: @@ -175,7 +176,7 @@ def get_jobs_for_path(path: str) -> List[str]: raise ValueError(f"File format is unknown - cannot read URLs to download.") -def get_jobs_for_url_or_path(path_or_url: str, api_key: str) -> List[str]: +def get_jobs_for_url_or_path(path_or_url: str, settings: Settings) -> List[str]: """Returns a list of Game URLs for a given itch.io URL or file.""" path_or_url = path_or_url.strip() @@ -184,7 +185,7 @@ def get_jobs_for_url_or_path(path_or_url: str, api_key: str) -> List[str]: path_or_url = "https://" + path_or_url[7:] if path_or_url.startswith("https://"): - client = ItchApiClient(api_key) + client = ItchApiClient(settings.api_key, settings.user_agent) return get_jobs_for_itch_url(path_or_url, client) elif os.path.isfile(path_or_url): return get_jobs_for_path(path_or_url) diff --git a/pyproject.toml b/pyproject.toml index dfadcb1..af156eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,8 @@ urllib3 = "^1.26.9" requests = "^2.27.1" python-slugify = "^6.1.2" beautifulsoup4 = "^4.11.1" -lxml = "^4.8.0" +lxml = "^4.9.0" +pydantic = "^1.9.1" [tool.poetry.scripts] itch-dl = "itch_dl.cli:run"