Add a Settings system

Allows permanently configuring itch-dl with an API key and other things
in the future. Adds a new dependency, Pydantic, to validate the config.

itch_dl/api.py

@@ -8,11 +8,12 @@ from .consts import ITCH_API
 
 
 class ItchApiClient:
-    def __init__(self, api_key: str, base_url: Optional[str] = None):
+    def __init__(self, api_key: str, user_agent: str, base_url: Optional[str] = None):
         self.base_url = base_url or ITCH_API
         self.api_key = api_key
 
         self.requests = Session()
+        self.requests.headers['User-Agent'] = user_agent
 
         retry_strategy = Retry(
             total=5,

itch_dl/cli.py

@@ -4,6 +4,7 @@ import argparse
 
 from .handlers import get_jobs_for_url_or_path
 from .downloader import drive_downloads
+from .config import Settings, load_config
 from .keys import get_download_keys
 from .api import ItchApiClient
 
@@ -11,12 +12,14 @@ logging.basicConfig()
 logging.getLogger().setLevel(logging.INFO)
 
 
-def parse_args():
+def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Bulk download stuff from Itch.io.")
     parser.add_argument("url_or_path",
                         help="itch.io URL or path to a game jam entries.json file")
-    parser.add_argument("--api-key", metavar="key", required=True,
+    parser.add_argument("--api-key", metavar="key", default=None,
                         help="itch.io API key - https://itch.io/user/settings/api-keys")
+    parser.add_argument("--profile", metavar="profile", default=None,
+                        help="configuration profile to load")
     parser.add_argument("--urls-only", action="store_true",
                         help="print scraped game URLs without downloading them")
     parser.add_argument("--download-to", metavar="path",
@@ -30,18 +33,36 @@ def parse_args():
     return parser.parse_args()
 
 
+def apply_args_on_settings(args: argparse.Namespace, settings: Settings):
+    if args.api_key:
+        settings.api_key = args.api_key
+
+
 def run() -> int:
     args = parse_args()
     if args.verbose:
         logging.getLogger().setLevel(logging.DEBUG)
 
-    jobs = get_jobs_for_url_or_path(args.url_or_path, args.api_key)
+    settings = load_config(profile=args.profile)
+    apply_args_on_settings(args, settings)
+
+    if not settings.api_key:
+        exit("You did not provide an API key which itch-dl requires.\n"
+             "See https://github.com/DragoonAethis/itch-dl/wiki/API-Keys for more info.")
+
+    # Check API key validity:
+    client = ItchApiClient(settings.api_key, settings.user_agent)
+    profile_req = client.get("/profile")
+    if not profile_req.ok:
+        exit(f"Provided API key appears to be invalid: {profile_req.text}\n"
+             "See https://github.com/DragoonAethis/itch-dl/wiki/API-Keys for more info.")
+
+    jobs = get_jobs_for_url_or_path(args.url_or_path, settings)
     jobs = list(set(jobs))  # Deduplicate, just in case...
     logging.info(f"Found {len(jobs)} URL(s).")
 
     if len(jobs) == 0:
-        print("No URLs to download.")
-        return 1
+        exit("No URLs to download.")
 
     if args.urls_only:
         for job in jobs:
@@ -54,15 +75,7 @@ def run() -> int:
         download_to = os.path.normpath(args.download_to)
         os.makedirs(download_to, exist_ok=True)
 
-    client = ItchApiClient(args.api_key)
-
-    # Check API key validity:
-    profile_req = client.get("/profile")
-    if not profile_req.ok:
-        print(f"Provided API key appears to be invalid: {profile_req.text}")
-        exit(1)
-
     # Grab all the download keys (there's no way to fetch them per title...):
     keys = get_download_keys(client)
 
-    return drive_downloads(jobs, download_to, args.mirror_web, args.api_key, keys, parallel=args.parallel)
+    return drive_downloads(jobs, download_to, args.mirror_web, settings, keys, parallel=args.parallel)

itch_dl/downloader.py

@@ -14,6 +14,7 @@ from tqdm.contrib.concurrent import thread_map
 from .api import ItchApiClient
 from .utils import ItchDownloadError, get_int_after_marker_in_json
 from .consts import ITCH_GAME_URL_REGEX
+from .config import Settings
 from .infobox import parse_infobox, InfoboxMetadata
 
 TARGET_PATHS = {
@@ -58,12 +59,12 @@ class GameMetadata(TypedDict, total=False):
 
 
 class GameDownloader:
-    def __init__(self, download_to: str, mirror_web: bool, api_key: str, keys: Dict[int, str]):
+    def __init__(self, download_to: str, mirror_web: bool, settings: Settings, keys: Dict[int, str]):
         self.download_to = download_to
         self.mirror_web = mirror_web
 
         self.download_keys = keys
-        self.client = ItchApiClient(api_key)
+        self.client = ItchApiClient(settings.api_key, settings.user_agent)
 
     @staticmethod
     def get_rating_json(site) -> Optional[dict]:
@@ -337,11 +338,11 @@ def drive_downloads(
         jobs: List[str],
         download_to: str,
         mirror_web: bool,
-        api_key: str,
+        settings: Settings,
         keys: Dict[int, str],
         parallel: int = 1
 ):
-    downloader = GameDownloader(download_to, mirror_web, api_key, keys)
+    downloader = GameDownloader(download_to, mirror_web, settings, keys)
     tqdm_args = {
         "desc": "Games",
         "unit": "game",

itch_dl/handlers.py

@@ -9,6 +9,7 @@ from bs4 import BeautifulSoup
 from .api import ItchApiClient
 from .utils import ItchDownloadError, get_int_after_marker_in_json
 from .consts import ITCH_BASE, ITCH_URL, ITCH_BROWSER_TYPES
+from .config import Settings
 
 
 def get_jobs_for_game_jam_json(game_jam_json: dict) -> List[str]:
@@ -175,7 +176,7 @@ def get_jobs_for_path(path: str) -> List[str]:
     raise ValueError(f"File format is unknown - cannot read URLs to download.")
 
 
-def get_jobs_for_url_or_path(path_or_url: str, api_key: str) -> List[str]:
+def get_jobs_for_url_or_path(path_or_url: str, settings: Settings) -> List[str]:
     """Returns a list of Game URLs for a given itch.io URL or file."""
     path_or_url = path_or_url.strip()
 
@@ -184,7 +185,7 @@ def get_jobs_for_url_or_path(path_or_url: str, api_key: str) -> List[str]:
         path_or_url = "https://" + path_or_url[7:]
 
     if path_or_url.startswith("https://"):
-        client = ItchApiClient(api_key)
+        client = ItchApiClient(settings.api_key, settings.user_agent)
         return get_jobs_for_itch_url(path_or_url, client)
     elif os.path.isfile(path_or_url):
         return get_jobs_for_path(path_or_url)

pyproject.toml

@@ -29,7 +29,8 @@ urllib3 = "^1.26.9"
 requests = "^2.27.1"
 python-slugify = "^6.1.2"
 beautifulsoup4 = "^4.11.1"
-lxml = "^4.8.0"
+lxml = "^4.9.0"
+pydantic = "^1.9.1"
 
 [tool.poetry.scripts]
 itch-dl = "itch_dl.cli:run"