From cb08443778e6065dbb408043a46a7f8afa128860 Mon Sep 17 00:00:00 2001 From: Ryszard Knop Date: Fri, 14 Feb 2025 14:55:54 +0100 Subject: [PATCH] Upgrade to Python 3.10+ We can use fancier type annotations now, newer deps, more checkers. --- itch_dl/api.py | 4 ++-- itch_dl/config.py | 12 ++++++------ itch_dl/downloader.py | 40 ++++++++++++++++++++-------------------- itch_dl/handlers.py | 23 +++++++++++------------ itch_dl/infobox.py | 38 +++++++++++++++++++------------------- itch_dl/keys.py | 11 +++++------ itch_dl/utils.py | 5 ++--- pyproject.toml | 4 ++-- 8 files changed, 67 insertions(+), 70 deletions(-) diff --git a/itch_dl/api.py b/itch_dl/api.py index a67791d..79dea82 100644 --- a/itch_dl/api.py +++ b/itch_dl/api.py @@ -1,4 +1,4 @@ -from typing import Optional, Any +from typing import Any import requests from requests import Session @@ -9,7 +9,7 @@ from .consts import ITCH_API class ItchApiClient: - def __init__(self, api_key: str, user_agent: str, base_url: Optional[str] = None) -> None: + def __init__(self, api_key: str, user_agent: str, base_url: str | None = None) -> None: self.base_url = base_url or ITCH_API self.api_key = api_key diff --git a/itch_dl/config.py b/itch_dl/config.py index 1f0df61..607ff21 100644 --- a/itch_dl/config.py +++ b/itch_dl/config.py @@ -4,7 +4,7 @@ import logging import platform import argparse from dataclasses import dataclass, fields -from typing import Optional, Any, get_type_hints +from typing import Any, get_type_hints import requests @@ -16,16 +16,16 @@ class Settings: """Available settings for itch-dl. Make sure all of them have default values, as the config file may not exist.""" - api_key: Optional[str] = None + api_key: str | None = None user_agent: str = f"python-requests/{requests.__version__} itch-dl/{__version__}" - download_to: Optional[str] = None + download_to: str | None = None mirror_web: bool = False urls_only: bool = False parallel: int = 1 - filter_files_glob: Optional[str] = None - filter_files_regex: Optional[str] = None + filter_files_glob: str | None = None + filter_files_regex: str | None = None verbose: bool = False @@ -73,7 +73,7 @@ def clean_config(config_data: dict[str, Any]) -> dict[str, Any]: return cleaned_config -def load_config(args: argparse.Namespace, profile: Optional[str] = None) -> Settings: +def load_config(args: argparse.Namespace, profile: str | None = None) -> Settings: """Loads the configuration from the file system if it exists, the returns a Settings object.""" config_path = create_and_get_config_path() diff --git a/itch_dl/downloader.py b/itch_dl/downloader.py index f2232ea..ba89f16 100644 --- a/itch_dl/downloader.py +++ b/itch_dl/downloader.py @@ -6,7 +6,7 @@ import logging import urllib.parse import zipfile import tarfile -from typing import List, Dict, TypedDict, Optional, Union, Any +from typing import TypedDict, Any from bs4 import BeautifulSoup from requests.exceptions import HTTPError, JSONDecodeError @@ -30,7 +30,7 @@ TARGET_PATHS = { class DownloadResult: - def __init__(self, url: str, success: bool, errors: Optional[List[str]], external_urls: List[str]) -> None: + def __init__(self, url: str, success: bool, errors: list[str] | None, external_urls: list[str]) -> None: self.url = url self.success = success self.errors = errors or [] @@ -42,17 +42,17 @@ class GameMetadata(TypedDict, total=False): title: str url: str - errors: List[str] - external_downloads: List[str] + errors: list[str] + external_downloads: list[str] author: str author_url: str - cover_url: Optional[str] - screenshots: List[str] - description: Optional[str] + cover_url: str | None + screenshots: list[str] + description: str | None - rating: Dict[str, Union[float, int]] + rating: dict[str, float | int] extra: InfoboxMetadata created_at: str @@ -62,13 +62,13 @@ class GameMetadata(TypedDict, total=False): class GameDownloader: - def __init__(self, settings: Settings, keys: Dict[int, str]) -> None: + def __init__(self, settings: Settings, keys: dict[int, str]) -> None: self.settings = settings self.download_keys = keys self.client = ItchApiClient(settings.api_key, settings.user_agent) @staticmethod - def get_rating_json(site: BeautifulSoup) -> Optional[dict]: + def get_rating_json(site: BeautifulSoup) -> dict | None: for ldjson_node in site.find_all("script", type="application/ld+json"): try: ldjson: dict = json.loads(ldjson_node.text.strip()) @@ -80,7 +80,7 @@ class GameDownloader: return None @staticmethod - def get_meta(site: BeautifulSoup, **kwargs: Any) -> Optional[str]: # noqa: ANN401 + def get_meta(site: BeautifulSoup, **kwargs: Any) -> str | None: # noqa: ANN401 """Grabs values.""" node = site.find("meta", attrs=kwargs) if not node: @@ -89,7 +89,7 @@ class GameDownloader: return node.get("content") def get_game_id(self, url: str, site: BeautifulSoup) -> int: - game_id: Optional[int] = None + game_id: int | None = None try: # Headers: @@ -134,14 +134,14 @@ class GameDownloader: return game_id def extract_metadata(self, game_id: int, url: str, site: BeautifulSoup) -> GameMetadata: - rating_json: Optional[dict] = self.get_rating_json(site) + rating_json: dict | None = self.get_rating_json(site) title = rating_json.get("name") if rating_json else None - description: Optional[str] = self.get_meta(site, property="og:description") + description: str | None = self.get_meta(site, property="og:description") if not description: description = self.get_meta(site, name="description") - screenshot_urls: List[str] = [] + screenshot_urls: list[str] = [] screenshots_node = site.find("div", class_="screenshot_list") if screenshots_node: screenshot_urls = [a["href"] for a in screenshots_node.find_all("a")] @@ -193,7 +193,7 @@ class GameDownloader: return credentials - def download_file(self, url: str, download_path: Optional[str], credentials: dict) -> str: + def download_file(self, url: str, download_path: str | None, credentials: dict) -> str: """Performs a request to download a given file, optionally saves the file to the provided path and returns the final URL that was downloaded.""" try: @@ -216,7 +216,7 @@ class GameDownloader: except HTTPError as e: raise ItchDownloadError(f"Unrecoverable download error: {e}") from e - def download_file_by_upload_id(self, upload_id: int, download_path: Optional[str], credentials: dict) -> str: + def download_file_by_upload_id(self, upload_id: int, download_path: str | None, credentials: dict) -> str: """Performs a request to download a given upload by its ID.""" return self.download_file(f"/uploads/{upload_id}/download", download_path, credentials) @@ -258,7 +258,7 @@ class GameDownloader: download_path = os.path.join(self.settings.download_to, author, game) os.makedirs(download_path, exist_ok=True) - paths: Dict[str, str] = {k: os.path.join(download_path, v) for k, v in TARGET_PATHS.items()} + paths: dict[str, str] = {k: os.path.join(download_path, v) for k, v in TARGET_PATHS.items()} if os.path.exists(paths["metadata"]) and skip_downloaded: # As metadata is the final file we write, all the files @@ -405,9 +405,9 @@ class GameDownloader: def drive_downloads( - jobs: List[str], + jobs: list[str], settings: Settings, - keys: Dict[int, str], + keys: dict[int, str], ) -> None: downloader = GameDownloader(settings, keys) tqdm_args = { diff --git a/itch_dl/handlers.py b/itch_dl/handlers.py index 33b7e9d..d5da373 100644 --- a/itch_dl/handlers.py +++ b/itch_dl/handlers.py @@ -2,7 +2,6 @@ import json import os.path import logging import urllib.parse -from typing import List, Set, Optional from http.client import responses from bs4 import BeautifulSoup @@ -14,7 +13,7 @@ from .config import Settings from .keys import get_owned_games -def get_jobs_for_game_jam_json(game_jam_json: dict) -> List[str]: +def get_jobs_for_game_jam_json(game_jam_json: dict) -> list[str]: if "jam_games" not in game_jam_json: raise Exception("Provided JSON is not a valid itch.io jam JSON.") @@ -26,7 +25,7 @@ def get_game_jam_json(jam_url: str, client: ItchApiClient) -> dict: if not r.ok: raise ItchDownloadError(f"Could not download the game jam site: {r.status_code} {r.reason}") - jam_id: Optional[int] = get_int_after_marker_in_json(r.text, "I.ViewJam", "id") + jam_id: int | None = get_int_after_marker_in_json(r.text, "I.ViewJam", "id") if jam_id is None: raise ItchDownloadError( "Provided site did not contain the Game Jam ID. Provide " @@ -42,7 +41,7 @@ def get_game_jam_json(jam_url: str, client: ItchApiClient) -> dict: return r.json() -def get_jobs_for_browse_url(url: str, client: ItchApiClient) -> List[str]: +def get_jobs_for_browse_url(url: str, client: ItchApiClient) -> list[str]: """ Every browser page has a hidden RSS feed that can be accessed by appending .xml to its URL. An optional "page" argument lets us @@ -53,7 +52,7 @@ def get_jobs_for_browse_url(url: str, client: ItchApiClient) -> List[str]: .xml?page=N suffix and iterate until we've caught 'em all. """ page = 1 - found_urls: Set[str] = set() + found_urls: set[str] = set() logging.info("Scraping game URLs from RSS feeds for %s", url) while True: @@ -87,9 +86,9 @@ def get_jobs_for_browse_url(url: str, client: ItchApiClient) -> List[str]: return list(found_urls) -def get_jobs_for_collection_json(url: str, client: ItchApiClient) -> List[str]: +def get_jobs_for_collection_json(url: str, client: ItchApiClient) -> list[str]: page = 1 - found_urls: Set[str] = set() + found_urls: set[str] = set() while True: logging.info("Downloading page %d (found %d URLs total)", page, len(found_urls)) @@ -118,7 +117,7 @@ def get_jobs_for_collection_json(url: str, client: ItchApiClient) -> List[str]: return list(found_urls) -def get_jobs_for_creator(creator: str, client: ItchApiClient) -> List[str]: +def get_jobs_for_creator(creator: str, client: ItchApiClient) -> list[str]: logging.info("Downloading public games for creator %s", creator) r = client.get(f"https://{ITCH_BASE}/profile/{creator}", append_api_key=False) if not r.ok: @@ -139,7 +138,7 @@ def get_jobs_for_creator(creator: str, client: ItchApiClient) -> List[str]: return sorted(game_links) -def get_jobs_for_itch_url(url: str, client: ItchApiClient) -> List[str]: +def get_jobs_for_itch_url(url: str, client: ItchApiClient) -> list[str]: if url.startswith("http://"): logging.info("HTTP link provided, upgrading to HTTPS") url = "https://" + url[7:] @@ -149,7 +148,7 @@ def get_jobs_for_itch_url(url: str, client: ItchApiClient) -> List[str]: url = ITCH_URL + "/" + url[20:] url_parts = urllib.parse.urlparse(url) - url_path_parts: List[str] = [x for x in str(url_parts.path).split("/") if len(x) > 0] + url_path_parts: list[str] = [x for x in str(url_parts.path).split("/") if len(x) > 0] if url_parts.netloc == ITCH_BASE: if len(url_path_parts) == 0: @@ -209,7 +208,7 @@ def get_jobs_for_itch_url(url: str, client: ItchApiClient) -> List[str]: raise ValueError(f"Unknown domain: {url_parts.netloc}") -def get_jobs_for_path(path: str) -> List[str]: +def get_jobs_for_path(path: str) -> list[str]: try: # Game Jam Entries JSON? with open(path, "rb") as f: json_data = json.load(f) @@ -237,7 +236,7 @@ def get_jobs_for_path(path: str) -> List[str]: raise ValueError("File format is unknown - cannot read URLs to download.") -def get_jobs_for_url_or_path(path_or_url: str, settings: Settings) -> List[str]: +def get_jobs_for_url_or_path(path_or_url: str, settings: Settings) -> list[str]: """Returns a list of Game URLs for a given itch.io URL or file.""" path_or_url = path_or_url.strip() diff --git a/itch_dl/infobox.py b/itch_dl/infobox.py index bcb9160..a7a7e34 100644 --- a/itch_dl/infobox.py +++ b/itch_dl/infobox.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import TypedDict, Dict, List, Any, Tuple, Optional +from typing import TypedDict, Any from bs4 import BeautifulSoup @@ -9,26 +9,26 @@ class InfoboxMetadata(TypedDict, total=False): released_at: datetime published_at: datetime status: str - platforms: List[str] # Windows/macOS/Linux/etc + platforms: list[str] # Windows/macOS/Linux/etc publisher: str - author: Dict[str, str] # See impl below! - authors: Dict[str, str] # Links - genre: Dict[str, str] # Links - tools: Dict[str, str] # Links - license: Dict[str, str] # Links - asset_license: Dict[str, str] # Links - tags: Dict[str, str] # Links + author: dict[str, str] # See impl below! + authors: dict[str, str] # Links + genre: dict[str, str] # Links + tools: dict[str, str] # Links + license: dict[str, str] # Links + asset_license: dict[str, str] # Links + tags: dict[str, str] # Links length: str - multiplayer: Dict[str, str] # Links + multiplayer: dict[str, str] # Links player_count: str - accessibility: Dict[str, str] # Links - inputs: Dict[str, str] # Links - links: Dict[str, str] # Links - mentions: Dict[str, str] # Links - category: Dict[str, str] # Links + accessibility: dict[str, str] # Links + inputs: dict[str, str] # Links + links: dict[str, str] # Links + mentions: dict[str, str] # Links + category: dict[str, str] # Links -def parse_date_block(td: BeautifulSoup) -> Optional[datetime]: +def parse_date_block(td: BeautifulSoup) -> datetime | None: abbr = td.find("abbr") if not abbr or "title" not in abbr.attrs: return None @@ -39,17 +39,17 @@ def parse_date_block(td: BeautifulSoup) -> Optional[datetime]: return datetime(date.year, date.month, date.day, time.hour, time.minute) -def parse_links(td: BeautifulSoup) -> Dict[str, str]: +def parse_links(td: BeautifulSoup) -> dict[str, str]: """Parses blocks of comma-separated blocks, returns a dict of link text -> URL it points at.""" return {link.text.strip(): link["href"] for link in td.find_all("a")} -def parse_text_from_links(td: BeautifulSoup) -> List[str]: +def parse_text_from_links(td: BeautifulSoup) -> list[str]: return list(parse_links(td).keys()) -def parse_tr(name: str, content: BeautifulSoup) -> Optional[Tuple[str, Any]]: +def parse_tr(name: str, content: BeautifulSoup) -> tuple[str, Any] | None: if name == "Updated": return "updated_at", parse_date_block(content) elif name == "Release date": diff --git a/itch_dl/keys.py b/itch_dl/keys.py index 2841c7b..369ed5e 100644 --- a/itch_dl/keys.py +++ b/itch_dl/keys.py @@ -1,11 +1,10 @@ import logging -from typing import Dict, List, Tuple from .api import ItchApiClient KEYS_CACHED: bool = False -DOWNLOAD_KEYS: Dict[int, str] = {} -GAME_URLS: List[str] = [] +DOWNLOAD_KEYS: dict[int, str] = {} +GAME_URLS: list[str] = [] def load_keys_and_urls(client: ItchApiClient) -> None: @@ -36,21 +35,21 @@ def load_keys_and_urls(client: ItchApiClient) -> None: KEYS_CACHED = True -def get_owned_keys(client: ItchApiClient) -> Tuple[Dict[int, str], List[str]]: +def get_owned_keys(client: ItchApiClient) -> tuple[dict[int, str], list[str]]: if not KEYS_CACHED: load_keys_and_urls(client) return DOWNLOAD_KEYS, GAME_URLS -def get_download_keys(client: ItchApiClient) -> Dict[int, str]: +def get_download_keys(client: ItchApiClient) -> dict[int, str]: if not KEYS_CACHED: load_keys_and_urls(client) return DOWNLOAD_KEYS -def get_owned_games(client: ItchApiClient) -> List[str]: +def get_owned_games(client: ItchApiClient) -> list[str]: if not KEYS_CACHED: load_keys_and_urls(client) diff --git a/itch_dl/utils.py b/itch_dl/utils.py index 9a002ce..fd49885 100644 --- a/itch_dl/utils.py +++ b/itch_dl/utils.py @@ -1,12 +1,11 @@ import re -from typing import Optional class ItchDownloadError(Exception): pass -def get_int_after_marker_in_json(text: str, marker: str, key: str) -> Optional[int]: +def get_int_after_marker_in_json(text: str, marker: str, key: str) -> int | None: """ Many itch.io sites use a pattern like this: Most of the HTML page is prerendered, but certain interactive objects are handled with @@ -14,7 +13,7 @@ def get_int_after_marker_in_json(text: str, marker: str, key: str) -> Optional[i somewhere near the end of each page. Those config blocks often contain metadata like game/page IDs that we want to extract. """ - marker_line: Optional[str] = None + marker_line: str | None = None for line in reversed(text.splitlines()): marker_index = line.find(marker) if marker_index != -1: diff --git a/pyproject.toml b/pyproject.toml index ab01414..eb218cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Donate" = "https://ko-fi.com/dragoonaethis" [tool.poetry.dependencies] -python = "^3.8" +python = "^3.10" tqdm = "^4.67.1" urllib3 = "^1.26.20" requests = "^2.32.3" @@ -41,7 +41,7 @@ build-backend = "poetry.core.masonry.api" [tool.ruff] line-length = 120 -target-version = "py38" +target-version = "py310" [tool.ruff.lint] # https://docs.astral.sh/ruff/rules/