Upgrade to Python 3.10+

We can use fancier type annotations now, newer deps, more checkers.

itch_dl/api.py

@@ -1,4 +1,4 @@
-from typing import Optional, Any
+from typing import Any
 
 import requests
 from requests import Session
@@ -9,7 +9,7 @@ from .consts import ITCH_API
 
 
 class ItchApiClient:
-    def __init__(self, api_key: str, user_agent: str, base_url: Optional[str] = None) -> None:
+    def __init__(self, api_key: str, user_agent: str, base_url: str | None = None) -> None:
         self.base_url = base_url or ITCH_API
         self.api_key = api_key
 

itch_dl/config.py

@@ -4,7 +4,7 @@ import logging
 import platform
 import argparse
 from dataclasses import dataclass, fields
-from typing import Optional, Any, get_type_hints
+from typing import Any, get_type_hints
 
 import requests
 
@@ -16,16 +16,16 @@ class Settings:
     """Available settings for itch-dl. Make sure all of them
     have default values, as the config file may not exist."""
 
-    api_key: Optional[str] = None
+    api_key: str | None = None
     user_agent: str = f"python-requests/{requests.__version__} itch-dl/{__version__}"
 
-    download_to: Optional[str] = None
+    download_to: str | None = None
     mirror_web: bool = False
     urls_only: bool = False
     parallel: int = 1
 
-    filter_files_glob: Optional[str] = None
+    filter_files_glob: str | None = None
-    filter_files_regex: Optional[str] = None
+    filter_files_regex: str | None = None
 
     verbose: bool = False
 
@@ -73,7 +73,7 @@ def clean_config(config_data: dict[str, Any]) -> dict[str, Any]:
     return cleaned_config
 
 
-def load_config(args: argparse.Namespace, profile: Optional[str] = None) -> Settings:
+def load_config(args: argparse.Namespace, profile: str | None = None) -> Settings:
     """Loads the configuration from the file system if it exists,
     the returns a Settings object."""
     config_path = create_and_get_config_path()

itch_dl/downloader.py

@@ -6,7 +6,7 @@ import logging
 import urllib.parse
 import zipfile
 import tarfile
-from typing import List, Dict, TypedDict, Optional, Union, Any
+from typing import TypedDict, Any
 
 from bs4 import BeautifulSoup
 from requests.exceptions import HTTPError, JSONDecodeError
@@ -30,7 +30,7 @@ TARGET_PATHS = {
 
 
 class DownloadResult:
-    def __init__(self, url: str, success: bool, errors: Optional[List[str]], external_urls: List[str]) -> None:
+    def __init__(self, url: str, success: bool, errors: list[str] | None, external_urls: list[str]) -> None:
         self.url = url
         self.success = success
         self.errors = errors or []
@@ -42,17 +42,17 @@ class GameMetadata(TypedDict, total=False):
     title: str
     url: str
 
-    errors: List[str]
+    errors: list[str]
-    external_downloads: List[str]
+    external_downloads: list[str]
 
     author: str
     author_url: str
 
-    cover_url: Optional[str]
+    cover_url: str | None
-    screenshots: List[str]
+    screenshots: list[str]
-    description: Optional[str]
+    description: str | None
 
-    rating: Dict[str, Union[float, int]]
+    rating: dict[str, float | int]
     extra: InfoboxMetadata
 
     created_at: str
@@ -62,13 +62,13 @@ class GameMetadata(TypedDict, total=False):
 
 
 class GameDownloader:
-    def __init__(self, settings: Settings, keys: Dict[int, str]) -> None:
+    def __init__(self, settings: Settings, keys: dict[int, str]) -> None:
         self.settings = settings
         self.download_keys = keys
         self.client = ItchApiClient(settings.api_key, settings.user_agent)
 
     @staticmethod
-    def get_rating_json(site: BeautifulSoup) -> Optional[dict]:
+    def get_rating_json(site: BeautifulSoup) -> dict | None:
         for ldjson_node in site.find_all("script", type="application/ld+json"):
             try:
                 ldjson: dict = json.loads(ldjson_node.text.strip())
@@ -80,7 +80,7 @@ class GameDownloader:
         return None
 
     @staticmethod
-    def get_meta(site: BeautifulSoup, **kwargs: Any) -> Optional[str]:  # noqa: ANN401
+    def get_meta(site: BeautifulSoup, **kwargs: Any) -> str | None:  # noqa: ANN401
         """Grabs <meta property="xyz" content="value"/> values."""
         node = site.find("meta", attrs=kwargs)
         if not node:
@@ -89,7 +89,7 @@ class GameDownloader:
         return node.get("content")
 
     def get_game_id(self, url: str, site: BeautifulSoup) -> int:
-        game_id: Optional[int] = None
+        game_id: int | None = None
 
         try:
             # Headers: <meta name="itch:path" content="games/12345" />
@@ -134,14 +134,14 @@ class GameDownloader:
         return game_id
 
     def extract_metadata(self, game_id: int, url: str, site: BeautifulSoup) -> GameMetadata:
-        rating_json: Optional[dict] = self.get_rating_json(site)
+        rating_json: dict | None = self.get_rating_json(site)
         title = rating_json.get("name") if rating_json else None
 
-        description: Optional[str] = self.get_meta(site, property="og:description")
+        description: str | None = self.get_meta(site, property="og:description")
         if not description:
             description = self.get_meta(site, name="description")
 
-        screenshot_urls: List[str] = []
+        screenshot_urls: list[str] = []
         screenshots_node = site.find("div", class_="screenshot_list")
         if screenshots_node:
             screenshot_urls = [a["href"] for a in screenshots_node.find_all("a")]
@@ -193,7 +193,7 @@ class GameDownloader:
 
         return credentials
 
-    def download_file(self, url: str, download_path: Optional[str], credentials: dict) -> str:
+    def download_file(self, url: str, download_path: str | None, credentials: dict) -> str:
         """Performs a request to download a given file, optionally saves the
         file to the provided path and returns the final URL that was downloaded."""
         try:
@@ -216,7 +216,7 @@ class GameDownloader:
         except HTTPError as e:
             raise ItchDownloadError(f"Unrecoverable download error: {e}") from e
 
-    def download_file_by_upload_id(self, upload_id: int, download_path: Optional[str], credentials: dict) -> str:
+    def download_file_by_upload_id(self, upload_id: int, download_path: str | None, credentials: dict) -> str:
         """Performs a request to download a given upload by its ID."""
         return self.download_file(f"/uploads/{upload_id}/download", download_path, credentials)
 
@@ -258,7 +258,7 @@ class GameDownloader:
         download_path = os.path.join(self.settings.download_to, author, game)
         os.makedirs(download_path, exist_ok=True)
 
-        paths: Dict[str, str] = {k: os.path.join(download_path, v) for k, v in TARGET_PATHS.items()}
+        paths: dict[str, str] = {k: os.path.join(download_path, v) for k, v in TARGET_PATHS.items()}
 
         if os.path.exists(paths["metadata"]) and skip_downloaded:
             # As metadata is the final file we write, all the files
@@ -405,9 +405,9 @@ class GameDownloader:
 
 
 def drive_downloads(
-    jobs: List[str],
+    jobs: list[str],
     settings: Settings,
-    keys: Dict[int, str],
+    keys: dict[int, str],
 ) -> None:
     downloader = GameDownloader(settings, keys)
     tqdm_args = {

itch_dl/handlers.py

@@ -2,7 +2,6 @@ import json
 import os.path
 import logging
 import urllib.parse
-from typing import List, Set, Optional
 
 from http.client import responses
 from bs4 import BeautifulSoup
@@ -14,7 +13,7 @@ from .config import Settings
 from .keys import get_owned_games
 
 
-def get_jobs_for_game_jam_json(game_jam_json: dict) -> List[str]:
+def get_jobs_for_game_jam_json(game_jam_json: dict) -> list[str]:
     if "jam_games" not in game_jam_json:
         raise Exception("Provided JSON is not a valid itch.io jam JSON.")
 
@@ -26,7 +25,7 @@ def get_game_jam_json(jam_url: str, client: ItchApiClient) -> dict:
     if not r.ok:
         raise ItchDownloadError(f"Could not download the game jam site: {r.status_code} {r.reason}")
 
-    jam_id: Optional[int] = get_int_after_marker_in_json(r.text, "I.ViewJam", "id")
+    jam_id: int | None = get_int_after_marker_in_json(r.text, "I.ViewJam", "id")
     if jam_id is None:
         raise ItchDownloadError(
             "Provided site did not contain the Game Jam ID. Provide "
@@ -42,7 +41,7 @@ def get_game_jam_json(jam_url: str, client: ItchApiClient) -> dict:
     return r.json()
 
 
-def get_jobs_for_browse_url(url: str, client: ItchApiClient) -> List[str]:
+def get_jobs_for_browse_url(url: str, client: ItchApiClient) -> list[str]:
     """
     Every browser page has a hidden RSS feed that can be accessed by
     appending .xml to its URL. An optional "page" argument lets us
@@ -53,7 +52,7 @@ def get_jobs_for_browse_url(url: str, client: ItchApiClient) -> List[str]:
     .xml?page=N suffix and iterate until we've caught 'em all.
     """
     page = 1
-    found_urls: Set[str] = set()
+    found_urls: set[str] = set()
     logging.info("Scraping game URLs from RSS feeds for %s", url)
 
     while True:
@@ -87,9 +86,9 @@ def get_jobs_for_browse_url(url: str, client: ItchApiClient) -> List[str]:
     return list(found_urls)
 
 
-def get_jobs_for_collection_json(url: str, client: ItchApiClient) -> List[str]:
+def get_jobs_for_collection_json(url: str, client: ItchApiClient) -> list[str]:
     page = 1
-    found_urls: Set[str] = set()
+    found_urls: set[str] = set()
 
     while True:
         logging.info("Downloading page %d (found %d URLs total)", page, len(found_urls))
@@ -118,7 +117,7 @@ def get_jobs_for_collection_json(url: str, client: ItchApiClient) -> List[str]:
     return list(found_urls)
 
 
-def get_jobs_for_creator(creator: str, client: ItchApiClient) -> List[str]:
+def get_jobs_for_creator(creator: str, client: ItchApiClient) -> list[str]:
     logging.info("Downloading public games for creator %s", creator)
     r = client.get(f"https://{ITCH_BASE}/profile/{creator}", append_api_key=False)
     if not r.ok:
@@ -139,7 +138,7 @@ def get_jobs_for_creator(creator: str, client: ItchApiClient) -> List[str]:
     return sorted(game_links)
 
 
-def get_jobs_for_itch_url(url: str, client: ItchApiClient) -> List[str]:
+def get_jobs_for_itch_url(url: str, client: ItchApiClient) -> list[str]:
     if url.startswith("http://"):
         logging.info("HTTP link provided, upgrading to HTTPS")
         url = "https://" + url[7:]
@@ -149,7 +148,7 @@ def get_jobs_for_itch_url(url: str, client: ItchApiClient) -> List[str]:
         url = ITCH_URL + "/" + url[20:]
 
     url_parts = urllib.parse.urlparse(url)
-    url_path_parts: List[str] = [x for x in str(url_parts.path).split("/") if len(x) > 0]
+    url_path_parts: list[str] = [x for x in str(url_parts.path).split("/") if len(x) > 0]
 
     if url_parts.netloc == ITCH_BASE:
         if len(url_path_parts) == 0:
@@ -209,7 +208,7 @@ def get_jobs_for_itch_url(url: str, client: ItchApiClient) -> List[str]:
         raise ValueError(f"Unknown domain: {url_parts.netloc}")
 
 
-def get_jobs_for_path(path: str) -> List[str]:
+def get_jobs_for_path(path: str) -> list[str]:
     try:  # Game Jam Entries JSON?
         with open(path, "rb") as f:
             json_data = json.load(f)
@@ -237,7 +236,7 @@ def get_jobs_for_path(path: str) -> List[str]:
     raise ValueError("File format is unknown - cannot read URLs to download.")
 
 
-def get_jobs_for_url_or_path(path_or_url: str, settings: Settings) -> List[str]:
+def get_jobs_for_url_or_path(path_or_url: str, settings: Settings) -> list[str]:
     """Returns a list of Game URLs for a given itch.io URL or file."""
     path_or_url = path_or_url.strip()
 

itch_dl/infobox.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import TypedDict, Dict, List, Any, Tuple, Optional
+from typing import TypedDict, Any
 
 from bs4 import BeautifulSoup
 
@@ -9,26 +9,26 @@ class InfoboxMetadata(TypedDict, total=False):
     released_at: datetime
     published_at: datetime
     status: str
-    platforms: List[str]  # Windows/macOS/Linux/etc
+    platforms: list[str]  # Windows/macOS/Linux/etc
     publisher: str
-    author: Dict[str, str]  # See impl below!
+    author: dict[str, str]  # See impl below!
-    authors: Dict[str, str]  # Links
+    authors: dict[str, str]  # Links
-    genre: Dict[str, str]  # Links
+    genre: dict[str, str]  # Links
-    tools: Dict[str, str]  # Links
+    tools: dict[str, str]  # Links
-    license: Dict[str, str]  # Links
+    license: dict[str, str]  # Links
-    asset_license: Dict[str, str]  # Links
+    asset_license: dict[str, str]  # Links
-    tags: Dict[str, str]  # Links
+    tags: dict[str, str]  # Links
     length: str
-    multiplayer: Dict[str, str]  # Links
+    multiplayer: dict[str, str]  # Links
     player_count: str
-    accessibility: Dict[str, str]  # Links
+    accessibility: dict[str, str]  # Links
-    inputs: Dict[str, str]  # Links
+    inputs: dict[str, str]  # Links
-    links: Dict[str, str]  # Links
+    links: dict[str, str]  # Links
-    mentions: Dict[str, str]  # Links
+    mentions: dict[str, str]  # Links
-    category: Dict[str, str]  # Links
+    category: dict[str, str]  # Links
 
 
-def parse_date_block(td: BeautifulSoup) -> Optional[datetime]:
+def parse_date_block(td: BeautifulSoup) -> datetime | None:
     abbr = td.find("abbr")
     if not abbr or "title" not in abbr.attrs:
         return None
@@ -39,17 +39,17 @@ def parse_date_block(td: BeautifulSoup) -> Optional[datetime]:
     return datetime(date.year, date.month, date.day, time.hour, time.minute)
 
 
-def parse_links(td: BeautifulSoup) -> Dict[str, str]:
+def parse_links(td: BeautifulSoup) -> dict[str, str]:
     """Parses blocks of comma-separated <a> blocks, returns a dict
     of link text -> URL it points at."""
     return {link.text.strip(): link["href"] for link in td.find_all("a")}
 
 
-def parse_text_from_links(td: BeautifulSoup) -> List[str]:
+def parse_text_from_links(td: BeautifulSoup) -> list[str]:
     return list(parse_links(td).keys())
 
 
-def parse_tr(name: str, content: BeautifulSoup) -> Optional[Tuple[str, Any]]:
+def parse_tr(name: str, content: BeautifulSoup) -> tuple[str, Any] | None:
     if name == "Updated":
         return "updated_at", parse_date_block(content)
     elif name == "Release date":

itch_dl/keys.py

@@ -1,11 +1,10 @@
 import logging
-from typing import Dict, List, Tuple
 
 from .api import ItchApiClient
 
 KEYS_CACHED: bool = False
-DOWNLOAD_KEYS: Dict[int, str] = {}
+DOWNLOAD_KEYS: dict[int, str] = {}
-GAME_URLS: List[str] = []
+GAME_URLS: list[str] = []
 
 
 def load_keys_and_urls(client: ItchApiClient) -> None:
@@ -36,21 +35,21 @@ def load_keys_and_urls(client: ItchApiClient) -> None:
     KEYS_CACHED = True
 
 
-def get_owned_keys(client: ItchApiClient) -> Tuple[Dict[int, str], List[str]]:
+def get_owned_keys(client: ItchApiClient) -> tuple[dict[int, str], list[str]]:
     if not KEYS_CACHED:
         load_keys_and_urls(client)
 
     return DOWNLOAD_KEYS, GAME_URLS
 
 
-def get_download_keys(client: ItchApiClient) -> Dict[int, str]:
+def get_download_keys(client: ItchApiClient) -> dict[int, str]:
     if not KEYS_CACHED:
         load_keys_and_urls(client)
 
     return DOWNLOAD_KEYS
 
 
-def get_owned_games(client: ItchApiClient) -> List[str]:
+def get_owned_games(client: ItchApiClient) -> list[str]:
     if not KEYS_CACHED:
         load_keys_and_urls(client)
 

itch_dl/utils.py

@@ -1,12 +1,11 @@
 import re
-from typing import Optional
 
 
 class ItchDownloadError(Exception):
     pass
 
 
-def get_int_after_marker_in_json(text: str, marker: str, key: str) -> Optional[int]:
+def get_int_after_marker_in_json(text: str, marker: str, key: str) -> int | None:
     """
     Many itch.io sites use a pattern like this: Most of the HTML page
     is prerendered, but certain interactive objects are handled with
@@ -14,7 +13,7 @@ def get_int_after_marker_in_json(text: str, marker: str, key: str) -> Optional[i
     somewhere near the end of each page. Those config blocks often
     contain metadata like game/page IDs that we want to extract.
     """
-    marker_line: Optional[str] = None
+    marker_line: str | None = None
     for line in reversed(text.splitlines()):
         marker_index = line.find(marker)
         if marker_index != -1:

pyproject.toml

@@ -25,7 +25,7 @@ classifiers = [
 "Donate" = "https://ko-fi.com/dragoonaethis"
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.10"
 tqdm = "^4.67.1"
 urllib3 = "^1.26.20"
 requests = "^2.32.3"
@@ -41,7 +41,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.ruff]
 line-length = 120
-target-version = "py38"
+target-version = "py310"
 
 [tool.ruff.lint]
 # https://docs.astral.sh/ruff/rules/