Mirrors/itch-dl
2
0
Fork 0
mirror of https://github.com/DragoonAethis/itch-dl.git synced 2025-02-28 18:43:38 +01:00
Code Issues Packages Projects Releases Wiki Activity

Upgrade to Python 3.10+

Browse Source 
We can use fancier type annotations now, newer deps, more checkers.
This commit is contained in:
Ryszard Knop 2025-02-14 14:55:54 +01:00
parent d307ae8db7
commit cb08443778
8 changed files with 67 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
itch_dl/api.py
View File

@ -1,4 +1,4 @@
from typing import Optional, Any
from typing import Any
import requests
from requests import Session
@ -9,7 +9,7 @@ from .consts import ITCH_API
class ItchApiClient:
def __init__(self, api_key: str, user_agent: str, base_url: Optional[str] = None) -> None:
def __init__(self, api_key: str, user_agent: str, base_url: str | None = None) -> None:
self.base_url = base_url or ITCH_API
self.api_key = api_key

12
itch_dl/config.py
View File

@ -4,7 +4,7 @@ import logging
import platform
import argparse
from dataclasses import dataclass, fields
from typing import Optional, Any, get_type_hints
from typing import Any, get_type_hints
import requests
@ -16,16 +16,16 @@ class Settings:
"""Available settings for itch-dl. Make sure all of them
have default values, as the config file may not exist."""
api_key: Optional[str] = None
api_key: str | None = None
user_agent: str = f"python-requests/{requests.__version__} itch-dl/{__version__}"
download_to: Optional[str] = None
download_to: str | None = None
mirror_web: bool = False
urls_only: bool = False
parallel: int = 1
filter_files_glob: Optional[str] = None
filter_files_regex: Optional[str] = None
filter_files_glob: str | None = None
filter_files_regex: str | None = None
verbose: bool = False
@ -73,7 +73,7 @@ def clean_config(config_data: dict[str, Any]) -> dict[str, Any]:
return cleaned_config
def load_config(args: argparse.Namespace, profile: Optional[str] = None) -> Settings:
def load_config(args: argparse.Namespace, profile: str | None = None) -> Settings:
"""Loads the configuration from the file system if it exists,
the returns a Settings object."""
config_path = create_and_get_config_path()

40
itch_dl/downloader.py
View File

@ -6,7 +6,7 @@ import logging
import urllib.parse
import zipfile
import tarfile
from typing import List, Dict, TypedDict, Optional, Union, Any
from typing import TypedDict, Any
from bs4 import BeautifulSoup
from requests.exceptions import HTTPError, JSONDecodeError
@ -30,7 +30,7 @@ TARGET_PATHS = {
class DownloadResult:
def __init__(self, url: str, success: bool, errors: Optional[List[str]], external_urls: List[str]) -> None:
def __init__(self, url: str, success: bool, errors: list[str] | None, external_urls: list[str]) -> None:
self.url = url
self.success = success
self.errors = errors or []
@ -42,17 +42,17 @@ class GameMetadata(TypedDict, total=False):
title: str
url: str
errors: List[str]
external_downloads: List[str]
errors: list[str]
external_downloads: list[str]
author: str
author_url: str
cover_url: Optional[str]
screenshots: List[str]
description: Optional[str]
cover_url: str | None
screenshots: list[str]
description: str | None
rating: Dict[str, Union[float, int]]
rating: dict[str, float | int]
extra: InfoboxMetadata
created_at: str
@ -62,13 +62,13 @@ class GameMetadata(TypedDict, total=False):
class GameDownloader:
def __init__(self, settings: Settings, keys: Dict[int, str]) -> None:
def __init__(self, settings: Settings, keys: dict[int, str]) -> None:
self.settings = settings
self.download_keys = keys
self.client = ItchApiClient(settings.api_key, settings.user_agent)
@staticmethod
def get_rating_json(site: BeautifulSoup) -> Optional[dict]:
def get_rating_json(site: BeautifulSoup) -> dict | None:
for ldjson_node in site.find_all("script", type="application/ld+json"):
try:
ldjson: dict = json.loads(ldjson_node.text.strip())
@ -80,7 +80,7 @@ class GameDownloader:
return None
@staticmethod
def get_meta(site: BeautifulSoup, **kwargs: Any) -> Optional[str]: # noqa: ANN401
def get_meta(site: BeautifulSoup, **kwargs: Any) -> str | None: # noqa: ANN401
"""Grabs <meta property="xyz" content="value"/> values."""
node = site.find("meta", attrs=kwargs)
if not node:
@ -89,7 +89,7 @@ class GameDownloader:
return node.get("content")
def get_game_id(self, url: str, site: BeautifulSoup) -> int:
game_id: Optional[int] = None
game_id: int | None = None
try:
# Headers: <meta name="itch:path" content="games/12345" />
@ -134,14 +134,14 @@ class GameDownloader:
return game_id
def extract_metadata(self, game_id: int, url: str, site: BeautifulSoup) -> GameMetadata:
rating_json: Optional[dict] = self.get_rating_json(site)
rating_json: dict | None = self.get_rating_json(site)
title = rating_json.get("name") if rating_json else None
description: Optional[str] = self.get_meta(site, property="og:description")
description: str | None = self.get_meta(site, property="og:description")
if not description:
description = self.get_meta(site, name="description")
screenshot_urls: List[str] = []
screenshot_urls: list[str] = []
screenshots_node = site.find("div", class_="screenshot_list")
if screenshots_node:
screenshot_urls = [a["href"] for a in screenshots_node.find_all("a")]
@ -193,7 +193,7 @@ class GameDownloader:
return credentials
def download_file(self, url: str, download_path: Optional[str], credentials: dict) -> str:
def download_file(self, url: str, download_path: str | None, credentials: dict) -> str:
"""Performs a request to download a given file, optionally saves the
file to the provided path and returns the final URL that was downloaded."""
try:
@ -216,7 +216,7 @@ class GameDownloader:
except HTTPError as e:
raise ItchDownloadError(f"Unrecoverable download error: {e}") from e
def download_file_by_upload_id(self, upload_id: int, download_path: Optional[str], credentials: dict) -> str:
def download_file_by_upload_id(self, upload_id: int, download_path: str | None, credentials: dict) -> str:
"""Performs a request to download a given upload by its ID."""
return self.download_file(f"/uploads/{upload_id}/download", download_path, credentials)
@ -258,7 +258,7 @@ class GameDownloader:
download_path = os.path.join(self.settings.download_to, author, game)
os.makedirs(download_path, exist_ok=True)
paths: Dict[str, str] = {k: os.path.join(download_path, v) for k, v in TARGET_PATHS.items()}
paths: dict[str, str] = {k: os.path.join(download_path, v) for k, v in TARGET_PATHS.items()}
if os.path.exists(paths["metadata"]) and skip_downloaded:
# As metadata is the final file we write, all the files
@ -405,9 +405,9 @@ class GameDownloader:
def drive_downloads(
jobs: List[str],
jobs: list[str],
settings: Settings,
keys: Dict[int, str],
keys: dict[int, str],
) -> None:
downloader = GameDownloader(settings, keys)
tqdm_args = {

23
itch_dl/handlers.py
View File

@ -2,7 +2,6 @@ import json
import os.path
import logging
import urllib.parse
from typing import List, Set, Optional
from http.client import responses
from bs4 import BeautifulSoup
@ -14,7 +13,7 @@ from .config import Settings
from .keys import get_owned_games
def get_jobs_for_game_jam_json(game_jam_json: dict) -> List[str]:
def get_jobs_for_game_jam_json(game_jam_json: dict) -> list[str]:
if "jam_games" not in game_jam_json:
raise Exception("Provided JSON is not a valid itch.io jam JSON.")
@ -26,7 +25,7 @@ def get_game_jam_json(jam_url: str, client: ItchApiClient) -> dict:
if not r.ok:
raise ItchDownloadError(f"Could not download the game jam site: {r.status_code} {r.reason}")
jam_id: Optional[int] = get_int_after_marker_in_json(r.text, "I.ViewJam", "id")
jam_id: int | None = get_int_after_marker_in_json(r.text, "I.ViewJam", "id")
if jam_id is None:
raise ItchDownloadError(
"Provided site did not contain the Game Jam ID. Provide "
@ -42,7 +41,7 @@ def get_game_jam_json(jam_url: str, client: ItchApiClient) -> dict:
return r.json()
def get_jobs_for_browse_url(url: str, client: ItchApiClient) -> List[str]:
def get_jobs_for_browse_url(url: str, client: ItchApiClient) -> list[str]:
"""
Every browser page has a hidden RSS feed that can be accessed by
appending .xml to its URL. An optional "page" argument lets us
@ -53,7 +52,7 @@ def get_jobs_for_browse_url(url: str, client: ItchApiClient) -> List[str]:
.xml?page=N suffix and iterate until we've caught 'em all.
"""
page = 1
found_urls: Set[str] = set()
found_urls: set[str] = set()
logging.info("Scraping game URLs from RSS feeds for %s", url)
while True:
@ -87,9 +86,9 @@ def get_jobs_for_browse_url(url: str, client: ItchApiClient) -> List[str]:
return list(found_urls)
def get_jobs_for_collection_json(url: str, client: ItchApiClient) -> List[str]:
def get_jobs_for_collection_json(url: str, client: ItchApiClient) -> list[str]:
page = 1
found_urls: Set[str] = set()
found_urls: set[str] = set()
while True:
logging.info("Downloading page %d (found %d URLs total)", page, len(found_urls))
@ -118,7 +117,7 @@ def get_jobs_for_collection_json(url: str, client: ItchApiClient) -> List[str]:
return list(found_urls)
def get_jobs_for_creator(creator: str, client: ItchApiClient) -> List[str]:
def get_jobs_for_creator(creator: str, client: ItchApiClient) -> list[str]:
logging.info("Downloading public games for creator %s", creator)
r = client.get(f"https://{ITCH_BASE}/profile/{creator}", append_api_key=False)
if not r.ok:
@ -139,7 +138,7 @@ def get_jobs_for_creator(creator: str, client: ItchApiClient) -> List[str]:
return sorted(game_links)
def get_jobs_for_itch_url(url: str, client: ItchApiClient) -> List[str]:
def get_jobs_for_itch_url(url: str, client: ItchApiClient) -> list[str]:
if url.startswith("http://"):
logging.info("HTTP link provided, upgrading to HTTPS")
url = "https://" + url[7:]
@ -149,7 +148,7 @@ def get_jobs_for_itch_url(url: str, client: ItchApiClient) -> List[str]:
url = ITCH_URL + "/" + url[20:]
url_parts = urllib.parse.urlparse(url)
url_path_parts: List[str] = [x for x in str(url_parts.path).split("/") if len(x) > 0]
url_path_parts: list[str] = [x for x in str(url_parts.path).split("/") if len(x) > 0]
if url_parts.netloc == ITCH_BASE:
if len(url_path_parts) == 0:
@ -209,7 +208,7 @@ def get_jobs_for_itch_url(url: str, client: ItchApiClient) -> List[str]:
raise ValueError(f"Unknown domain: {url_parts.netloc}")
def get_jobs_for_path(path: str) -> List[str]:
def get_jobs_for_path(path: str) -> list[str]:
try: # Game Jam Entries JSON?
with open(path, "rb") as f:
json_data = json.load(f)
@ -237,7 +236,7 @@ def get_jobs_for_path(path: str) -> List[str]:
raise ValueError("File format is unknown - cannot read URLs to download.")
def get_jobs_for_url_or_path(path_or_url: str, settings: Settings) -> List[str]:
def get_jobs_for_url_or_path(path_or_url: str, settings: Settings) -> list[str]:
"""Returns a list of Game URLs for a given itch.io URL or file."""
path_or_url = path_or_url.strip()

38
itch_dl/infobox.py
View File

@ -1,5 +1,5 @@
from datetime import datetime
from typing import TypedDict, Dict, List, Any, Tuple, Optional
from typing import TypedDict, Any
from bs4 import BeautifulSoup
@ -9,26 +9,26 @@ class InfoboxMetadata(TypedDict, total=False):
released_at: datetime
published_at: datetime
status: str
platforms: List[str] # Windows/macOS/Linux/etc
platforms: list[str] # Windows/macOS/Linux/etc
publisher: str
author: Dict[str, str] # See impl below!
authors: Dict[str, str] # Links
genre: Dict[str, str] # Links
tools: Dict[str, str] # Links
license: Dict[str, str] # Links
asset_license: Dict[str, str] # Links
tags: Dict[str, str] # Links
author: dict[str, str] # See impl below!
authors: dict[str, str] # Links
genre: dict[str, str] # Links
tools: dict[str, str] # Links
license: dict[str, str] # Links
asset_license: dict[str, str] # Links
tags: dict[str, str] # Links
length: str
multiplayer: Dict[str, str] # Links
multiplayer: dict[str, str] # Links
player_count: str
accessibility: Dict[str, str] # Links
inputs: Dict[str, str] # Links
links: Dict[str, str] # Links
mentions: Dict[str, str] # Links
category: Dict[str, str] # Links
accessibility: dict[str, str] # Links
inputs: dict[str, str] # Links
links: dict[str, str] # Links
mentions: dict[str, str] # Links
category: dict[str, str] # Links
def parse_date_block(td: BeautifulSoup) -> Optional[datetime]:
def parse_date_block(td: BeautifulSoup) -> datetime | None:
abbr = td.find("abbr")
if not abbr or "title" not in abbr.attrs:
return None
@ -39,17 +39,17 @@ def parse_date_block(td: BeautifulSoup) -> Optional[datetime]:
return datetime(date.year, date.month, date.day, time.hour, time.minute)
def parse_links(td: BeautifulSoup) -> Dict[str, str]:
def parse_links(td: BeautifulSoup) -> dict[str, str]:
"""Parses blocks of comma-separated <a> blocks, returns a dict
of link text -> URL it points at."""
return {link.text.strip(): link["href"] for link in td.find_all("a")}
def parse_text_from_links(td: BeautifulSoup) -> List[str]:
def parse_text_from_links(td: BeautifulSoup) -> list[str]:
return list(parse_links(td).keys())
def parse_tr(name: str, content: BeautifulSoup) -> Optional[Tuple[str, Any]]:
def parse_tr(name: str, content: BeautifulSoup) -> tuple[str, Any] | None:
if name == "Updated":
return "updated_at", parse_date_block(content)
elif name == "Release date":

11
itch_dl/keys.py
View File

@ -1,11 +1,10 @@
import logging
from typing import Dict, List, Tuple
from .api import ItchApiClient
KEYS_CACHED: bool = False
DOWNLOAD_KEYS: Dict[int, str] = {}
GAME_URLS: List[str] = []
DOWNLOAD_KEYS: dict[int, str] = {}
GAME_URLS: list[str] = []
def load_keys_and_urls(client: ItchApiClient) -> None:
@ -36,21 +35,21 @@ def load_keys_and_urls(client: ItchApiClient) -> None:
KEYS_CACHED = True
def get_owned_keys(client: ItchApiClient) -> Tuple[Dict[int, str], List[str]]:
def get_owned_keys(client: ItchApiClient) -> tuple[dict[int, str], list[str]]:
if not KEYS_CACHED:
load_keys_and_urls(client)
return DOWNLOAD_KEYS, GAME_URLS
def get_download_keys(client: ItchApiClient) -> Dict[int, str]:
def get_download_keys(client: ItchApiClient) -> dict[int, str]:
if not KEYS_CACHED:
load_keys_and_urls(client)
return DOWNLOAD_KEYS
def get_owned_games(client: ItchApiClient) -> List[str]:
def get_owned_games(client: ItchApiClient) -> list[str]:
if not KEYS_CACHED:
load_keys_and_urls(client)

5
itch_dl/utils.py
View File

@ -1,12 +1,11 @@
import re
from typing import Optional
class ItchDownloadError(Exception):
pass
def get_int_after_marker_in_json(text: str, marker: str, key: str) -> Optional[int]:
def get_int_after_marker_in_json(text: str, marker: str, key: str) -> int | None:
"""
Many itch.io sites use a pattern like this: Most of the HTML page
is prerendered, but certain interactive objects are handled with
@ -14,7 +13,7 @@ def get_int_after_marker_in_json(text: str, marker: str, key: str) -> Optional[i
somewhere near the end of each page. Those config blocks often
contain metadata like game/page IDs that we want to extract.
"""
marker_line: Optional[str] = None
marker_line: str | None = None
for line in reversed(text.splitlines()):
marker_index = line.find(marker)
if marker_index != -1:

4
pyproject.toml
View File

@ -25,7 +25,7 @@ classifiers = [
"Donate" = "https://ko-fi.com/dragoonaethis"
[tool.poetry.dependencies]
python = "^3.8"
python = "^3.10"
tqdm = "^4.67.1"
urllib3 = "^1.26.20"
requests = "^2.32.3"
@ -41,7 +41,7 @@ build-backend = "poetry.core.masonry.api"
[tool.ruff]
line-length = 120
target-version = "py38"
target-version = "py310"
[tool.ruff.lint]
# https://docs.astral.sh/ruff/rules/