diff --git a/itch_dl/__init__.py b/itch_dl/__init__.py index fc79d63..0404d81 100644 --- a/itch_dl/__init__.py +++ b/itch_dl/__init__.py @@ -1 +1 @@ -__version__ = '0.2.1' +__version__ = '0.3.0' diff --git a/itch_dl/api.py b/itch_dl/api.py index 1fd8005..f416a63 100644 --- a/itch_dl/api.py +++ b/itch_dl/api.py @@ -1,5 +1,6 @@ from typing import Optional +import requests from requests import Session from urllib3.util.retry import Retry from requests.adapters import HTTPAdapter @@ -27,7 +28,7 @@ class ItchApiClient: self.requests.mount("https://", adapter) self.requests.mount("http://", adapter) - def get(self, endpoint: str, append_api_key: bool = True, **kwargs): + def get(self, endpoint: str, append_api_key: bool = True, **kwargs) -> requests.Response: if append_api_key: params = kwargs.get('data') or {} diff --git a/itch_dl/downloader.py b/itch_dl/downloader.py index 6f41c8f..3923852 100644 --- a/itch_dl/downloader.py +++ b/itch_dl/downloader.py @@ -45,9 +45,9 @@ class GameMetadata(TypedDict, total=False): author: str author_url: str - cover_url: str + cover_url: Optional[str] screenshots: List[str] - description: str + description: Optional[str] rating: Dict[str, Union[float, int]] extra: InfoboxMetadata @@ -125,7 +125,7 @@ class GameDownloader: def extract_metadata(self, game_id: int, url: str, site: BeautifulSoup) -> GameMetadata: rating_json: Optional[dict] = self.get_rating_json(site) - title = rating_json.get("name") + title = rating_json.get("name") if rating_json else None description: Optional[str] = self.get_meta(site, property="og:description") if not description: @@ -150,9 +150,8 @@ class GameDownloader: infobox = parse_infobox(infobox_div) for dt in ('created_at', 'updated_at', 'released_at', 'published_at'): if dt in infobox: - # noinspection PyTypedDict - metadata[dt] = infobox[dt].isoformat() - del infobox[dt] + metadata[dt] = infobox[dt].isoformat() # noqa (non-literal TypedDict keys) + del infobox[dt] # noqa (non-literal TypedDict keys) if 'author' in infobox: metadata['author'] = infobox['author']['author'] @@ -166,7 +165,7 @@ class GameDownloader: metadata['extra'] = infobox - agg_rating = rating_json.get('aggregateRating') + agg_rating = rating_json.get('aggregateRating') if rating_json else None if agg_rating: try: metadata['rating'] = { @@ -269,8 +268,8 @@ class GameDownloader: upload_is_external = upload['storage'] == 'external' logging.debug("Downloading '%s' (%d), %s", - file_name, upload_id, - f"{file_size} bytes" if file_size is not None else "unknown size") + file_name, upload_id, + f"{file_size} bytes" if file_size is not None else "unknown size") target_path = None if upload_is_external else os.path.join(paths['files'], file_name) @@ -286,8 +285,8 @@ class GameDownloader: try: downloaded_file_size = os.stat(target_path).st_size - if file_size is not None and downloaded_file_size != file_size: - errors.append(f"File size is {downloaded_file_size}, but expected {file_size} for upload {upload}") + if target_path is not None and file_size is not None and downloaded_file_size != file_size: + errors.append(f"File size is {downloaded_file_size}, expected {file_size} for upload {upload}") except FileNotFoundError: errors.append(f"Downloaded file not found for upload {upload}") @@ -314,9 +313,9 @@ class GameDownloader: except Exception as e: errors.append(f"Screenshot download failed (this is not fatal): {e}") - if metadata.get('cover_url'): + cover_url = metadata.get('cover_url') + if cover_url: try: - cover_url = metadata['cover_url'] self.download_file(cover_url, paths['cover'] + os.path.splitext(cover_url)[-1], credentials={}) except Exception as e: errors.append(f"Cover art download failed (this is not fatal): {e}") diff --git a/itch_dl/handlers.py b/itch_dl/handlers.py index b17abb5..ce1d5c1 100644 --- a/itch_dl/handlers.py +++ b/itch_dl/handlers.py @@ -2,7 +2,7 @@ import json import os.path import logging import urllib.parse -from typing import List, Optional +from typing import List, Set, Optional from bs4 import BeautifulSoup @@ -49,7 +49,7 @@ def get_jobs_for_browse_url(url: str, client: ItchApiClient) -> List[str]: .xml?page=N suffix and iterate until we've caught 'em all. """ page = 1 - found_urls = set() + found_urls: Set[str] = set() logging.info(f"Scraping game URLs from RSS feeds for %s", url) while True: @@ -189,3 +189,5 @@ def get_jobs_for_url_or_path(path_or_url: str, settings: Settings) -> List[str]: return get_jobs_for_itch_url(path_or_url, client) elif os.path.isfile(path_or_url): return get_jobs_for_path(path_or_url) + else: + raise NotImplementedError(f"Cannot handle path or URL: {path_or_url}") diff --git a/itch_dl/infobox.py b/itch_dl/infobox.py index e644e72..c59dcfe 100644 --- a/itch_dl/infobox.py +++ b/itch_dl/infobox.py @@ -103,7 +103,7 @@ def parse_tr(name: str, content: BeautifulSoup) -> Optional[Tuple[str, Any]]: raise NotImplementedError(f"Unknown infobox block name '{name}' - please file a new itch-dl issue.") -def parse_infobox(infobox: BeautifulSoup) -> dict: +def parse_infobox(infobox: BeautifulSoup) -> InfoboxMetadata: """Feed it