Akamaru/itchdl
1
0
Fork 0
forked from Mirrors/itch-dl
Code Issues Activity

Reformat the codebase with Ruff

Browse Source
This commit is contained in:
Ryszard Knop
2024-03-17 01:17:19 +01:00
parent 25ace8f358
commit 2a18cea131
9 changed files with 109 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
itch_dl/downloader.py
View File

@@ -18,11 +18,11 @@ from .config import Settings
from .infobox import parse_infobox, InfoboxMetadata
TARGET_PATHS = {
'site': 'site.html',
'cover': 'cover',
'metadata': 'metadata.json',
'files': 'files',
'screenshots': 'screenshots'
"site": "site.html",
"cover": "cover",
"metadata": "metadata.json",
"files": "files",
"screenshots": "screenshots",
}
@@ -110,7 +110,7 @@ class GameDownloader:
if game_id is None:
# We have to hit the server again :(
data_url = url.rstrip('/') + "/data.json"
data_url = url.rstrip("/") + "/data.json"
data_request = self.client.get(data_url, append_api_key=False)
if data_request.ok:
try:
@@ -134,7 +134,7 @@ class GameDownloader:
screenshot_urls: List[str] = []
screenshots_node = site.find("div", class_="screenshot_list")
if screenshots_node:
screenshot_urls = [a['href'] for a in screenshots_node.find_all('a')]
screenshot_urls = [a["href"] for a in screenshots_node.find_all("a")]
metadata = GameMetadata(
game_id=game_id,
@@ -148,30 +148,27 @@ class GameDownloader:
infobox_div = site.find("div", class_="game_info_panel_widget")
if infobox_div:
infobox = parse_infobox(infobox_div)
for dt in ('created_at', 'updated_at', 'released_at', 'published_at'):
for dt in ("created_at", "updated_at", "released_at", "published_at"):
if dt in infobox:
metadata[dt] = infobox[dt].isoformat() # noqa (non-literal TypedDict keys)
del infobox[dt] # noqa (non-literal TypedDict keys)
if 'author' in infobox:
metadata['author'] = infobox['author']['author']
metadata['author_url'] = infobox['author']['author_url']
del infobox['author']
if "author" in infobox:
metadata["author"] = infobox["author"]["author"]
metadata["author_url"] = infobox["author"]["author_url"]
del infobox["author"]
if 'authors' in infobox and 'author' not in metadata:
if "authors" in infobox and "author" not in metadata:
# Some games may have multiple authors (ex. compilations).
metadata['author'] = "Multiple authors"
metadata['author_url'] = f"https://{urllib.parse.urlparse(url).netloc}"
metadata["author"] = "Multiple authors"
metadata["author_url"] = f"https://{urllib.parse.urlparse(url).netloc}"
metadata['extra'] = infobox
metadata["extra"] = infobox
agg_rating = rating_json.get('aggregateRating') if rating_json else None
agg_rating = rating_json.get("aggregateRating") if rating_json else None
if agg_rating:
try:
metadata['rating'] = {
'average': float(agg_rating['ratingValue']),
'votes': agg_rating['ratingCount']
}
metadata["rating"] = {"average": float(agg_rating["ratingValue"]), "votes": agg_rating["ratingCount"]}
except: # noqa
logging.exception("Could not extract the rating metadata...")
pass # Nope, just, don't
@@ -181,7 +178,7 @@ class GameDownloader:
def get_credentials(self, title: str, game_id: int) -> dict:
credentials = {}
if game_id in self.download_keys:
credentials['download_key_id'] = self.download_keys[game_id]
credentials["download_key_id"] = self.download_keys[game_id]
logging.debug("Got credentials for %s: %s", title, str(credentials))
return credentials
@@ -195,9 +192,13 @@ class GameDownloader:
r.raise_for_status()
if download_path is not None: # ...and it will be for external downloads.
with tqdm.wrapattr(open(download_path, "wb"), "write",
miniters=1, desc=url,
total=int(r.headers.get('content-length', 0))) as f:
with tqdm.wrapattr(
open(download_path, "wb"),
"write",
miniters=1,
desc=url,
total=int(r.headers.get("content-length", 0)),
) as f:
for chunk in r.iter_content(chunk_size=1048576): # 1MB chunks
f.write(chunk)
@@ -214,14 +215,14 @@ class GameDownloader:
if not match:
return DownloadResult(url, False, [f"Game URL is invalid: {url} - please file a new issue."], [])
author, game = match['author'], match['game']
author, game = match["author"], match["game"]
download_path = os.path.join(self.download_to, author, game)
os.makedirs(download_path, exist_ok=True)
paths: Dict[str, str] = {k: os.path.join(download_path, v) for k, v in TARGET_PATHS.items()}
if os.path.exists(paths['metadata']) and skip_downloaded:
if os.path.exists(paths["metadata"]) and skip_downloaded:
# As metadata is the final file we write, all the files
# should already be downloaded at this point.
logging.info("Skipping already-downloaded game for URL: %s", url)
@@ -238,7 +239,7 @@ class GameDownloader:
try:
game_id = self.get_game_id(url, site)
metadata = self.extract_metadata(game_id, url, site)
title = metadata['title'] or game
title = metadata["title"] or game
except ItchDownloadError as e:
return DownloadResult(url, False, [str(e)], [])
@@ -249,29 +250,32 @@ class GameDownloader:
except Exception as e:
return DownloadResult(url, False, [f"Could not fetch game uploads for {title}: {e}"], [])
game_uploads = game_uploads_req.json()['uploads']
game_uploads = game_uploads_req.json()["uploads"]
logging.debug("Found %d upload(s): %s", len(game_uploads), str(game_uploads))
external_urls = []
errors = []
try:
os.makedirs(paths['files'], exist_ok=True)
os.makedirs(paths["files"], exist_ok=True)
for upload in game_uploads:
if any(key not in upload for key in ('id', 'filename', 'storage')):
if any(key not in upload for key in ("id", "filename", "storage")):
errors.append(f"Upload metadata incomplete: {upload}")
continue
upload_id = upload['id']
file_name = upload['filename']
file_size = upload.get('size')
upload_is_external = upload['storage'] == 'external'
upload_id = upload["id"]
file_name = upload["filename"]
file_size = upload.get("size")
upload_is_external = upload["storage"] == "external"
logging.debug("Downloading '%s' (%d), %s",
file_name, upload_id,
f"{file_size} bytes" if file_size is not None else "unknown size")
logging.debug(
"Downloading '%s' (%d), %s",
file_name,
upload_id,
f"{file_size} bytes" if file_size is not None else "unknown size",
)
target_path = None if upload_is_external else os.path.join(paths['files'], file_name)
target_path = None if upload_is_external else os.path.join(paths["files"], file_name)
try:
target_url = self.download_file_by_upload_id(upload_id, target_path, credentials)
@@ -294,36 +298,36 @@ class GameDownloader:
except Exception as e:
errors.append(f"Download failed for {title}: {e}")
metadata['errors'] = errors
metadata['external_downloads'] = external_urls
metadata["errors"] = errors
metadata["external_downloads"] = external_urls
if len(external_urls) > 0:
logging.warning(f"Game {title} has external download URLs: {external_urls}")
# TODO: Mirror JS/CSS assets
if self.mirror_web:
os.makedirs(paths['screenshots'], exist_ok=True)
for screenshot in metadata['screenshots']:
os.makedirs(paths["screenshots"], exist_ok=True)
for screenshot in metadata["screenshots"]:
if not screenshot:
continue
file_name = os.path.basename(screenshot)
try:
self.download_file(screenshot, os.path.join(paths['screenshots'], file_name), credentials={})
self.download_file(screenshot, os.path.join(paths["screenshots"], file_name), credentials={})
except Exception as e:
errors.append(f"Screenshot download failed (this is not fatal): {e}")
cover_url = metadata.get('cover_url')
cover_url = metadata.get("cover_url")
if cover_url:
try:
self.download_file(cover_url, paths['cover'] + os.path.splitext(cover_url)[-1], credentials={})
self.download_file(cover_url, paths["cover"] + os.path.splitext(cover_url)[-1], credentials={})
except Exception as e:
errors.append(f"Cover art download failed (this is not fatal): {e}")
with open(paths['site'], 'wb') as f:
f.write(site.prettify(encoding='utf-8'))
with open(paths["site"], "wb") as f:
f.write(site.prettify(encoding="utf-8"))
with open(paths['metadata'], 'w') as f:
with open(paths["metadata"], "w") as f:
json.dump(metadata, f, indent=4)
if len(errors) > 0:
@@ -334,12 +338,12 @@ class GameDownloader:
def drive_downloads(
jobs: List[str],
download_to: str,
mirror_web: bool,
settings: Settings,
keys: Dict[int, str],
parallel: int = 1
jobs: List[str],
download_to: str,
mirror_web: bool,
settings: Settings,
keys: Dict[int, str],
parallel: int = 1,
):
downloader = GameDownloader(download_to, mirror_web, settings, keys)
tqdm_args = {