forked from Mirrors/itch-dl
New Flag to force re-downloading cover/screenshots
This commit is contained in:
Akamaru
@@ -47,6 +47,8 @@ def parse_args() -> argparse.Namespace:
|
||||
help="directory to save results into (default: current working dir)")
|
||||
parser.add_argument("--mirror-web", action="store_true",
|
||||
help="try to fetch assets on game sites")
|
||||
parser.add_argument("--refresh-media", action="store_true",
|
||||
help="force re-download cover images and screenshots (skips game files if already downloaded)")
|
||||
parser.add_argument("--urls-only", action="store_true",
|
||||
help="print scraped game URLs without downloading them")
|
||||
parser.add_argument("--parallel", metavar="parallel", type=int, default=None,
|
||||
|
||||
@@ -21,6 +21,7 @@ class Settings:
|
||||
|
||||
download_to: str | None = None
|
||||
mirror_web: bool = False
|
||||
refresh_media: bool = False
|
||||
urls_only: bool = False
|
||||
parallel: int = 1
|
||||
|
||||
|
||||
@@ -259,12 +259,19 @@ class GameDownloader:
|
||||
|
||||
paths: dict[str, str] = {k: os.path.join(download_path, v) for k, v in TARGET_PATHS.items()}
|
||||
|
||||
if os.path.exists(paths["metadata"]) and skip_downloaded:
|
||||
# Check if we should skip or only refresh media
|
||||
game_already_downloaded = os.path.exists(paths["metadata"])
|
||||
media_only_mode = game_already_downloaded and self.settings.refresh_media
|
||||
|
||||
if game_already_downloaded and skip_downloaded and not self.settings.refresh_media:
|
||||
# As metadata is the final file we write, all the files
|
||||
# should already be downloaded at this point.
|
||||
logging.info("Skipping already-downloaded game for URL: %s", url)
|
||||
return DownloadResult(url, True, ["Game already downloaded."], [])
|
||||
|
||||
if media_only_mode:
|
||||
logging.info("Refreshing media for already-downloaded game: %s", url)
|
||||
|
||||
try:
|
||||
logging.info("Downloading %s", url)
|
||||
r = self.client.get(url, append_api_key=False)
|
||||
@@ -293,82 +300,83 @@ class GameDownloader:
|
||||
external_urls = []
|
||||
errors = []
|
||||
|
||||
try:
|
||||
os.makedirs(paths["files"], exist_ok=True)
|
||||
for upload in game_uploads:
|
||||
if any(key not in upload for key in ("id", "filename", "type", "traits", "storage")):
|
||||
errors.append(f"Upload metadata incomplete: {upload}")
|
||||
continue
|
||||
if not media_only_mode:
|
||||
try:
|
||||
os.makedirs(paths["files"], exist_ok=True)
|
||||
for upload in game_uploads:
|
||||
if any(key not in upload for key in ("id", "filename", "type", "traits", "storage")):
|
||||
errors.append(f"Upload metadata incomplete: {upload}")
|
||||
continue
|
||||
|
||||
logging.debug(upload)
|
||||
upload_id = upload["id"]
|
||||
file_name = upload["filename"]
|
||||
file_type = upload["type"]
|
||||
file_traits = upload["traits"]
|
||||
expected_size = upload.get("size")
|
||||
upload_is_external = upload["storage"] == "external"
|
||||
logging.debug(upload)
|
||||
upload_id = upload["id"]
|
||||
file_name = upload["filename"]
|
||||
file_type = upload["type"]
|
||||
file_traits = upload["traits"]
|
||||
expected_size = upload.get("size")
|
||||
upload_is_external = upload["storage"] == "external"
|
||||
|
||||
if self.settings.filter_files_type and file_type not in self.settings.filter_files_type:
|
||||
logging.info("File '%s' has ignored type '%s', skipping", file_name, file_type)
|
||||
continue
|
||||
if self.settings.filter_files_type and file_type not in self.settings.filter_files_type:
|
||||
logging.info("File '%s' has ignored type '%s', skipping", file_name, file_type)
|
||||
continue
|
||||
|
||||
if (
|
||||
self.settings.filter_files_platform
|
||||
and file_type == "default"
|
||||
and not any(trait in self.settings.filter_files_platform for trait in file_traits)
|
||||
):
|
||||
# Setup for filter_files_platform is in config.py, including the trait listing.
|
||||
logging.info("File '%s' not for requested platforms, skipping", file_name)
|
||||
continue
|
||||
if (
|
||||
self.settings.filter_files_platform
|
||||
and file_type == "default"
|
||||
and not any(trait in self.settings.filter_files_platform for trait in file_traits)
|
||||
):
|
||||
# Setup for filter_files_platform is in config.py, including the trait listing.
|
||||
logging.info("File '%s' not for requested platforms, skipping", file_name)
|
||||
continue
|
||||
|
||||
if should_skip_item_by_glob("File", file_name, self.settings.filter_files_glob):
|
||||
continue
|
||||
if should_skip_item_by_glob("File", file_name, self.settings.filter_files_glob):
|
||||
continue
|
||||
|
||||
if should_skip_item_by_regex("File", file_name, self.settings.filter_files_regex):
|
||||
continue
|
||||
if should_skip_item_by_regex("File", file_name, self.settings.filter_files_regex):
|
||||
continue
|
||||
|
||||
logging.debug(
|
||||
"Downloading '%s' (%d), %s",
|
||||
file_name,
|
||||
upload_id,
|
||||
f"{expected_size} bytes" if expected_size is not None else "unknown size",
|
||||
)
|
||||
|
||||
target_path = None if upload_is_external else os.path.join(paths["files"], file_name)
|
||||
|
||||
try:
|
||||
target_url = self.download_file_by_upload_id(upload_id, target_path, credentials)
|
||||
except ItchDownloadError as e:
|
||||
errors.append(f"Download failed for upload {upload}: {e}")
|
||||
continue
|
||||
|
||||
if upload_is_external:
|
||||
logging.debug("Found external download URL for %s: %s", title, target_url)
|
||||
external_urls.append(target_url)
|
||||
continue
|
||||
|
||||
try:
|
||||
downloaded_file_stat = os.stat(target_path)
|
||||
except FileNotFoundError:
|
||||
errors.append(f"Downloaded file not found for upload {upload}")
|
||||
continue
|
||||
|
||||
downloaded_size = downloaded_file_stat.st_size
|
||||
content_size = self.get_decompressed_content_size(target_path)
|
||||
|
||||
if (
|
||||
all(x is not None for x in (target_path, expected_size, downloaded_size))
|
||||
and downloaded_size != expected_size
|
||||
and content_size != expected_size
|
||||
):
|
||||
errors.append(
|
||||
f"Downloaded file size is {downloaded_size} (content {content_size}), "
|
||||
f"expected {expected_size} for upload {upload}"
|
||||
logging.debug(
|
||||
"Downloading '%s' (%d), %s",
|
||||
file_name,
|
||||
upload_id,
|
||||
f"{expected_size} bytes" if expected_size is not None else "unknown size",
|
||||
)
|
||||
|
||||
logging.debug("Done downloading files for %s", title)
|
||||
except Exception as e:
|
||||
errors.append(f"Download failed for {title}: {e}")
|
||||
target_path = None if upload_is_external else os.path.join(paths["files"], file_name)
|
||||
|
||||
try:
|
||||
target_url = self.download_file_by_upload_id(upload_id, target_path, credentials)
|
||||
except ItchDownloadError as e:
|
||||
errors.append(f"Download failed for upload {upload}: {e}")
|
||||
continue
|
||||
|
||||
if upload_is_external:
|
||||
logging.debug("Found external download URL for %s: %s", title, target_url)
|
||||
external_urls.append(target_url)
|
||||
continue
|
||||
|
||||
try:
|
||||
downloaded_file_stat = os.stat(target_path)
|
||||
except FileNotFoundError:
|
||||
errors.append(f"Downloaded file not found for upload {upload}")
|
||||
continue
|
||||
|
||||
downloaded_size = downloaded_file_stat.st_size
|
||||
content_size = self.get_decompressed_content_size(target_path)
|
||||
|
||||
if (
|
||||
all(x is not None for x in (target_path, expected_size, downloaded_size))
|
||||
and downloaded_size != expected_size
|
||||
and content_size != expected_size
|
||||
):
|
||||
errors.append(
|
||||
f"Downloaded file size is {downloaded_size} (content {content_size}), "
|
||||
f"expected {expected_size} for upload {upload}"
|
||||
)
|
||||
|
||||
logging.debug("Done downloading files for %s", title)
|
||||
except Exception as e:
|
||||
errors.append(f"Download failed for {title}: {e}")
|
||||
|
||||
metadata["errors"] = errors
|
||||
metadata["external_downloads"] = external_urls
|
||||
@@ -384,17 +392,34 @@ class GameDownloader:
|
||||
continue
|
||||
|
||||
file_name = os.path.basename(screenshot)
|
||||
try:
|
||||
self.download_file(screenshot, os.path.join(paths["screenshots"], file_name), credentials={})
|
||||
except Exception as e:
|
||||
errors.append(f"Screenshot download failed (this is not fatal): {e}")
|
||||
screenshot_path = os.path.join(paths["screenshots"], file_name)
|
||||
|
||||
# Check if screenshot already exists and if we should skip or force re-download
|
||||
screenshot_exists = os.path.exists(screenshot_path)
|
||||
should_download_screenshot = not screenshot_exists or self.settings.refresh_media
|
||||
|
||||
if should_download_screenshot:
|
||||
try:
|
||||
self.download_file(screenshot, screenshot_path, credentials={})
|
||||
except Exception as e:
|
||||
errors.append(f"Screenshot download failed (this is not fatal): {e}")
|
||||
else:
|
||||
logging.debug("Screenshot '%s' already exists, skipping download", file_name)
|
||||
|
||||
cover_url = metadata.get("cover_url")
|
||||
if cover_url:
|
||||
try:
|
||||
self.download_file(cover_url, paths["cover"] + os.path.splitext(cover_url)[-1], credentials={})
|
||||
except Exception as e:
|
||||
errors.append(f"Cover art download failed (this is not fatal): {e}")
|
||||
cover_path = paths["cover"] + os.path.splitext(cover_url)[-1]
|
||||
# Check if cover already exists and if we should skip or force re-download
|
||||
cover_exists = os.path.exists(cover_path)
|
||||
should_download_cover = not cover_exists or self.settings.refresh_media
|
||||
|
||||
if should_download_cover:
|
||||
try:
|
||||
self.download_file(cover_url, cover_path, credentials={})
|
||||
except Exception as e:
|
||||
errors.append(f"Cover art download failed (this is not fatal): {e}")
|
||||
else:
|
||||
logging.debug("Cover already exists, skipping download")
|
||||
|
||||
with open(paths["site"], "wb") as f:
|
||||
f.write(site.prettify(encoding="utf-8"))
|
||||
|
||||
Reference in New Issue
Block a user