diff --git a/itch_dl/cli.py b/itch_dl/cli.py index ba2a8bb..b28ee66 100644 --- a/itch_dl/cli.py +++ b/itch_dl/cli.py @@ -3,7 +3,7 @@ import sys import logging import argparse -from .handlers import get_jobs_for_url_or_path +from .handlers import get_jobs_for_url_or_path, preprocess_job_urls from .downloader import drive_downloads from .config import Settings, load_config from .keys import get_download_keys @@ -50,6 +50,11 @@ def parse_args() -> argparse.Namespace: help="print scraped game URLs without downloading them") parser.add_argument("--parallel", metavar="parallel", type=int, default=None, help="how many threads to use for downloading games (default: 1)") + + parser.add_argument("--filter-files-platform", metavar="platforms", action="extend", nargs='+', + help="filter downloaded files by platform (windows, mac, linux, android, native)") + parser.add_argument("--filter-files-type", metavar="types", action="extend", nargs='+', + help="filter downloaded files by type (see wiki for valid values)") parser.add_argument("--filter-files-glob", metavar="glob", default=None, help="filter downloaded files with a shell-style glob/fnmatch (unmatched files are skipped)") parser.add_argument("--filter-files-regex", metavar="regex", default=None, diff --git a/itch_dl/config.py b/itch_dl/config.py index de149f4..19aee18 100644 --- a/itch_dl/config.py +++ b/itch_dl/config.py @@ -24,6 +24,8 @@ class Settings: urls_only: bool = False parallel: int = 1 + filter_files_platform: list | None = None + filter_files_type: list | None = None filter_files_glob: str | None = None filter_files_regex: str | None = None @@ -33,6 +35,44 @@ class Settings: verbose: bool = False +def process_platform_traits(platforms: list[str]) -> list[str] | None: + """Converts the user-friendly platform strings into itch.io upload p_traits.""" + if not platforms: + return None + + trait_mapping = { + "win": "p_windows", + "lin": "p_linux", + "mac": "p_osx", + "osx": "p_osx", + "darwin": "p_osx", + "and": "p_android", + } + + traits = set() + for p in platforms: + platform_trait = None + p = p.strip().lower().removeprefix("p_") + + if p.startswith("native"): + p = platform.system().lower() + if p.endswith("bsd"): + logging.warning("Note: Native downloads for *BSDs are not available - Linux binaries will be used.") + p = "linux" + + for key, trait in trait_mapping.items(): + if p.startswith(key): + platform_trait = trait + break + + if not platform_trait: + raise ValueError(f"Platform {p} not known!") + + traits.add(platform_trait) + + return list(traits) + + def create_and_get_config_path() -> str: """Returns the configuration directory in the appropriate location for the current OS. The directory may not exist.""" @@ -106,4 +146,7 @@ def load_config(args: argparse.Namespace, profile: str | None = None) -> Setting if value := getattr(args, key): setattr(settings, key, value) + # Extra handling for special settings: + settings.filter_files_platform = process_platform_traits(settings.filter_files_platform) + return settings diff --git a/itch_dl/downloader.py b/itch_dl/downloader.py index ba89f16..5faa2d5 100644 --- a/itch_dl/downloader.py +++ b/itch_dl/downloader.py @@ -15,7 +15,7 @@ from tqdm import tqdm from tqdm.contrib.concurrent import thread_map from .api import ItchApiClient -from .utils import ItchDownloadError, get_int_after_marker_in_json +from .utils import ItchDownloadError, get_int_after_marker_in_json, should_skip_item_by_glob, should_skip_item_by_regex from .consts import ITCH_GAME_URL_REGEX from .config import Settings from .infobox import parse_infobox, InfoboxMetadata @@ -297,29 +297,35 @@ class GameDownloader: try: os.makedirs(paths["files"], exist_ok=True) for upload in game_uploads: - if any(key not in upload for key in ("id", "filename", "storage")): + if any(key not in upload for key in ("id", "filename", "type", "traits", "storage")): errors.append(f"Upload metadata incomplete: {upload}") continue + logging.info(upload) upload_id = upload["id"] file_name = upload["filename"] + file_type = upload["type"] + file_traits = upload["traits"] expected_size = upload.get("size") upload_is_external = upload["storage"] == "external" - if self.settings.filter_files_glob and not fnmatch.fnmatch(file_name, self.settings.filter_files_glob): - logging.info( - "File '%s' does not match the glob filter '%s', skipping", - file_name, - self.settings.filter_files_glob, - ) + if self.settings.filter_files_type and file_type not in self.settings.filter_files_type: + logging.info("File '%s' has ignored type '%s', skipping", file_name, file_type) continue - if self.settings.filter_files_regex and not re.fullmatch(self.settings.filter_files_regex, file_name): - logging.info( - "File '%s' does not match the regex filter '%s', skipping", - file_name, - self.settings.filter_files_regex, - ) + if ( + self.settings.filter_files_platform + and file_type == "default" + and not any(trait in self.settings.filter_files_platform for trait in file_traits) + ): + # Setup for filter_files_platform is in config.py, including the trait listing. + logging.info("File '%s' not for requested platforms, skipping", file_name) + continue + + if should_skip_item_by_glob("File", file_name, self.settings.filter_files_glob): + continue + + if should_skip_item_by_regex("File", file_name, self.settings.filter_files_regex): continue logging.debug(