Added ci for releasing new versions to pypi

.gitignore

@@ -0,0 +1,2 @@
+*egg-info
+__pycache__

.gitlab-ci.yml

@@ -0,0 +1,37 @@
+image: python:3
+
+stages:
+  - test
+  - release
+
+flake8:
+  stage: test
+  script:
+    - pip install tox
+    - tox -e flake8
+
+release-package:
+  stage: release
+  only:
+    - master
+  script:
+    - pip install twine
+    - rm -f dist/*
+    - python setup.py sdist
+    - twine upload -u ${PYPI_USER} -p ${PYPI_PASS} dist/*
+
+
+.write_permission: &write_permission |
+    git config --global user.email "gitlab-ci"; git config --global user.name "gitlab-ci"
+    url_host=`git remote get-url origin | sed -e "s/https:\/\/gitlab-ci-token:.*@//g"`
+    git remote set-url origin "https://gitlab-ci-token:${CI_TAG_UPLOAD_TOKEN}@${url_host}"
+
+tag:
+  stage: release
+  only:
+    - master
+  script:
+    - *write_permission
+    - export VERSION=$(echo $(python -c "import _version; print(_version.__version__)"))
+    - git tag -a $VERSION -m "Version created by gitlab-ci Build"
+    - git push origin $VERSION

README.md

@@ -0,0 +1,22 @@
+# Humble Bundle Downloader
+Download all of you content from your Humble Bundle Library.  
+
+The very first time this runs it may take a while to download everything, but after that it will only download the content that is missing.  
+
+## Features
+- Download new or updated content in your Library
+- Progress bar for each download _(with the `--progress` flag)_
+- Easy cookie generation so script
+
+## Install
+`pip install humblebundle-downloader`
+
+
+## Getting started
+First thing to do is generate cookies, this will open up a chrome window, just login and a cookie will be saved to a file to be used later to download the files.  
+`hbd gen-cookies -h`  
+
+Now download your library:  
+`hbd download -h`  
+
+Inside your library folder a file called `.cache.json` is saved and keeps track of the files that have been downloaded, so running the download command pointing to the same directory will only download new files or update files if needed.

_version.py

@@ -0,0 +1 @@
+__version__ = '0.0.1'

humblebundle_downloader/__init__.py

@@ -0,0 +1,2 @@
+from .generate_cookie import generate_cookie  # noqa: F401
+from .download_library import download_library  # noqa: F401

humblebundle_downloader/cli.py

@@ -0,0 +1,53 @@
+import os
+import logging
+import argparse
+
+logger = logging.getLogger(__name__)
+
+LOG_LEVEL = os.environ.get('HBD_LOGLEVEL', 'INFO').upper()
+logging.basicConfig(
+    level=LOG_LEVEL,
+    format='%(message)s',
+)
+
+
+def cli():
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers(dest='action')
+    subparsers.required = True
+
+    ###
+    # Generate cookie
+    ###
+    parser_gencookie = subparsers.add_parser('gen-cookie',
+                                             help="Generate cookie used to access your library")
+    parser_gencookie.add_argument('-c', '--cookie-file', type=str,
+                                  help='Location of the file to store the cookie',
+                                  default='hbd-cookies.txt')
+
+    ###
+    # Download Library
+    ###
+    # TODO: for download: have option to only get types, ebooks, videos, etc do not enforce,
+    # but lower and just string match to the type in the api
+    parser_download = subparsers.add_parser('download',
+                                            help="Download content in your humble bundle library")
+    parser_download.add_argument('-c', '--cookie-file', type=str,
+                                 help='Location of the file to store the cookie',
+                                 default='hbd-cookies.txt')
+    parser_download.add_argument('-l', '--library-path', type=str,
+                                 help='Folder to download all content to',
+                                 required=True)
+    parser_download.add_argument('--progress', action='store_true',
+                                 help="Display progress bar for downloads")
+
+    cli_args = parser.parse_args()
+
+    if cli_args.action == 'gen-cookie':
+        from . import generate_cookie
+        generate_cookie(cli_args.cookie_file)
+
+    elif cli_args.action == 'download':
+        from . import download_library
+        download_library(cli_args.cookie_file, cli_args.library_path,
+                         progress_bar=cli_args.progress)

humblebundle_downloader/download_library.py

@@ -0,0 +1,90 @@
+import os
+import json
+import parsel
+import logging
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+def _clean_name(dirty_str):
+    allowed_chars = (' ', '_', '.', '-', ':', '[', ']')
+    return "".join([c for c in dirty_str.replace('+', '_') if c.isalpha() or c.isdigit() or c in allowed_chars]).strip()
+
+
+def download_library(cookie_path, library_path, progress_bar=False):
+    # Load cookies
+    with open(cookie_path, 'r') as f:
+        account_cookies = f.read()
+
+    cache_file = os.path.join(library_path, '.cache.json')
+
+    try:
+        with open(cache_file, 'r') as f:
+            cache_data = json.load(f)
+    except FileNotFoundError:
+        cache_data = {}
+
+    library_r = requests.get('https://www.humblebundle.com/home/library',
+                             headers={'cookie': account_cookies})
+    logger.debug(f"Library request: {library_r}")
+    library_page = parsel.Selector(text=library_r.text)
+
+    for order_id in json.loads(library_page.css('#user-home-json-data').xpath('string()').extract_first())['gamekeys']:
+        order_r = requests.get(f'https://www.humblebundle.com/api/v1/order/{order_id}?all_tpkds=true',
+                               headers={'cookie': account_cookies})
+        logger.debug(f"Order request: {order_r}")
+        order = order_r.json()
+        bundle_title = _clean_name(order['product']['human_name'])
+        logger.info(f"Checking bundle: {bundle_title}")
+        for item in order['subproducts']:
+            item_title = _clean_name(item['human_name'])
+            # Get all types of download for a product
+            for download_type in item['downloads']:
+                # platform = download_type['platform']  # Type of product, ebook, videos, etc...
+                item_folder = os.path.join(library_path, bundle_title, item_title)
+
+                # Create directory to save the files to
+                try: os.makedirs(item_folder)  # noqa: E701
+                except OSError: pass  # noqa: E701
+
+                # Download each file type of a product
+                for file_type in download_type['download_struct']:
+                    url = file_type['url']['web']
+                    ext = url.split('?')[0].split('.')[-1]
+                    filename = os.path.join(item_folder, f"{item_title}.{ext}")
+                    item_r = requests.get(url, stream=True)
+                    logger.debug(f"Item request: {item_r}, Url: {url}")
+                    # Not sure which value will be best to use, so save them all for now
+                    file_info = {
+                        'md5': file_type['md5'],
+                        'sha1': file_type['sha1'],
+                        'url_last_modified': item_r.headers['Last-Modified'],
+                        'url_etag': item_r.headers['ETag'][1:-1],
+                        'url_crc': item_r.headers['X-HW-Cache-CRC'],
+                    }
+                    if file_info != cache_data.get(filename, {}):
+                        if not progress_bar:
+                            logger.info(f"Downloading: {item_title}.{ext}")
+
+                        with open(filename, 'wb') as outfile:
+                            total_length = item_r.headers.get('content-length')
+                            if total_length is None:  # no content length header
+                                outfile.write(item_r.content)
+                            else:
+                                dl = 0
+                                total_length = int(total_length)
+                                for data in item_r.iter_content(chunk_size=4096):
+                                    dl += len(data)
+                                    outfile.write(data)
+                                    pb_width = 50
+                                    done = int(pb_width * dl / total_length)
+                                    if progress_bar: print(f"Downloading: {item_title}.{ext}: {int(done * (100 / pb_width))}% [{'=' * done}{' ' * (pb_width-done)}]", end='\r')  # noqa: E501, E701
+
+                        if progress_bar:
+                            print()  # print new line so next progress bar is on its own line
+
+                        cache_data[filename] = file_info
+                        # Update cache file with newest data so if the script quits it can keep track of the progress
+                        with open(cache_file, 'w') as outfile:
+                            json.dump(cache_data, outfile, sort_keys=True, indent=4)

humblebundle_downloader/generate_cookie.py

@@ -0,0 +1,38 @@
+import time
+import logging
+from selenium import webdriver
+from webdriverdownloader import ChromeDriverDownloader
+
+
+logger = logging.getLogger(__name__)
+
+
+def _get_cookie_str(driver):
+    raw_cookies = driver.get_cookies()
+    baked_cookies = ''
+    for cookie in raw_cookies:
+        baked_cookies += f"{cookie['name']}={cookie['value']};"
+    # Remove the trailing ;
+    return baked_cookies[:-1]
+
+
+def generate_cookie(cookie_path):
+    gdd = ChromeDriverDownloader()
+    chrome_driver = gdd.download_and_install()
+
+    # TODO: load previous cookies so it does not ask to re verify using an email code each time
+    driver = webdriver.Chrome(executable_path=chrome_driver[1])
+
+    driver.get('https://www.humblebundle.com/login')
+
+    while '/login' in driver.current_url:
+        # Waiting for the user to login
+        time.sleep(.25)
+
+    cookie_str = _get_cookie_str(driver)
+    with open(cookie_path, 'w') as f:
+        f.write(cookie_str)
+
+    logger.info(f"Saved cookie to {cookie_path}")
+
+    driver.quit()

setup.py

@@ -0,0 +1,35 @@
+from setuptools import setup
+from _version import __version__
+
+
+with open('README.md', 'r') as f:
+    long_description = f.read()
+
+setup(
+    name='humblebundle-downloader',
+    packages=['humblebundle_downloader'],
+    version=__version__,
+    description='Download your Humbdle Bundle library',
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    author='Eddy Hintze',
+    author_email="eddy@hintze.co",
+    url="https://gitx.codes/xtream1101/humblebundle-downloader",
+    license='MIT',
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "Operating System :: OS Independent",
+    ],
+    entry_points={
+        'console_scripts': [
+            'hbd=humblebundle_downloader.cli:cli',
+        ],
+    },
+    install_requires=[
+        'requests',
+        'parsel',
+        'selenium',
+        'webdriverdownloader',
+    ],
+
+)