diff --git a/CHANGELOG.md b/CHANGELOG.md index 50630d8..1af1039 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ # Change log -### WIP + +### 0.3.1 +- Added support for netscape cookies + + +### 0.3.0 - pip install now requires python version 3.4+ - `--trove` will only download trove products, nothing else - Filtering flags now work when downloading trove content diff --git a/README.md b/README.md index 48cd82c..0afda8d 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,6 @@ The first time this runs it may take a while because it will download everything - cli command for easy use (downloading will also work on a headless system) - works for SSO and 2FA accounts - optional progress bar for each item downloaded _(`--progress` flag)_ -- optional cookie generation script - optional filter by file types using an include _or_ exclude list _(`--include/--exclude` flag)_ - optional filter by platform types like video, ebook, etc... _(`--platform` flag)_ @@ -25,24 +24,13 @@ The first time this runs it may take a while because it will download everything ## Instructions ### 1. Getting cookies -First thing to do is get your account cookies, they will be used later to download the files. -There are 2 ways to get your cookies: manual or scripted. - -#### Method 1: Manual -Use this method if you know how to get cookies from your browser after you are logged in. -Once you have your cookies, save them to a text file named `hbd-cookies.txt` in this format: -`hbflash=None;_fbp=fb.1.000000.000000;__ssid=XXXXXXXX;_gat=1;_gid=GA1.2.1111111.11111111;hbreqsec=True;_ga=GA1.2.1111111.111111;_simpleauth_sess=XXXXXXXXXXXXX;csrf_cookie=XXXXXXXXX` - -#### Method 2: Scripted -**WARNING: This method may not work on all systems!** -Requires: Chrome and a desktop-like environment (not headless). - -Run the command below to open a chrome window. After you login, the cookies will automatically be saved to a text file and the window will close. -`hbd gen-cookies --cookie-file hbd-cookies.txt` +First thing to do is get your account cookies. The cookies should be in the Netscape format. You can get them by using a browser extension. These are the ones that I tested, but others may work as well... +- Firefox: https://addons.mozilla.org/en-US/firefox/addon/export-cookies-txt/ +- Chrome: https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg/ ### 2. Downloading your library Use the following command to download your Humble Bundle Library: -`hbd download --cookie-file hbd-cookies.txt --library-path "Downloaded Library" --progress` +`hbd download --cookie-file cookies.txt --library-path "Downloaded Library" --progress` This directory structure will be used: `Downloaded Library/Purchase Name/Item Name/downloaded_file.ext` diff --git a/humblebundle_downloader/_version.py b/humblebundle_downloader/_version.py index 0404d81..e1424ed 100644 --- a/humblebundle_downloader/_version.py +++ b/humblebundle_downloader/_version.py @@ -1 +1 @@ -__version__ = '0.3.0' +__version__ = '0.3.1' diff --git a/humblebundle_downloader/cli.py b/humblebundle_downloader/cli.py index 66585ae..b5f3e4c 100644 --- a/humblebundle_downloader/cli.py +++ b/humblebundle_downloader/cli.py @@ -18,19 +18,6 @@ def cli(): subparsers = parser.add_subparsers(dest='action') subparsers.required = True - ### - # Generate cookie - ### - parser_gencookie = subparsers.add_parser( - 'gen-cookies', - help="Generate cookies used to access your library", - ) - parser_gencookie.add_argument( - '-c', '--cookie-file', type=str, - help="Location of the file to store the cookie", - required=True, - ) - ### # Download Library ### @@ -89,11 +76,8 @@ def cli(): cli_args = parser.parse_args() - if cli_args.action == 'gen-cookies': - from .generate_cookie import generate_cookie - generate_cookie(cli_args.cookie_file) - - elif cli_args.action == 'download': + if cli_args.action == 'download': + # Still keep the download action to keep compatibility from .download_library import DownloadLibrary DownloadLibrary( cli_args.cookie_file, diff --git a/humblebundle_downloader/download_library.py b/humblebundle_downloader/download_library.py index 3808a3e..f4f4064 100644 --- a/humblebundle_downloader/download_library.py +++ b/humblebundle_downloader/download_library.py @@ -6,7 +6,7 @@ import parsel import logging import datetime import requests -from http.cookiejar import MozillaCookieJar +import http.cookiejar logger = logging.getLogger(__name__) @@ -26,7 +26,6 @@ class DownloadLibrary: def __init__(self, cookie_path, library_path, progress_bar=False, ext_include=None, ext_exclude=None, platform_include=None, purchase_keys=None, trove=False, update=False): - self.cookie_path = cookie_path self.library_path = library_path self.progress_bar = progress_bar self.ext_include = [] if ext_include is None else list(map(str.lower, ext_include)) # noqa: E501 @@ -42,9 +41,18 @@ class DownloadLibrary: self.trove = trove self.update = update + self.session = requests.Session() + try: + cookie_jar = http.cookiejar.MozillaCookieJar(cookie_path) + cookie_jar.load() + self.session.cookies = cookie_jar + except http.cookiejar.LoadError: + # Still support the original cookie method + with open(cookie_path, 'r') as f: + self.session.headers.update({'cookie': f.read().strip()}) + def start(self): - self.cookiejar = MozillaCookieJar(self.cookie_path) - self.cookiejar.load() + self.cache_data = self._load_cache_data(self.cache_file) self.purchase_keys = self.purchase_keys if self.purchase_keys else self._get_purchase_keys() # noqa: E501 @@ -59,13 +67,12 @@ class DownloadLibrary: def _get_trove_download_url(self, machine_name, web_name): try: - sign_r = requests.post( + sign_r = self.session.post( 'https://www.humblebundle.com/api/v1/user/download/sign', data={ 'machine_name': machine_name, 'filename': web_name, }, - cookies=self.cookiejar, ) except Exception: logger.error("Failed to get download url for trove product {title}" @@ -131,7 +138,7 @@ class DownloadLibrary: continue try: - product_r = requests.get(signed_url, stream=True) + product_r = self.session.get(signed_url, stream=True) except Exception: logger.error("Failed to get trove product {title}" .format(title=web_name)) @@ -162,8 +169,7 @@ class DownloadLibrary: .format(idx=idx)) trove_page_url = trove_base_url.format(idx=idx) try: - trove_r = requests.get(trove_page_url, - cookies=self.cookiejar) + trove_r = self.session.get(trove_page_url) except Exception: logger.error("Failed to get products from Humble Trove") return [] @@ -181,12 +187,13 @@ class DownloadLibrary: def _process_order_id(self, order_id): order_url = 'https://www.humblebundle.com/api/v1/order/{order_id}?all_tpkds=true'.format(order_id=order_id) # noqa: E501 try: - order_r = requests.get(order_url, - cookies=self.cookiejar, - headers={ - 'content-type': 'application/json', - 'content-encoding': 'gzip', - }) + order_r = self.session.get( + order_url, + headers={ + 'content-type': 'application/json', + 'content-encoding': 'gzip', + }, + ) except Exception: logger.error("Failed to get order key {order_id}" .format(order_id=order_id)) @@ -254,7 +261,7 @@ class DownloadLibrary: continue try: - product_r = requests.get(url, stream=True) + product_r = self.session.get(url, stream=True) except Exception: logger.error("Failed to download {url}".format(url=url)) continue @@ -362,15 +369,14 @@ class DownloadLibrary: def _get_purchase_keys(self): try: - library_r = requests.get('https://www.humblebundle.com/home/library', # noqa: E501 - cookies=self.cookiejar) + library_r = self.session.get('https://www.humblebundle.com/home/library') # noqa: E501 except Exception: logger.error("Failed to get list of purchases") return [] logger.debug("Library request: " + str(library_r)) library_page = parsel.Selector(text=library_r.text) - user_data = library_page.css('#user-home-json-data').xpath('string()').extract_first() + user_data = library_page.css('#user-home-json-data').xpath('string()').extract_first() # noqa: E501 if user_data is None: raise Exception("Unable to download user-data, cookies missing?") orders_json = json.loads(user_data) diff --git a/humblebundle_downloader/generate_cookie.py b/humblebundle_downloader/generate_cookie.py deleted file mode 100644 index 5ed8ac9..0000000 --- a/humblebundle_downloader/generate_cookie.py +++ /dev/null @@ -1,39 +0,0 @@ -import time -import logging -from selenium import webdriver -from webdriverdownloader import ChromeDriverDownloader - - -logger = logging.getLogger(__name__) - - -def _get_cookie_str(driver): - raw_cookies = driver.get_cookies() - baked_cookies = '' - for cookie in raw_cookies: - baked_cookies += cookie['name'] + "=" + cookie['value'] + ";" - # Remove the trailing ; - return baked_cookies[:-1] - - -def generate_cookie(cookie_path): - gdd = ChromeDriverDownloader() - chrome_driver = gdd.download_and_install() - - # TODO: load previous cookies so it does not ask to re verify using an - # email code each time - driver = webdriver.Chrome(executable_path=chrome_driver[1]) - - driver.get('https://www.humblebundle.com/login') - - while '/home/library' not in driver.current_url: - # Waiting for the user to login - time.sleep(.25) - - cookie_str = _get_cookie_str(driver) - with open(cookie_path, 'w') as f: - f.write(cookie_str) - - logger.info("Saved cookies to " + cookie_path) - - driver.quit()