Support both types of cookie files. Removed cookie gen script. Version bump

CHANGELOG.md

@@ -1,6 +1,11 @@
 # Change log
 
-### WIP
+
+### 0.3.1
+- Added support for netscape cookies
+ 
+
+### 0.3.0
 - pip install now requires python version 3.4+
 - `--trove` will only download trove products, nothing else
 - Filtering flags now work when downloading trove content

README.md

@@ -13,7 +13,6 @@ The first time this runs it may take a while because it will download everything
 - cli command for easy use (downloading will also work on a headless system)
 - works for SSO and 2FA accounts
 - optional progress bar for each item downloaded _(`--progress` flag)_
-- optional cookie generation script
 - optional filter by file types using an include _or_ exclude list _(`--include/--exclude` flag)_
 - optional filter by platform types like video, ebook, etc... _(`--platform` flag)_
 
@@ -25,24 +24,13 @@ The first time this runs it may take a while because it will download everything
 ## Instructions
 
 ### 1. Getting cookies
-First thing to do is get your account cookies, they will be used later to download the files.  
+First thing to do is get your account cookies. The cookies should be in the Netscape format. You can get them by using a browser extension. These are the ones that I tested, but others may work as well...  
-There are 2 ways to get your cookies: manual or scripted.  
+- Firefox: https://addons.mozilla.org/en-US/firefox/addon/export-cookies-txt/
-
+- Chrome: https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg/
-#### Method 1: Manual
-Use this method if you know how to get cookies from your browser after you are logged in.  
-Once you have your cookies, save them to a text file named `hbd-cookies.txt` in this format:  
-`hbflash=None;_fbp=fb.1.000000.000000;__ssid=XXXXXXXX;_gat=1;_gid=GA1.2.1111111.11111111;hbreqsec=True;_ga=GA1.2.1111111.111111;_simpleauth_sess=XXXXXXXXXXXXX;csrf_cookie=XXXXXXXXX`
-
-#### Method 2: Scripted
-**WARNING: This method may not work on all systems!**  
-Requires: Chrome and a desktop-like environment (not headless).  
-
-Run the command below to open a chrome window. After you login, the cookies will automatically be saved to a text file and the window will close.  
-`hbd gen-cookies --cookie-file hbd-cookies.txt`  
 
 ### 2. Downloading your library
 Use the following command to download your Humble Bundle Library:  
-`hbd download --cookie-file hbd-cookies.txt --library-path "Downloaded Library" --progress`  
+`hbd download --cookie-file cookies.txt --library-path "Downloaded Library" --progress`  
 
 This directory structure will be used:  
 `Downloaded Library/Purchase Name/Item Name/downloaded_file.ext`

humblebundle_downloader/_version.py

@@ -1 +1 @@
-__version__ = '0.3.0'
+__version__ = '0.3.1'

humblebundle_downloader/cli.py

@@ -18,19 +18,6 @@ def cli():
     subparsers = parser.add_subparsers(dest='action')
     subparsers.required = True
 
-    ###
-    # Generate cookie
-    ###
-    parser_gencookie = subparsers.add_parser(
-        'gen-cookies',
-        help="Generate cookies used to access your library",
-    )
-    parser_gencookie.add_argument(
-        '-c', '--cookie-file', type=str,
-        help="Location of the file to store the cookie",
-        required=True,
-    )
-
     ###
     # Download Library
     ###
@@ -89,11 +76,8 @@ def cli():
 
     cli_args = parser.parse_args()
 
-    if cli_args.action == 'gen-cookies':
+    if cli_args.action == 'download':
-        from .generate_cookie import generate_cookie
+        # Still keep the download action to keep compatibility
-        generate_cookie(cli_args.cookie_file)
-
-    elif cli_args.action == 'download':
         from .download_library import DownloadLibrary
         DownloadLibrary(
             cli_args.cookie_file,

humblebundle_downloader/download_library.py

@@ -6,7 +6,7 @@ import parsel
 import logging
 import datetime
 import requests
-from http.cookiejar import MozillaCookieJar
+import http.cookiejar
 
 logger = logging.getLogger(__name__)
 
@@ -26,7 +26,6 @@ class DownloadLibrary:
     def __init__(self, cookie_path, library_path, progress_bar=False,
                  ext_include=None, ext_exclude=None, platform_include=None,
                  purchase_keys=None, trove=False, update=False):
-        self.cookie_path = cookie_path
         self.library_path = library_path
         self.progress_bar = progress_bar
         self.ext_include = [] if ext_include is None else list(map(str.lower, ext_include))  # noqa: E501
@@ -42,9 +41,18 @@ class DownloadLibrary:
         self.trove = trove
         self.update = update
 
+        self.session = requests.Session()
+        try:
+            cookie_jar = http.cookiejar.MozillaCookieJar(cookie_path)
+            cookie_jar.load()
+            self.session.cookies = cookie_jar
+        except http.cookiejar.LoadError:
+            # Still support the original cookie method
+            with open(cookie_path, 'r') as f:
+                self.session.headers.update({'cookie': f.read().strip()})
+
     def start(self):
-        self.cookiejar = MozillaCookieJar(self.cookie_path)
+
-        self.cookiejar.load()
         self.cache_data = self._load_cache_data(self.cache_file)
         self.purchase_keys = self.purchase_keys if self.purchase_keys else self._get_purchase_keys()  # noqa: E501
 
@@ -59,13 +67,12 @@ class DownloadLibrary:
 
     def _get_trove_download_url(self, machine_name, web_name):
         try:
-            sign_r = requests.post(
+            sign_r = self.session.post(
                 'https://www.humblebundle.com/api/v1/user/download/sign',
                 data={
                     'machine_name': machine_name,
                     'filename': web_name,
                 },
-                cookies=self.cookiejar,
             )
         except Exception:
             logger.error("Failed to get download url for trove product {title}"
@@ -131,7 +138,7 @@ class DownloadLibrary:
                     continue
 
                 try:
-                    product_r = requests.get(signed_url, stream=True)
+                    product_r = self.session.get(signed_url, stream=True)
                 except Exception:
                     logger.error("Failed to get trove product {title}"
                                  .format(title=web_name))
@@ -162,8 +169,7 @@ class DownloadLibrary:
                          .format(idx=idx))
             trove_page_url = trove_base_url.format(idx=idx)
             try:
-                trove_r = requests.get(trove_page_url,
+                trove_r = self.session.get(trove_page_url)
-                                       cookies=self.cookiejar)
             except Exception:
                 logger.error("Failed to get products from Humble Trove")
                 return []
@@ -181,12 +187,13 @@ class DownloadLibrary:
     def _process_order_id(self, order_id):
         order_url = 'https://www.humblebundle.com/api/v1/order/{order_id}?all_tpkds=true'.format(order_id=order_id)  # noqa: E501
         try:
-            order_r = requests.get(order_url,
+            order_r = self.session.get(
-                                   cookies=self.cookiejar,
+                order_url,
-                                   headers={
+                headers={
-                                            'content-type': 'application/json',
+                    'content-type': 'application/json',
-                                            'content-encoding': 'gzip',
+                    'content-encoding': 'gzip',
-                                            })
+                },
+            )
         except Exception:
             logger.error("Failed to get order key {order_id}"
                          .format(order_id=order_id))
@@ -254,7 +261,7 @@ class DownloadLibrary:
                     continue
 
                 try:
-                    product_r = requests.get(url, stream=True)
+                    product_r = self.session.get(url, stream=True)
                 except Exception:
                     logger.error("Failed to download {url}".format(url=url))
                     continue
@@ -362,15 +369,14 @@ class DownloadLibrary:
 
     def _get_purchase_keys(self):
         try:
-            library_r = requests.get('https://www.humblebundle.com/home/library',  # noqa: E501
+            library_r = self.session.get('https://www.humblebundle.com/home/library')  # noqa: E501
-                                     cookies=self.cookiejar)
         except Exception:
             logger.error("Failed to get list of purchases")
             return []
 
         logger.debug("Library request: " + str(library_r))
         library_page = parsel.Selector(text=library_r.text)
-        user_data = library_page.css('#user-home-json-data').xpath('string()').extract_first()
+        user_data = library_page.css('#user-home-json-data').xpath('string()').extract_first()  # noqa: E501
         if user_data is None:
             raise Exception("Unable to download user-data, cookies missing?")
         orders_json = json.loads(user_data)

humblebundle_downloader/generate_cookie.py

@@ -1,39 +0,0 @@
-import time
-import logging
-from selenium import webdriver
-from webdriverdownloader import ChromeDriverDownloader
-
-
-logger = logging.getLogger(__name__)
-
-
-def _get_cookie_str(driver):
-    raw_cookies = driver.get_cookies()
-    baked_cookies = ''
-    for cookie in raw_cookies:
-        baked_cookies += cookie['name'] + "=" + cookie['value'] + ";"
-    # Remove the trailing ;
-    return baked_cookies[:-1]
-
-
-def generate_cookie(cookie_path):
-    gdd = ChromeDriverDownloader()
-    chrome_driver = gdd.download_and_install()
-
-    # TODO: load previous cookies so it does not ask to re verify using an
-    # email code each time
-    driver = webdriver.Chrome(executable_path=chrome_driver[1])
-
-    driver.get('https://www.humblebundle.com/login')
-
-    while '/home/library' not in driver.current_url:
-        # Waiting for the user to login
-        time.sleep(.25)
-
-    cookie_str = _get_cookie_str(driver)
-    with open(cookie_path, 'w') as f:
-        f.write(cookie_str)
-
-    logger.info("Saved cookies to " + cookie_path)
-
-    driver.quit()