mirror of
https://github.com/xtream1101/humblebundle-downloader.git
synced 2024-11-20 08:49:17 +01:00
Support both types of cookie files. Removed cookie gen script. Version bump
This commit is contained in:
Eddy Hintze
parent
0d6a06f950
commit
aaa8ecc7ba
@ -1,6 +1,11 @@
|
||||
# Change log
|
||||
|
||||
### WIP
|
||||
|
||||
### 0.3.1
|
||||
- Added support for netscape cookies
|
||||
|
||||
|
||||
### 0.3.0
|
||||
- pip install now requires python version 3.4+
|
||||
- `--trove` will only download trove products, nothing else
|
||||
- Filtering flags now work when downloading trove content
|
||||
|
||||
20
README.md
20
README.md
@ -13,7 +13,6 @@ The first time this runs it may take a while because it will download everything
|
||||
- cli command for easy use (downloading will also work on a headless system)
|
||||
- works for SSO and 2FA accounts
|
||||
- optional progress bar for each item downloaded _(`--progress` flag)_
|
||||
- optional cookie generation script
|
||||
- optional filter by file types using an include _or_ exclude list _(`--include/--exclude` flag)_
|
||||
- optional filter by platform types like video, ebook, etc... _(`--platform` flag)_
|
||||
|
||||
@ -25,24 +24,13 @@ The first time this runs it may take a while because it will download everything
|
||||
## Instructions
|
||||
|
||||
### 1. Getting cookies
|
||||
First thing to do is get your account cookies, they will be used later to download the files.
|
||||
There are 2 ways to get your cookies: manual or scripted.
|
||||
|
||||
#### Method 1: Manual
|
||||
Use this method if you know how to get cookies from your browser after you are logged in.
|
||||
Once you have your cookies, save them to a text file named `hbd-cookies.txt` in this format:
|
||||
`hbflash=None;_fbp=fb.1.000000.000000;__ssid=XXXXXXXX;_gat=1;_gid=GA1.2.1111111.11111111;hbreqsec=True;_ga=GA1.2.1111111.111111;_simpleauth_sess=XXXXXXXXXXXXX;csrf_cookie=XXXXXXXXX`
|
||||
|
||||
#### Method 2: Scripted
|
||||
**WARNING: This method may not work on all systems!**
|
||||
Requires: Chrome and a desktop-like environment (not headless).
|
||||
|
||||
Run the command below to open a chrome window. After you login, the cookies will automatically be saved to a text file and the window will close.
|
||||
`hbd gen-cookies --cookie-file hbd-cookies.txt`
|
||||
First thing to do is get your account cookies. The cookies should be in the Netscape format. You can get them by using a browser extension. These are the ones that I tested, but others may work as well...
|
||||
- Firefox: https://addons.mozilla.org/en-US/firefox/addon/export-cookies-txt/
|
||||
- Chrome: https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg/
|
||||
|
||||
### 2. Downloading your library
|
||||
Use the following command to download your Humble Bundle Library:
|
||||
`hbd download --cookie-file hbd-cookies.txt --library-path "Downloaded Library" --progress`
|
||||
`hbd download --cookie-file cookies.txt --library-path "Downloaded Library" --progress`
|
||||
|
||||
This directory structure will be used:
|
||||
`Downloaded Library/Purchase Name/Item Name/downloaded_file.ext`
|
||||
|
||||
@ -1 +1 @@
|
||||
__version__ = '0.3.0'
|
||||
__version__ = '0.3.1'
|
||||
|
||||
@ -18,19 +18,6 @@ def cli():
|
||||
subparsers = parser.add_subparsers(dest='action')
|
||||
subparsers.required = True
|
||||
|
||||
###
|
||||
# Generate cookie
|
||||
###
|
||||
parser_gencookie = subparsers.add_parser(
|
||||
'gen-cookies',
|
||||
help="Generate cookies used to access your library",
|
||||
)
|
||||
parser_gencookie.add_argument(
|
||||
'-c', '--cookie-file', type=str,
|
||||
help="Location of the file to store the cookie",
|
||||
required=True,
|
||||
)
|
||||
|
||||
###
|
||||
# Download Library
|
||||
###
|
||||
@ -89,11 +76,8 @@ def cli():
|
||||
|
||||
cli_args = parser.parse_args()
|
||||
|
||||
if cli_args.action == 'gen-cookies':
|
||||
from .generate_cookie import generate_cookie
|
||||
generate_cookie(cli_args.cookie_file)
|
||||
|
||||
elif cli_args.action == 'download':
|
||||
if cli_args.action == 'download':
|
||||
# Still keep the download action to keep compatibility
|
||||
from .download_library import DownloadLibrary
|
||||
DownloadLibrary(
|
||||
cli_args.cookie_file,
|
||||
|
||||
@ -6,7 +6,7 @@ import parsel
|
||||
import logging
|
||||
import datetime
|
||||
import requests
|
||||
from http.cookiejar import MozillaCookieJar
|
||||
import http.cookiejar
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@ -26,7 +26,6 @@ class DownloadLibrary:
|
||||
def __init__(self, cookie_path, library_path, progress_bar=False,
|
||||
ext_include=None, ext_exclude=None, platform_include=None,
|
||||
purchase_keys=None, trove=False, update=False):
|
||||
self.cookie_path = cookie_path
|
||||
self.library_path = library_path
|
||||
self.progress_bar = progress_bar
|
||||
self.ext_include = [] if ext_include is None else list(map(str.lower, ext_include)) # noqa: E501
|
||||
@ -42,9 +41,18 @@ class DownloadLibrary:
|
||||
self.trove = trove
|
||||
self.update = update
|
||||
|
||||
self.session = requests.Session()
|
||||
try:
|
||||
cookie_jar = http.cookiejar.MozillaCookieJar(cookie_path)
|
||||
cookie_jar.load()
|
||||
self.session.cookies = cookie_jar
|
||||
except http.cookiejar.LoadError:
|
||||
# Still support the original cookie method
|
||||
with open(cookie_path, 'r') as f:
|
||||
self.session.headers.update({'cookie': f.read().strip()})
|
||||
|
||||
def start(self):
|
||||
self.cookiejar = MozillaCookieJar(self.cookie_path)
|
||||
self.cookiejar.load()
|
||||
|
||||
self.cache_data = self._load_cache_data(self.cache_file)
|
||||
self.purchase_keys = self.purchase_keys if self.purchase_keys else self._get_purchase_keys() # noqa: E501
|
||||
|
||||
@ -59,13 +67,12 @@ class DownloadLibrary:
|
||||
|
||||
def _get_trove_download_url(self, machine_name, web_name):
|
||||
try:
|
||||
sign_r = requests.post(
|
||||
sign_r = self.session.post(
|
||||
'https://www.humblebundle.com/api/v1/user/download/sign',
|
||||
data={
|
||||
'machine_name': machine_name,
|
||||
'filename': web_name,
|
||||
},
|
||||
cookies=self.cookiejar,
|
||||
)
|
||||
except Exception:
|
||||
logger.error("Failed to get download url for trove product {title}"
|
||||
@ -131,7 +138,7 @@ class DownloadLibrary:
|
||||
continue
|
||||
|
||||
try:
|
||||
product_r = requests.get(signed_url, stream=True)
|
||||
product_r = self.session.get(signed_url, stream=True)
|
||||
except Exception:
|
||||
logger.error("Failed to get trove product {title}"
|
||||
.format(title=web_name))
|
||||
@ -162,8 +169,7 @@ class DownloadLibrary:
|
||||
.format(idx=idx))
|
||||
trove_page_url = trove_base_url.format(idx=idx)
|
||||
try:
|
||||
trove_r = requests.get(trove_page_url,
|
||||
cookies=self.cookiejar)
|
||||
trove_r = self.session.get(trove_page_url)
|
||||
except Exception:
|
||||
logger.error("Failed to get products from Humble Trove")
|
||||
return []
|
||||
@ -181,12 +187,13 @@ class DownloadLibrary:
|
||||
def _process_order_id(self, order_id):
|
||||
order_url = 'https://www.humblebundle.com/api/v1/order/{order_id}?all_tpkds=true'.format(order_id=order_id) # noqa: E501
|
||||
try:
|
||||
order_r = requests.get(order_url,
|
||||
cookies=self.cookiejar,
|
||||
order_r = self.session.get(
|
||||
order_url,
|
||||
headers={
|
||||
'content-type': 'application/json',
|
||||
'content-encoding': 'gzip',
|
||||
})
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
logger.error("Failed to get order key {order_id}"
|
||||
.format(order_id=order_id))
|
||||
@ -254,7 +261,7 @@ class DownloadLibrary:
|
||||
continue
|
||||
|
||||
try:
|
||||
product_r = requests.get(url, stream=True)
|
||||
product_r = self.session.get(url, stream=True)
|
||||
except Exception:
|
||||
logger.error("Failed to download {url}".format(url=url))
|
||||
continue
|
||||
@ -362,15 +369,14 @@ class DownloadLibrary:
|
||||
|
||||
def _get_purchase_keys(self):
|
||||
try:
|
||||
library_r = requests.get('https://www.humblebundle.com/home/library', # noqa: E501
|
||||
cookies=self.cookiejar)
|
||||
library_r = self.session.get('https://www.humblebundle.com/home/library') # noqa: E501
|
||||
except Exception:
|
||||
logger.error("Failed to get list of purchases")
|
||||
return []
|
||||
|
||||
logger.debug("Library request: " + str(library_r))
|
||||
library_page = parsel.Selector(text=library_r.text)
|
||||
user_data = library_page.css('#user-home-json-data').xpath('string()').extract_first()
|
||||
user_data = library_page.css('#user-home-json-data').xpath('string()').extract_first() # noqa: E501
|
||||
if user_data is None:
|
||||
raise Exception("Unable to download user-data, cookies missing?")
|
||||
orders_json = json.loads(user_data)
|
||||
|
||||
@ -1,39 +0,0 @@
|
||||
import time
|
||||
import logging
|
||||
from selenium import webdriver
|
||||
from webdriverdownloader import ChromeDriverDownloader
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_cookie_str(driver):
|
||||
raw_cookies = driver.get_cookies()
|
||||
baked_cookies = ''
|
||||
for cookie in raw_cookies:
|
||||
baked_cookies += cookie['name'] + "=" + cookie['value'] + ";"
|
||||
# Remove the trailing ;
|
||||
return baked_cookies[:-1]
|
||||
|
||||
|
||||
def generate_cookie(cookie_path):
|
||||
gdd = ChromeDriverDownloader()
|
||||
chrome_driver = gdd.download_and_install()
|
||||
|
||||
# TODO: load previous cookies so it does not ask to re verify using an
|
||||
# email code each time
|
||||
driver = webdriver.Chrome(executable_path=chrome_driver[1])
|
||||
|
||||
driver.get('https://www.humblebundle.com/login')
|
||||
|
||||
while '/home/library' not in driver.current_url:
|
||||
# Waiting for the user to login
|
||||
time.sleep(.25)
|
||||
|
||||
cookie_str = _get_cookie_str(driver)
|
||||
with open(cookie_path, 'w') as f:
|
||||
f.write(cookie_str)
|
||||
|
||||
logger.info("Saved cookies to " + cookie_path)
|
||||
|
||||
driver.quit()
|
||||
Loading…
Reference in New Issue
Block a user