mirror of
https://github.com/DragoonAethis/itch-dl.git
synced 2025-01-04 01:11:52 +01:00
Force use UTF-8 end-to-end for site downloading
Itch always uses UTF-8. This change prevents garbled output with non-Latin scripts if requests or BS4 does not guess that correctly.
This commit is contained in:
parent
fb2e4c6736
commit
e61ef0cba3
itch_dl
@ -28,7 +28,19 @@ class ItchApiClient:
|
||||
self.requests.mount("https://", adapter)
|
||||
self.requests.mount("http://", adapter)
|
||||
|
||||
def get(self, endpoint: str, append_api_key: bool = True, **kwargs) -> requests.Response:
|
||||
def get(
|
||||
self,
|
||||
endpoint: str,
|
||||
append_api_key: bool = True,
|
||||
guess_encoding: bool = False,
|
||||
**kwargs
|
||||
) -> requests.Response:
|
||||
"""Wrapper around `requests.get`.
|
||||
|
||||
:param endpoint: Path to fetch on the specified base URL.
|
||||
:param append_api_key: Send an authenticated API request.
|
||||
:param guess_encoding: Let requests guess the response encoding.
|
||||
"""
|
||||
if append_api_key:
|
||||
params = kwargs.get('data') or {}
|
||||
|
||||
@ -42,4 +54,11 @@ class ItchApiClient:
|
||||
else:
|
||||
url = self.base_url + endpoint
|
||||
|
||||
return self.requests.get(url, **kwargs)
|
||||
r = self.requests.get(url, **kwargs)
|
||||
|
||||
# Itch always returns UTF-8 pages and API responses. Force
|
||||
# UTF-8 everywhere, except for binary file downloads.
|
||||
if not guess_encoding:
|
||||
r.encoding = 'utf-8'
|
||||
|
||||
return r
|
||||
|
@ -191,7 +191,7 @@ class GameDownloader:
|
||||
file to the provided path and returns the final URL that was downloaded."""
|
||||
try:
|
||||
# No timeouts, chunked uploads, default retry strategy, should be all good?
|
||||
with self.client.get(url, data=credentials, stream=True) as r:
|
||||
with self.client.get(url, data=credentials, stream=True, guess_encoding=True) as r:
|
||||
r.raise_for_status()
|
||||
|
||||
if download_path is not None: # ...and it will be for external downloads.
|
||||
@ -320,8 +320,8 @@ class GameDownloader:
|
||||
except Exception as e:
|
||||
errors.append(f"Cover art download failed (this is not fatal): {e}")
|
||||
|
||||
with open(paths['site'], 'w') as f:
|
||||
f.write(site.prettify())
|
||||
with open(paths['site'], 'wb') as f:
|
||||
f.write(site.prettify(encoding='utf-8'))
|
||||
|
||||
with open(paths['metadata'], 'w') as f:
|
||||
json.dump(metadata, f, indent=4)
|
||||
|
Loading…
Reference in New Issue
Block a user