Multi-threaded download with aria2

Close #24
This commit is contained in:
Howard20181 2022-08-24 01:54:24 +08:00
parent 9905925e64
commit 9768767b17
8 changed files with 251 additions and 95 deletions

View File

@ -1,42 +0,0 @@
#!/usr/bin/python
import sys
import requests
import zipfile
import os
import urllib.request
import json
import re
from pathlib import Path
if not os.path.exists(Path.cwd().parent / "download"):
os.makedirs(Path.cwd().parent / "download")
download_dir = Path.cwd().parent / "download"
arch = sys.argv[1]
variant = sys.argv[2]
abi_map = {"x64": "x86_64", "arm64": "arm64"}
# TODO: keep it 11.0 since opengapps does not support 12+ yet
# As soon as opengapps is available for 12+, we need to get the sdk/release from build.prop and
# download the corresponding version
release = "11.0"
try:
res = requests.get(f"https://api.opengapps.org/list")
j = json.loads(res.content)
link = {i["name"]: i for i in j["archs"][abi_map[arch]]
["apis"][release]["variants"]}[variant]["zip"]
except Exception:
print("Failed to fetch from opengapps api, fallbacking to sourceforge rss...")
res = requests.get(
f'https://sourceforge.net/projects/opengapps/rss?path=/{abi_map[arch]}&limit=100')
link = re.search(f'https://.*{abi_map[arch]}/.*{release}.*{variant}.*\.zip/download', res.text).group().replace(
'.zip/download', '.zip').replace('sourceforge.net/projects/opengapps/files', 'downloads.sourceforge.net/project/opengapps')
print(f"downloading link: {link}", flush=True)
out_file = download_dir / "gapps.zip"
if not os.path.isfile(out_file):
urllib.request.urlretrieve(link, out_file)

View File

@ -1,51 +1,47 @@
#
# This file is part of MagiskOnWSALocal.
#
# MagiskOnWSALocal is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# MagiskOnWSALocal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with MagiskOnWSALocal. If not, see <https://www.gnu.org/licenses/>.
#
# Copyright (C) 2022 LSPosed Contributors
#
#!/usr/bin/python #!/usr/bin/python
import sys import sys
import urllib.request
import zipfile import zipfile
import os import os
import json
import requests
from pathlib import Path from pathlib import Path
import platform import platform
magisk_apk = sys.argv[2]
is_x86_64 = platform.machine() in ("AMD64", "x86_64") is_x86_64 = platform.machine() in ("AMD64", "x86_64")
host_abi = "x64" if is_x86_64 else "arm64" host_abi = "x64" if is_x86_64 else "arm64"
arch = sys.argv[1]
if not os.path.exists(Path.cwd().parent / "download"): magisk_zip = sys.argv[2]
os.makedirs(Path.cwd().parent / "download")
download_dir = Path.cwd().parent / "download"
if not os.path.exists(Path.cwd().parent / sys.argv[3] / "magisk"): if not os.path.exists(Path.cwd().parent / sys.argv[3] / "magisk"):
os.makedirs(Path.cwd().parent / sys.argv[3] / "magisk") os.makedirs(Path.cwd().parent / sys.argv[3] / "magisk")
workdir = Path.cwd().parent / sys.argv[3] / "magisk" workdir = Path.cwd().parent / sys.argv[3] / "magisk"
if not magisk_apk:
magisk_apk = "stable"
if magisk_apk == "stable" or magisk_apk == "beta" or magisk_apk == "canary" or magisk_apk == "debug":
magisk_apk = json.loads(requests.get(
f"https://github.com/topjohnwu/magisk-files/raw/master/{magisk_apk}.json").content)['magisk']['link']
out_file = download_dir / "magisk.zip"
arch = sys.argv[1]
abi_map = {"x64": ["x86_64", "x86"], "arm64": ["arm64-v8a", "armeabi-v7a"]} abi_map = {"x64": ["x86_64", "x86"], "arm64": ["arm64-v8a", "armeabi-v7a"]}
if not os.path.isfile(out_file):
urllib.request.urlretrieve(magisk_apk, out_file)
def extract_as(zip, name, as_name, dir): def extract_as(zip, name, as_name, dir):
info = zip.getinfo(name) info = zip.getinfo(name)
info.filename = as_name info.filename = as_name
zip.extract(info, workdir / dir) zip.extract(info, workdir / dir)
with zipfile.ZipFile(magisk_zip) as zip:
with zipfile.ZipFile(out_file) as zip:
extract_as( extract_as(
zip, f"lib/{ abi_map[arch][0] }/libmagisk64.so", "magisk64", "magisk") zip, f"lib/{ abi_map[arch][0] }/libmagisk64.so", "magisk64", "magisk")
extract_as( extract_as(

View File

@ -1,3 +1,22 @@
#
# This file is part of MagiskOnWSALocal.
#
# MagiskOnWSALocal is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# MagiskOnWSALocal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with MagiskOnWSALocal. If not, see <https://www.gnu.org/licenses/>.
#
# Copyright (C) 2022 LSPosed Contributors
#
#!/usr/bin/python #!/usr/bin/python
import sys import sys

View File

@ -1,3 +1,22 @@
#
# This file is part of MagiskOnWSALocal.
#
# MagiskOnWSALocal is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# MagiskOnWSALocal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with MagiskOnWSALocal. If not, see <https://www.gnu.org/licenses/>.
#
# Copyright (C) 2022 LSPosed Contributors
#
from __future__ import annotations from __future__ import annotations
from io import TextIOWrapper from io import TextIOWrapper
from os import system, path from os import system, path

View File

@ -0,0 +1,62 @@
#
# This file is part of MagiskOnWSALocal.
#
# MagiskOnWSALocal is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# MagiskOnWSALocal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with MagiskOnWSALocal. If not, see <https://www.gnu.org/licenses/>.
#
# Copyright (C) 2022 LSPosed Contributors
#
#!/usr/bin/python
import sys
import requests
import os
import json
import re
from pathlib import Path
arch = sys.argv[1]
variant = sys.argv[2]
download_dir = Path.cwd().parent / "download" if sys.argv[3] == "" else Path(sys.argv[3]).resolve()
tempScript = sys.argv[4]
print(f"Generating OpenGapps download link: arch={arch} variant={variant}", flush=True)
abi_map = {"x64": "x86_64", "arm64": "arm64"}
# TODO: keep it 11.0 since opengapps does not support 12+ yet
# As soon as opengapps is available for 12+, we need to get the sdk/release from build.prop and
# download the corresponding version
release = "11.0"
try:
res = requests.get(f"https://api.opengapps.org/list")
j = json.loads(res.content)
link = {i["name"]: i for i in j["archs"][abi_map[arch]]
["apis"][release]["variants"]}[variant]["zip"]
except Exception:
print("Failed to fetch from opengapps api, fallbacking to sourceforge rss...")
res = requests.get(
f'https://sourceforge.net/projects/opengapps/rss?path=/{abi_map[arch]}&limit=100')
link = re.search(f'https://.*{abi_map[arch]}/.*{release}.*{variant}.*\.zip/download', res.text).group().replace(
'.zip/download', '.zip').replace('sourceforge.net/projects/opengapps/files', 'downloads.sourceforge.net/project/opengapps')
print(f"download link: {link}", flush=True)
out_file = download_dir / "gapps.zip"
if not os.path.isfile(out_file):
# urllib.request.urlretrieve(link, out_file)
with open(download_dir/tempScript, 'a') as f:
f.writelines(f'{link}\n')
f.writelines(f' dir={download_dir}\n')
f.writelines(f' out=gapps.zip\n')
f.close

View File

@ -0,0 +1,46 @@
#
# This file is part of MagiskOnWSALocal.
#
# MagiskOnWSALocal is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# MagiskOnWSALocal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with MagiskOnWSALocal. If not, see <https://www.gnu.org/licenses/>.
#
# Copyright (C) 2022 LSPosed Contributors
#
#!/usr/bin/python
import sys
import os
import json
import requests
from pathlib import Path
magisk_apk = sys.argv[1]
download_dir = Path.cwd().parent / "download" if sys.argv[2] == "" else Path(sys.argv[2]).resolve()
tempScript = sys.argv[3]
print(f"Generating Magisk download link: release type={magisk_apk}", flush=True)
if not magisk_apk:
magisk_apk = "stable"
if magisk_apk == "stable" or magisk_apk == "beta" or magisk_apk == "canary" or magisk_apk == "debug":
magisk_apk = json.loads(requests.get(
f"https://github.com/topjohnwu/magisk-files/raw/master/{magisk_apk}.json").content)['magisk']['link']
print(f"download link: {magisk_apk}", flush=True)
out_file = download_dir / "magisk.zip"
if not os.path.isfile(out_file):
# urllib.request.urlretrieve(magisk_apk, out_file)
with open(download_dir/tempScript, 'a') as f:
f.writelines(f'{magisk_apk}\n')
f.writelines(f' dir={download_dir}\n')
f.writelines(f' out=magisk.zip\n')

View File

@ -1,3 +1,22 @@
#
# This file is part of MagiskOnWSALocal.
#
# MagiskOnWSALocal is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# MagiskOnWSALocal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with MagiskOnWSALocal. If not, see <https://www.gnu.org/licenses/>.
#
# Copyright (C) 2022 LSPosed Contributors
#
#!/usr/bin/python #!/usr/bin/python
import sys import sys
@ -7,20 +26,20 @@ from xml.dom import minidom
import html import html
import warnings import warnings
import re import re
import zipfile
import os import os
import urllib.request
from pathlib import Path from pathlib import Path
warnings.filterwarnings("ignore") warnings.filterwarnings("ignore")
arch = sys.argv[1] arch = sys.argv[1]
release_type_map = {"retail": "Retail", "release preview": "RP", release_type_map = {"retail": "Retail", "release preview": "RP",
"insider slow": "WIS", "insider fast": "WIF"} "insider slow": "WIS", "insider fast": "WIF"}
release_type = release_type_map[sys.argv[2]] if sys.argv[2] != "" else "Retail" release_type = release_type_map[sys.argv[2]] if sys.argv[2] != "" else "Retail"
download_dir = Path.cwd().parent / "download" if sys.argv[3] == "" else Path(sys.argv[3]).resolve()
tempScript = sys.argv[4]
cat_id = '858014f3-3934-4abe-8078-4aa193e74ca8' cat_id = '858014f3-3934-4abe-8078-4aa193e74ca8'
print(f"Generating WSA download link: arch={arch} release_type={release_type}", flush=True)
with open(Path.cwd().parent / ("xml/GetCookie.xml"), "r") as f: with open(Path.cwd().parent / ("xml/GetCookie.xml"), "r") as f:
cookie_content = f.read() cookie_content = f.read()
@ -61,15 +80,20 @@ for node in doc.getElementsByTagName('SecuredFragment'):
with open(Path.cwd().parent / "xml/FE3FileUrl.xml", "r") as f: with open(Path.cwd().parent / "xml/FE3FileUrl.xml", "r") as f:
file_content = f.read() file_content = f.read()
if not os.path.exists(Path.cwd().parent / "download"):
os.makedirs(Path.cwd().parent / "download") if not os.path.exists(download_dir):
os.makedirs(download_dir)
tmpdownlist = open(download_dir/tempScript, 'a')
for i, v, f in identities: for i, v, f in identities:
if re.match(f"Microsoft\.UI\.Xaml\..*_{arch}_.*\.appx", f): if re.match(f"Microsoft\.UI\.Xaml\..*_{arch}_.*\.appx", f):
out_file = Path.cwd().parent / "download/xaml.appx" out_file = download_dir / "xaml.appx"
elif re.match(f"Microsoft\.VCLibs\..*_{arch}_.*\.appx", f): out_file_name = "xaml.appx"
out_file = Path.cwd().parent / "download/vclibs.appx" # elif re.match(f"Microsoft\.VCLibs\..+\.UWPDesktop_.*_{arch}_.*\.appx", f):
# out_file = download_dir / "vclibs.appx"
# out_file_name = "vclibs.appx"
elif re.match(f"MicrosoftCorporationII\.WindowsSubsystemForAndroid_.*\.msixbundle", f): elif re.match(f"MicrosoftCorporationII\.WindowsSubsystemForAndroid_.*\.msixbundle", f):
out_file = Path.cwd().parent / "download/wsa.zip" out_file = download_dir / "wsa.zip"
out_file_name = "wsa.zip"
else: else:
continue continue
out = requests.post( out = requests.post(
@ -83,5 +107,12 @@ for i, v, f in identities:
url = l.getElementsByTagName("Url")[0].firstChild.nodeValue url = l.getElementsByTagName("Url")[0].firstChild.nodeValue
if len(url) != 99: if len(url) != 99:
if not os.path.isfile(out_file): if not os.path.isfile(out_file):
print(f"downloading link: {url} to {out_file}", flush=True) print(f"download link: {url} to {out_file}", flush=True)
urllib.request.urlretrieve(url, out_file) # urllib.request.urlretrieve(url, out_file)
tmpdownlist.writelines(url + '\n')
tmpdownlist.writelines(f' dir={download_dir}\n')
tmpdownlist.writelines(f' out={out_file_name}\n')
tmpdownlist.writelines(f'https://aka.ms/Microsoft.VCLibs.{arch}.14.00.Desktop.appx\n')
tmpdownlist.writelines(f' dir={download_dir}\n')
tmpdownlist.writelines(f' out=vclibs.appx\n')
tmpdownlist.close

View File

@ -1,3 +1,22 @@
#
# This file is part of MagiskOnWSALocal.
#
# MagiskOnWSALocal is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# MagiskOnWSALocal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with MagiskOnWSALocal. If not, see <https://www.gnu.org/licenses/>.
#
# Copyright (C) 2022 LSPosed Contributors
#
#!/bin/bash #!/bin/bash
if [ ! "$BASH_VERSION" ] ; then if [ ! "$BASH_VERSION" ] ; then
echo "Please do not use sh to run this script, just execute it directly" 1>&2 echo "Please do not use sh to run this script, just execute it directly" 1>&2
@ -12,6 +31,7 @@ cd "$(dirname "$0")" || exit 1
trap 'rm -rf -- "${WORK_DIR:?}"' EXIT trap 'rm -rf -- "${WORK_DIR:?}"' EXIT
WORK_DIR=$(mktemp -d -t wsa-build-XXXXXXXXXX_) || exit 1 WORK_DIR=$(mktemp -d -t wsa-build-XXXXXXXXXX_) || exit 1
DOWNLOAD_DIR=../download DOWNLOAD_DIR=../download
DOWNLOAD_CONF_NAME=download.list
OUTPUT_DIR=../output OUTPUT_DIR=../output
MOUNT_DIR="$WORK_DIR"/system MOUNT_DIR="$WORK_DIR"/system
@ -64,7 +84,7 @@ function Gen_Rand_Str {
} }
echo "Dependencies" echo "Dependencies"
sudo apt update && sudo apt -y install setools lzip wine winetricks patchelf whiptail e2fsprogs python3-pip sudo apt update && sudo apt -y install setools lzip wine winetricks patchelf whiptail e2fsprogs python3-pip aria2
sudo python3 -m pip install requests sudo python3 -m pip install requests
cp -r ../wine/.cache/* ~/.cache cp -r ../wine/.cache/* ~/.cache
winetricks msxml6 || abort winetricks msxml6 || abort
@ -154,9 +174,23 @@ fi
clear clear
echo -e "ARCH=$ARCH\nRELEASE_TYPE=$RELEASE_TYPE\nMAGISK_VER=$MAGISK_VER\nGAPPS_VARIANT=$GAPPS_VARIANT\nREMOVE_AMAZON=$REMOVE_AMAZON\nROOT_SOL=$ROOT_SOL\nCOMPRESS_OUTPUT=$COMPRESS_OUTPUT" echo -e "ARCH=$ARCH\nRELEASE_TYPE=$RELEASE_TYPE\nMAGISK_VER=$MAGISK_VER\nGAPPS_VARIANT=$GAPPS_VARIANT\nREMOVE_AMAZON=$REMOVE_AMAZON\nROOT_SOL=$ROOT_SOL\nCOMPRESS_OUTPUT=$COMPRESS_OUTPUT"
echo "Download WSA" echo "Generate Download Links"
python3 downloadWSA.py "$ARCH" "$RELEASE_TYPE" || abort python3 generateWSALinks.py "$ARCH" "$RELEASE_TYPE" "$DOWNLOAD_DIR" "$DOWNLOAD_CONF_NAME" || abort
echo -e "Download done\n" python3 generateMagiskLink.py "$MAGISK_VER" "$DOWNLOAD_DIR" "$DOWNLOAD_CONF_NAME" || abort
if [ $GAPPS_VARIANT != 'none' ] && [ $GAPPS_VARIANT != '' ]; then
if [ $GAPPS_BRAND = "OpenGapps" ]; then
# TODO: Keep it pico since other variants of opengapps are unable to boot successfully
python3 generateGappsLink.py "$ARCH" "pico" "$DOWNLOAD_DIR" "$DOWNLOAD_CONF_NAME" || abort
# python3 generateGappsLink.py "$ARCH" "$GAPPS_VARIANT" "$DOWNLOAD_DIR" "$DOWNLOAD_CONF_NAME" || abort
fi
fi
trap 'rm -f -- "${DOWNLOAD_DIR:?}/${DOWNLOAD_CONF_NAME}"' EXIT
echo "Download Artifacts"
if ! aria2c --no-conf --log-level=info --log="$DOWNLOAD_DIR/aria2_download.log" -x16 -s16 -j5 -c -R -m0 -d"$DOWNLOAD_DIR" -i"$DOWNLOAD_DIR"/"$DOWNLOAD_CONF_NAME"; then
echo "We have encountered an error while downloading files."
exit 1
fi
echo "Extract WSA" echo "Extract WSA"
WSA_WORK_ENV="${WORK_DIR:?}"/ENV WSA_WORK_ENV="${WORK_DIR:?}"/ENV
@ -166,20 +200,11 @@ python3 extractWSA.py "$ARCH" "$WORK_DIR" || abort
echo -e "Extract done\n" echo -e "Extract done\n"
source "${WORK_DIR:?}/ENV" source "${WORK_DIR:?}/ENV"
echo "Download Magisk" echo "Extract Magisk"
python3 downloadMagisk.py "$ARCH" "$MAGISK_VER" "$WORK_DIR" || abort python3 extractMagisk.py "$ARCH" "$DOWNLOAD_DIR/magisk.zip" "$WORK_DIR" || abort
echo -e "done\n" echo -e "done\n"
if [ $GAPPS_VARIANT != 'none' ] && [ $GAPPS_VARIANT != '' ]; then if [ $GAPPS_VARIANT != 'none' ] && [ $GAPPS_VARIANT != '' ]; then
if [ $GAPPS_BRAND = "OpenGapps" ]; then
echo "Download OpenGApps"
if [ "$WSA_MAIN_VER" -ge 2204 ]; then
python3 downloadGapps.py "$ARCH" "pico" || abort # TODO: Keep it pico since other variants of opengapps are unable to boot successfully
else
python3 downloadGapps.py "$ARCH" "$GAPPS_VARIANT" || abort
fi
echo -e "Download done\n"
fi
echo "Extract GApps" echo "Extract GApps"
mkdir -p "$WORK_DIR"/gapps || abort mkdir -p "$WORK_DIR"/gapps || abort
if [ $GAPPS_BRAND = "OpenGapps" ]; then if [ $GAPPS_BRAND = "OpenGapps" ]; then