Multi-threaded download with aria2

Close #24

scripts/downloadGapps.py

@@ -1,42 +0,0 @@
-#!/usr/bin/python
-
-import sys
-
-import requests
-import zipfile
-import os
-import urllib.request
-import json
-import re
-from pathlib import Path
-
-if not os.path.exists(Path.cwd().parent / "download"):
-    os.makedirs(Path.cwd().parent / "download")
-download_dir = Path.cwd().parent / "download"
-
-arch = sys.argv[1]
-
-variant = sys.argv[2]
-abi_map = {"x64": "x86_64", "arm64": "arm64"}
-# TODO: keep it 11.0 since opengapps does not support 12+ yet
-# As soon as opengapps is available for 12+, we need to get the sdk/release from build.prop and
-# download the corresponding version
-release = "11.0"
-try:
-    res = requests.get(f"https://api.opengapps.org/list")
-    j = json.loads(res.content)
-    link = {i["name"]: i for i in j["archs"][abi_map[arch]]
-            ["apis"][release]["variants"]}[variant]["zip"]
-except Exception:
-    print("Failed to fetch from opengapps api, fallbacking to sourceforge rss...")
-    res = requests.get(
-        f'https://sourceforge.net/projects/opengapps/rss?path=/{abi_map[arch]}&limit=100')
-    link = re.search(f'https://.*{abi_map[arch]}/.*{release}.*{variant}.*\.zip/download', res.text).group().replace(
-        '.zip/download', '.zip').replace('sourceforge.net/projects/opengapps/files', 'downloads.sourceforge.net/project/opengapps')
-
-print(f"downloading link: {link}", flush=True)
-
-out_file = download_dir / "gapps.zip"
-
-if not os.path.isfile(out_file):
-    urllib.request.urlretrieve(link, out_file)

scripts/extractMagisk.py

@@ -1,51 +1,47 @@
+# 
+# This file is part of MagiskOnWSALocal.
+#
+# MagiskOnWSALocal is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# MagiskOnWSALocal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with MagiskOnWSALocal.  If not, see <https://www.gnu.org/licenses/>.
+#
+# Copyright (C) 2022 LSPosed Contributors
+#
+
 #!/usr/bin/python
 
 import sys
 
-import urllib.request
 import zipfile
 import os
-import json
-import requests
 from pathlib import Path
 import platform
 
-magisk_apk = sys.argv[2]
-
 is_x86_64 = platform.machine() in ("AMD64", "x86_64")
 host_abi = "x64" if is_x86_64 else "arm64"
-
+arch = sys.argv[1]
-if not os.path.exists(Path.cwd().parent / "download"):
+magisk_zip = sys.argv[2]
-    os.makedirs(Path.cwd().parent / "download")
-download_dir = Path.cwd().parent / "download"
-
 if not os.path.exists(Path.cwd().parent / sys.argv[3] / "magisk"):
     os.makedirs(Path.cwd().parent / sys.argv[3] / "magisk")
 workdir = Path.cwd().parent / sys.argv[3] / "magisk"
 
-if not magisk_apk:
-    magisk_apk = "stable"
-if magisk_apk == "stable" or magisk_apk == "beta" or magisk_apk == "canary" or magisk_apk == "debug":
-    magisk_apk = json.loads(requests.get(
-        f"https://github.com/topjohnwu/magisk-files/raw/master/{magisk_apk}.json").content)['magisk']['link']
-
-out_file = download_dir / "magisk.zip"
-
-arch = sys.argv[1]
-
 abi_map = {"x64": ["x86_64", "x86"], "arm64": ["arm64-v8a", "armeabi-v7a"]}
 
-if not os.path.isfile(out_file):
-    urllib.request.urlretrieve(magisk_apk, out_file)
-
-
 def extract_as(zip, name, as_name, dir):
     info = zip.getinfo(name)
     info.filename = as_name
     zip.extract(info, workdir / dir)
 
-
+with zipfile.ZipFile(magisk_zip) as zip:
-with zipfile.ZipFile(out_file) as zip:
     extract_as(
         zip, f"lib/{ abi_map[arch][0] }/libmagisk64.so", "magisk64", "magisk")
     extract_as(

scripts/extractWSA.py

@@ -1,3 +1,22 @@
+# 
+# This file is part of MagiskOnWSALocal.
+#
+# MagiskOnWSALocal is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# MagiskOnWSALocal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with MagiskOnWSALocal.  If not, see <https://www.gnu.org/licenses/>.
+#
+# Copyright (C) 2022 LSPosed Contributors
+#
+
 #!/usr/bin/python
 
 import sys

scripts/fixGappsProp.py

@@ -1,3 +1,22 @@
+# 
+# This file is part of MagiskOnWSALocal.
+#
+# MagiskOnWSALocal is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# MagiskOnWSALocal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with MagiskOnWSALocal.  If not, see <https://www.gnu.org/licenses/>.
+#
+# Copyright (C) 2022 LSPosed Contributors
+#
+
 from __future__ import annotations
 from io import TextIOWrapper
 from os import system, path

scripts/generateGappsLink.py

@@ -0,0 +1,62 @@
+# 
+# This file is part of MagiskOnWSALocal.
+#
+# MagiskOnWSALocal is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# MagiskOnWSALocal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with MagiskOnWSALocal.  If not, see <https://www.gnu.org/licenses/>.
+#
+# Copyright (C) 2022 LSPosed Contributors
+#
+
+#!/usr/bin/python
+
+import sys
+
+import requests
+import os
+import json
+import re
+from pathlib import Path
+
+arch = sys.argv[1]
+variant = sys.argv[2]
+download_dir = Path.cwd().parent / "download" if sys.argv[3] == "" else Path(sys.argv[3]).resolve()
+tempScript = sys.argv[4]
+print(f"Generating OpenGapps download link: arch={arch} variant={variant}", flush=True)
+abi_map = {"x64": "x86_64", "arm64": "arm64"}
+# TODO: keep it 11.0 since opengapps does not support 12+ yet
+# As soon as opengapps is available for 12+, we need to get the sdk/release from build.prop and
+# download the corresponding version
+release = "11.0"
+try:
+    res = requests.get(f"https://api.opengapps.org/list")
+    j = json.loads(res.content)
+    link = {i["name"]: i for i in j["archs"][abi_map[arch]]
+            ["apis"][release]["variants"]}[variant]["zip"]
+except Exception:
+    print("Failed to fetch from opengapps api, fallbacking to sourceforge rss...")
+    res = requests.get(
+        f'https://sourceforge.net/projects/opengapps/rss?path=/{abi_map[arch]}&limit=100')
+    link = re.search(f'https://.*{abi_map[arch]}/.*{release}.*{variant}.*\.zip/download', res.text).group().replace(
+        '.zip/download', '.zip').replace('sourceforge.net/projects/opengapps/files', 'downloads.sourceforge.net/project/opengapps')
+
+print(f"download link: {link}", flush=True)
+
+out_file = download_dir / "gapps.zip"
+
+if not os.path.isfile(out_file):
+    # urllib.request.urlretrieve(link, out_file)
+    with open(download_dir/tempScript, 'a') as f:
+        f.writelines(f'{link}\n')
+        f.writelines(f'  dir={download_dir}\n')
+        f.writelines(f'  out=gapps.zip\n')
+        f.close

scripts/generateMagiskLink.py

@@ -0,0 +1,46 @@
+# 
+# This file is part of MagiskOnWSALocal.
+#
+# MagiskOnWSALocal is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# MagiskOnWSALocal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with MagiskOnWSALocal.  If not, see <https://www.gnu.org/licenses/>.
+#
+# Copyright (C) 2022 LSPosed Contributors
+#
+
+#!/usr/bin/python
+
+import sys
+
+import os
+import json
+import requests
+from pathlib import Path
+
+magisk_apk = sys.argv[1]
+download_dir = Path.cwd().parent / "download" if sys.argv[2] == "" else Path(sys.argv[2]).resolve()
+tempScript = sys.argv[3]
+print(f"Generating Magisk download link: release type={magisk_apk}", flush=True)
+if not magisk_apk:
+    magisk_apk = "stable"
+if magisk_apk == "stable" or magisk_apk == "beta" or magisk_apk == "canary" or magisk_apk == "debug":
+    magisk_apk = json.loads(requests.get(
+        f"https://github.com/topjohnwu/magisk-files/raw/master/{magisk_apk}.json").content)['magisk']['link']
+print(f"download link: {magisk_apk}", flush=True)
+out_file = download_dir / "magisk.zip"
+
+if not os.path.isfile(out_file):
+    # urllib.request.urlretrieve(magisk_apk, out_file)
+    with open(download_dir/tempScript, 'a') as f:
+        f.writelines(f'{magisk_apk}\n')
+        f.writelines(f'  dir={download_dir}\n')
+        f.writelines(f'  out=magisk.zip\n')

scripts/generateWSALinks.py

@@ -1,3 +1,22 @@
+# 
+# This file is part of MagiskOnWSALocal.
+#
+# MagiskOnWSALocal is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# MagiskOnWSALocal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with MagiskOnWSALocal.  If not, see <https://www.gnu.org/licenses/>.
+#
+# Copyright (C) 2022 LSPosed Contributors
+#
+
 #!/usr/bin/python
 
 import sys
@@ -7,20 +26,20 @@ from xml.dom import minidom
 import html
 import warnings
 import re
-import zipfile
 import os
-import urllib.request
 from pathlib import Path
 
 warnings.filterwarnings("ignore")
 
 arch = sys.argv[1]
+
 release_type_map = {"retail": "Retail", "release preview": "RP",
                     "insider slow": "WIS", "insider fast": "WIF"}
 release_type = release_type_map[sys.argv[2]] if sys.argv[2] != "" else "Retail"
-
+download_dir = Path.cwd().parent / "download" if sys.argv[3] == "" else Path(sys.argv[3]).resolve()
+tempScript = sys.argv[4]
 cat_id = '858014f3-3934-4abe-8078-4aa193e74ca8'
-
+print(f"Generating WSA download link: arch={arch} release_type={release_type}", flush=True)
 with open(Path.cwd().parent / ("xml/GetCookie.xml"), "r") as f:
     cookie_content = f.read()
 
@@ -61,15 +80,20 @@ for node in doc.getElementsByTagName('SecuredFragment'):
 
 with open(Path.cwd().parent / "xml/FE3FileUrl.xml", "r") as f:
     file_content = f.read()
-if not os.path.exists(Path.cwd().parent / "download"):
+
-    os.makedirs(Path.cwd().parent / "download")
+if not os.path.exists(download_dir):
+    os.makedirs(download_dir)
+tmpdownlist = open(download_dir/tempScript, 'a')
 for i, v, f in identities:
     if re.match(f"Microsoft\.UI\.Xaml\..*_{arch}_.*\.appx", f):
-        out_file = Path.cwd().parent / "download/xaml.appx"
+        out_file = download_dir / "xaml.appx"
-    elif re.match(f"Microsoft\.VCLibs\..*_{arch}_.*\.appx", f):
+        out_file_name = "xaml.appx"
-        out_file = Path.cwd().parent / "download/vclibs.appx"
+    # elif re.match(f"Microsoft\.VCLibs\..+\.UWPDesktop_.*_{arch}_.*\.appx", f):
+    #     out_file = download_dir / "vclibs.appx"
+    #     out_file_name = "vclibs.appx"
     elif re.match(f"MicrosoftCorporationII\.WindowsSubsystemForAndroid_.*\.msixbundle", f):
-        out_file = Path.cwd().parent / "download/wsa.zip"
+        out_file = download_dir / "wsa.zip"
+        out_file_name = "wsa.zip"
     else:
         continue
     out = requests.post(
@@ -83,5 +107,12 @@ for i, v, f in identities:
         url = l.getElementsByTagName("Url")[0].firstChild.nodeValue
         if len(url) != 99:
             if not os.path.isfile(out_file):
-                print(f"downloading link: {url} to {out_file}", flush=True)
+                print(f"download link: {url} to {out_file}", flush=True)
-                urllib.request.urlretrieve(url, out_file)
+                # urllib.request.urlretrieve(url, out_file)
+                tmpdownlist.writelines(url + '\n')
+                tmpdownlist.writelines(f'  dir={download_dir}\n')
+                tmpdownlist.writelines(f'  out={out_file_name}\n')
+tmpdownlist.writelines(f'https://aka.ms/Microsoft.VCLibs.{arch}.14.00.Desktop.appx\n')
+tmpdownlist.writelines(f'  dir={download_dir}\n')
+tmpdownlist.writelines(f'  out=vclibs.appx\n')
+tmpdownlist.close

scripts/run.sh

@@ -1,3 +1,22 @@
+# 
+# This file is part of MagiskOnWSALocal.
+#
+# MagiskOnWSALocal is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# MagiskOnWSALocal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with MagiskOnWSALocal.  If not, see <https://www.gnu.org/licenses/>.
+#
+# Copyright (C) 2022 LSPosed Contributors
+#
+
 #!/bin/bash
 if [ ! "$BASH_VERSION" ] ; then
     echo "Please do not use sh to run this script, just execute it directly" 1>&2
@@ -12,6 +31,7 @@ cd "$(dirname "$0")" || exit 1
 trap 'rm -rf -- "${WORK_DIR:?}"' EXIT
 WORK_DIR=$(mktemp -d -t wsa-build-XXXXXXXXXX_) || exit 1
 DOWNLOAD_DIR=../download
+DOWNLOAD_CONF_NAME=download.list
 OUTPUT_DIR=../output
 MOUNT_DIR="$WORK_DIR"/system
 
@@ -64,7 +84,7 @@ function Gen_Rand_Str {
 }
 
 echo "Dependencies"
-sudo apt update && sudo apt -y install setools lzip wine winetricks patchelf whiptail e2fsprogs python3-pip
+sudo apt update && sudo apt -y install setools lzip wine winetricks patchelf whiptail e2fsprogs python3-pip aria2
 sudo python3 -m pip install requests
 cp -r ../wine/.cache/* ~/.cache
 winetricks msxml6 || abort
@@ -154,9 +174,23 @@ fi
 clear
 echo -e "ARCH=$ARCH\nRELEASE_TYPE=$RELEASE_TYPE\nMAGISK_VER=$MAGISK_VER\nGAPPS_VARIANT=$GAPPS_VARIANT\nREMOVE_AMAZON=$REMOVE_AMAZON\nROOT_SOL=$ROOT_SOL\nCOMPRESS_OUTPUT=$COMPRESS_OUTPUT"
 
-echo "Download WSA"
+echo "Generate Download Links"
-python3 downloadWSA.py "$ARCH" "$RELEASE_TYPE" || abort
+python3 generateWSALinks.py "$ARCH" "$RELEASE_TYPE" "$DOWNLOAD_DIR" "$DOWNLOAD_CONF_NAME" || abort
-echo -e "Download done\n"
+python3 generateMagiskLink.py "$MAGISK_VER" "$DOWNLOAD_DIR" "$DOWNLOAD_CONF_NAME" || abort
+if [ $GAPPS_VARIANT != 'none' ] && [ $GAPPS_VARIANT != '' ]; then
+    if [ $GAPPS_BRAND = "OpenGapps" ]; then
+        # TODO: Keep it pico since other variants of opengapps are unable to boot successfully
+        python3 generateGappsLink.py "$ARCH" "pico" "$DOWNLOAD_DIR" "$DOWNLOAD_CONF_NAME" || abort
+        # python3 generateGappsLink.py "$ARCH" "$GAPPS_VARIANT" "$DOWNLOAD_DIR" "$DOWNLOAD_CONF_NAME" || abort
+    fi
+fi
+trap 'rm -f -- "${DOWNLOAD_DIR:?}/${DOWNLOAD_CONF_NAME}"' EXIT
+
+echo "Download Artifacts"
+if ! aria2c --no-conf --log-level=info --log="$DOWNLOAD_DIR/aria2_download.log" -x16 -s16 -j5 -c -R -m0 -d"$DOWNLOAD_DIR" -i"$DOWNLOAD_DIR"/"$DOWNLOAD_CONF_NAME"; then
+  echo "We have encountered an error while downloading files."
+  exit 1
+fi
 
 echo "Extract WSA"
 WSA_WORK_ENV="${WORK_DIR:?}"/ENV
@@ -166,20 +200,11 @@ python3 extractWSA.py "$ARCH" "$WORK_DIR" || abort
 echo -e "Extract done\n"
 source "${WORK_DIR:?}/ENV"
 
-echo "Download Magisk"
+echo "Extract Magisk"
-python3 downloadMagisk.py "$ARCH" "$MAGISK_VER" "$WORK_DIR" || abort
+python3 extractMagisk.py "$ARCH" "$DOWNLOAD_DIR/magisk.zip" "$WORK_DIR" || abort
 echo -e "done\n"
 
 if [ $GAPPS_VARIANT != 'none' ] && [ $GAPPS_VARIANT != '' ]; then
-    if [ $GAPPS_BRAND = "OpenGapps" ]; then
-        echo "Download OpenGApps"
-        if [ "$WSA_MAIN_VER" -ge 2204 ]; then
-            python3 downloadGapps.py "$ARCH" "pico" || abort # TODO: Keep it pico since other variants of opengapps are unable to boot successfully
-        else
-            python3 downloadGapps.py "$ARCH" "$GAPPS_VARIANT" || abort
-        fi
-        echo -e "Download done\n"
-    fi
     echo "Extract GApps"
     mkdir -p "$WORK_DIR"/gapps || abort
     if [ $GAPPS_BRAND = "OpenGapps" ]; then