diff --git a/SourceStream/file/download.py b/SourceStream/file/download.py index d8bbc32..9fb3265 100644 --- a/SourceStream/file/download.py +++ b/SourceStream/file/download.py @@ -2,27 +2,54 @@ from ..vendor.SplashPyUtils import logger -import requests +import asyncio +import aiohttp +import time import os -def download_tool(package): - package_name = package["package"] - package_version = package["version"] - package_version_split = package["version"].split(".") - - package_url = SourceStream.YAML_SOURCES[package_name]["url"].replace("{VERSION}", package_version) - package_url = package_url.replace("{VERSION_SEPARATE_UNDERSCORE}", package_version.replace(".", "_")) - - if(len(package_version_split) >= 1): package_url = package_url.replace("{VERSION_MAJOR}", package_version_split[0]) - if(len(package_version_split) >= 2): package_url = package_url.replace("{VERSION_MINOR}", package_version_split[1]) - if(len(package_version_split) >= 3): package_url = package_url.replace("{VERSION_REVISION}", package_version_split[2]) - if(len(package_version_split) >= 4): package_url = package_url.replace("{VERSION_BUILD}", package_version_split[3]) - - package_filename = os.path.basename(package_url) - package_path = os.path.join(SourceStream.DIR_INPUT, package_filename) - - if not os.path.exists(package_path): - r = requests.get(package_url) - open(package_path, 'wb').write(r.content) - logger.log.ok(f"Downloaded package \x1b[1;37m{package_name}({package_version})\x1b[0m Successfully.") +async def download_tool(packages): + + package_list = [] + + for package in packages: + package_name = package["package"] + package_version = package["version"] + package_version_split = package["version"].split(".") + + package_url = SourceStream.YAML_SOURCES[package_name]["url"].replace("{VERSION}", package_version) + package_url = package_url.replace("{VERSION_SEPARATE_UNDERSCORE}", package_version.replace(".", "_")) + + if(len(package_version_split) >= 1): package_url = package_url.replace("{VERSION_MAJOR}", package_version_split[0]) + if(len(package_version_split) >= 2): package_url = package_url.replace("{VERSION_MINOR}", package_version_split[1]) + if(len(package_version_split) >= 3): package_url = package_url.replace("{VERSION_REVISION}", package_version_split[2]) + if(len(package_version_split) >= 4): package_url = package_url.replace("{VERSION_BUILD}", package_version_split[3]) + + package_filename = os.path.basename(package_url) + package_path = os.path.join(SourceStream.DIR_INPUT, package_filename) + + if not os.path.exists(package_path): + package_list.append({"name": package_name, "version": package_version, "url": package_url, "path": package_path}) + else: + logger.log.ok(f"Package \x1b[1;37m{package_name}({package_version})\x1b[0m already downloaded.") + + async with aiohttp.ClientSession() as session: + tasks = [] + for package in package_list: + tasks.append(download_package(session, package)) + + await asyncio.gather(*tasks) + +async def download_package(session, package): + async with session.get(package["url"]) as response: + if response.status == 200: + with open(package["path"], 'wb') as f_handle: + while True: + chunk = await response.content.read(1024) + if not chunk: + break + f_handle.write(chunk) + logger.log.ok(f"Downloaded package \x1b[1;37m{package['name']}({package['version']})\x1b[0m Successfully.") + else: + logger.log.fail(f"Failed to download package \x1b[1;37m{package['name']}({package['version']})\x1b[0m.") + logger.log.fail(f"Status code: {response.status}") diff --git a/SourceStream/file/repack.py b/SourceStream/file/repack.py index 08757d5..42190b0 100644 --- a/SourceStream/file/repack.py +++ b/SourceStream/file/repack.py @@ -3,8 +3,9 @@ from ..vendor.SplashPyUtils import logger import shutil -import tarfile +# import tarfile import os +import subprocess def repack_tool(packages): package_name = packages["package"] @@ -17,12 +18,7 @@ def repack_tool(packages): try: # Extract tar file in the input directory. - with tarfile.open(package_path_input) as f: - if not SourceStream.YAML_SOURCES[package_name]["subdir"]: - os.makedirs(package_path_folder, exist_ok=True) - f.extractall(package_path_folder) - else: - f.extractall(SourceStream.DIR_INPUT) + subprocess.run(["tar", "-xf", package_path_input, "-C", SourceStream.DIR_INPUT], check=True) # Check if the folder just extracted needs to be renamed. if isinstance(SourceStream.YAML_SOURCES[package_name]["subdir"], str): @@ -33,22 +29,24 @@ def repack_tool(packages): # Create package directory where repacked tar file will be saved in. os.makedirs(os.path.join(SourceStream.DIR_OUTPUT, package_name), exist_ok=True) + # Create a list of files and directories to be included in the tar file + tar_items = [package_fullname] + + # Check if configuration files exist for these packages + if os.path.exists(os.path.join(SourceStream.DIR_BUILTSCRIPTS, package_name, package_version)): + tar_items.append(os.path.join(SourceStream.DIR_BUILTSCRIPTS, package_name, package_version, "manifest.yml")) + + # Check if there are some extra folders included + if os.path.exists(os.path.join(SourceStream.DIR_BUILTSCRIPTS, package_name, package_version, "build")): + tar_items.append(os.path.join(SourceStream.DIR_BUILTSCRIPTS, package_name, package_version, "build")) + if os.path.exists(os.path.join(SourceStream.DIR_BUILTSCRIPTS, package_name, package_version, "patch")): + tar_items.append(os.path.join(SourceStream.DIR_BUILTSCRIPTS, package_name, package_version, "patch")) + else: + logger.log.warn(f"The package {package_name} does not have a manifest.") + # Create tar file - with tarfile.open(os.path.join(SourceStream.DIR_OUTPUT, package_name, f"{package_fullname}.tar.xz"), "w:xz") as tar: - tar.add(os.path.join(SourceStream.DIR_INPUT, package_fullname), arcname=package_fullname) - - # Check if configuration files exist for these packages - if os.path.exists(os.path.join(SourceStream.DIR_BUILTSCRIPTS, package_name, package_version)): - tar.add(os.path.join(SourceStream.DIR_BUILTSCRIPTS, package_name, package_version, "manifest.yml"), arcname="manifest.yml") - - # Check if there are some extra folders included - if os.path.exists(os.path.join(SourceStream.DIR_BUILTSCRIPTS, package_name, package_version, "build")): - tar.add(os.path.join(SourceStream.DIR_BUILTSCRIPTS, package_name, package_version, "build"), arcname="build") - if os.path.exists(os.path.join(SourceStream.DIR_BUILTSCRIPTS, package_name, package_version, "patch")): - tar.add(os.path.join(SourceStream.DIR_BUILTSCRIPTS, package_name, package_version, "patch"), arcname="patch") - - else: - logger.log.warn(f"The package {package_name} does not have a manifest.") + tar_output_path = os.path.join(SourceStream.DIR_OUTPUT, package_name, f"{package_fullname}.tar.xz") + subprocess.run(["tar", "-cJf", tar_output_path, "-C", SourceStream.DIR_INPUT] + tar_items, check=True) # Cleanup the input folder shutil.rmtree(package_path_folder) diff --git a/SourceStream/task/tasks/repackPackages.py b/SourceStream/task/tasks/repackPackages.py index 5f0d16f..19f82a8 100644 --- a/SourceStream/task/tasks/repackPackages.py +++ b/SourceStream/task/tasks/repackPackages.py @@ -6,6 +6,7 @@ import multiprocessing +import asyncio import shutil import dload import sys @@ -48,9 +49,9 @@ def run(): tools = SourceStream.PACKAGES logger.log.info("Starting to download and check packages, this can take a while...") - for tool in tools: - download.download_tool(tool) + asyncio.run(download.download_tool(tools)) + logger.log.info("Starting to repack packages, this can take a while...") # Repack packages with multiprocessing