fix: only do 8 jobs at once in update & verify
Prevent lag which can lead to unusable system (especially when using hdiffpatch) Also print more
This commit is contained in:
parent
1957f8265b
commit
cc6f3996af
4
setup.py
4
setup.py
@ -10,12 +10,12 @@ README = (HERE / "README.md").read_text()
|
||||
|
||||
setup(
|
||||
name='worthless',
|
||||
version='2.2.13',
|
||||
version='2.2.14',
|
||||
packages=['worthless', 'worthless.classes', 'worthless.classes.launcher', 'worthless.classes.installer'],
|
||||
url='https://git.froggi.es/tretrauit/worthless-launcher',
|
||||
license='MIT License',
|
||||
author='tretrauit',
|
||||
author_email='tretrauit@cachyos.org',
|
||||
author_email='tretrauit@gmail.org',
|
||||
description='A worthless CLI launcher written in Python.',
|
||||
long_description=README,
|
||||
long_description_content_type="text/markdown",
|
||||
|
@ -4,4 +4,4 @@ Launcher = launcher.Launcher
|
||||
Installer = installer.Installer
|
||||
|
||||
|
||||
__version__ = "2.2.13"
|
||||
__version__ = "2.2.14"
|
||||
|
0
worthless/__main__.py
Normal file → Executable file
0
worthless/__main__.py
Normal file → Executable file
0
worthless/cli.py
Normal file → Executable file
0
worthless/cli.py
Normal file → Executable file
@ -56,7 +56,7 @@ def calculate_md5(file_to_calculate):
|
||||
return ""
|
||||
with file_to_calculate.open("rb") as f:
|
||||
file_hash = hashlib.md5()
|
||||
while chunk := f.read(8192):
|
||||
while chunk := f.read(1024 * 1024):
|
||||
file_hash.update(chunk)
|
||||
return file_hash.hexdigest()
|
||||
|
||||
@ -181,7 +181,7 @@ class Installer:
|
||||
Path(self.temp_path).mkdir(parents=True, exist_ok=True)
|
||||
config_file = self._gamedir.joinpath("config.ini")
|
||||
self._config_file = config_file
|
||||
self._download_chunk = 8192
|
||||
self._download_chunk = 1024 * 1024
|
||||
self._overseas = overseas
|
||||
self._version = None
|
||||
self._launcher = Launcher(self._gamedir, overseas=self._overseas)
|
||||
@ -347,8 +347,10 @@ class Installer:
|
||||
hdifffiles = []
|
||||
for x in (await asyncio.to_thread(archive.read, "hdifffiles.txt")).decode().split("\n"):
|
||||
if x:
|
||||
hdifffiles.append(json.loads(x)["remoteName"])
|
||||
hdifffiles.append(json.loads(x.strip())["remoteName"])
|
||||
patch_jobs = []
|
||||
cur_jobs = []
|
||||
count = 0
|
||||
for file in hdifffiles:
|
||||
current_game_file = self._gamedir.joinpath(file)
|
||||
if not await current_game_file.exists():
|
||||
@ -381,9 +383,19 @@ class Installer:
|
||||
await old_file.rename(old_file.with_suffix(old_suffix))
|
||||
|
||||
files.remove(patch_file)
|
||||
patch_jobs.append(extract_and_patch(current_game_file, patch_file))
|
||||
# Limit to 8 process running so it doesn't hang the PC.
|
||||
if count == 7:
|
||||
patch_jobs.append(cur_jobs)
|
||||
cur_jobs = []
|
||||
count = 0
|
||||
cur_jobs.append(extract_and_patch(current_game_file, patch_file))
|
||||
count += 1
|
||||
|
||||
# The last list may have count < 7 and the above code will not add them
|
||||
patch_jobs.append(cur_jobs)
|
||||
for jobs in patch_jobs:
|
||||
await asyncio.gather(*jobs)
|
||||
|
||||
await asyncio.gather(*patch_jobs)
|
||||
except Exception as e:
|
||||
print(f"Error while reading hdifffiles.txt: {e}")
|
||||
|
||||
@ -411,7 +423,7 @@ class Installer:
|
||||
if not game_archive.exists():
|
||||
raise FileNotFoundError(f"Update archive {game_archive} not found")
|
||||
|
||||
self._update(game_archive=game_archive)
|
||||
await self._update(game_archive=game_archive)
|
||||
# Update game version on local variable.
|
||||
self._version = await self.get_game_version()
|
||||
self.set_version_config()
|
||||
@ -544,21 +556,33 @@ class Installer:
|
||||
contents = await pkg_version.read_text()
|
||||
|
||||
async def verify_file(file_to_verify, md5):
|
||||
print("Verifying file:", file_to_verify)
|
||||
file_md5 = await asyncio.to_thread(calculate_md5, file_to_verify)
|
||||
if file_md5 == md5:
|
||||
return None
|
||||
if ignore_mismatch:
|
||||
# print(f"MD5 does not match for {file_to_verify}, expected md5: {md5}, actual md5: {file_md5}")
|
||||
return file_to_verify, md5, file_md5
|
||||
raise ValueError(f"MD5 does not match for {file_to_verify}, expected md5: {md5}, actual md5: {file_md5}")
|
||||
|
||||
verify_jobs = []
|
||||
cur_jobs = []
|
||||
count = 0
|
||||
for content in contents.split("\r\n"):
|
||||
if not content.strip():
|
||||
continue
|
||||
if count >= 7:
|
||||
verify_jobs.append(cur_jobs)
|
||||
cur_jobs = []
|
||||
count = 0
|
||||
info = json.loads(content)
|
||||
verify_jobs.append(verify_file(self._gamedir.joinpath(info["remoteName"]), info["md5"]))
|
||||
cur_jobs.append(verify_file(self._gamedir.joinpath(info["remoteName"]), info["md5"]))
|
||||
count += 1
|
||||
|
||||
verify_result = await asyncio.gather(*verify_jobs)
|
||||
verify_jobs.append(cur_jobs)
|
||||
verify_result = []
|
||||
for jobs in verify_jobs:
|
||||
verify_result.extend(await asyncio.gather(*jobs))
|
||||
failed_files = []
|
||||
for file in verify_result:
|
||||
if file is not None:
|
||||
|
Loading…
Reference in New Issue
Block a user