Virtual-World-RE/NeoGF
4
0
Fork 0
mirror of https://github.com/Virtual-World-RE/NeoGF.git synced 2024-11-15 12:45:34 +01:00
Code Issues Releases Wiki Activity

Update pzztest.py

Browse Source
This commit is contained in:
tmpz23 2022-01-16 17:38:39 +01:00 committed by GitHub
parent 50a80cc439
commit 4b5b5b6026
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 72 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
pzztest.py
View File

@ -8,25 +8,25 @@ import shutil
TPL_MAGIC_NUMBER = b"\x00\x20\xAF\x30" # http://virtualre.rf.gd/index.php/TPL_(Format_de_fichier) TPL_MAGIC_NUMBER = b"\x00\x20\xAF\x30" # http://virtualre.rf.gd/index.php/TPL_(Format_de_fichier)
unpack_path = Path("unpack")
repack_path = Path("repack")
afsdump_path = Path("afs_data/root")
# compare le sha256 de chaque PZZ des dossiers passés en argument # compare all files sha256 from folder1 and folder2
# -> affiche le nom de fichier en cas de différence # -> print the filename if there is a difference
def verify_sha256(folder1: Path, folder2: Path): def verify_sha256(folder1: Path, folder2: Path):
invalid_files_count = 0 invalid_files_count = 0
for pzz_file_name in os.listdir(folder1): for pzz_path in folder1.glob("*.pzz"):
with (folder1 / pzz_file_name).open("rb") as f1, (folder2 / pzz_file_name).open("rb") as f2: if hashlib.sha256( pzz_path.read_bytes() ).hexdigest() != hashlib.sha256( (folder2 / pzz_path.name).read_bytes() ).hexdigest() :
if hashlib.sha256( f1.read() ).hexdigest() != hashlib.sha256( f2.read() ).hexdigest() : print(f"ERROR - INVALID FILE : {pzz_file_name}")
print(f"ERROR - INVALID FILE : {pzz_file_name}") invalid_files_count +=1
invalid_files_count +=1 print(f"Invalid files : {invalid_files_count}/{len(list(folder1.glob('*')))}")
print(f"Invalid files : {invalid_files_count}/{len(os.listdir(folder1))}")
# compare le sha256 des deux fichiers passés en argument
# -> affiche le nom de fichier en cas de différence # compare sha256 of two files
# -> print the filename if there is a difference
def verify_sha256_2(file1: Path, file2: Path): def verify_sha256_2(file1: Path, file2: Path):
with file1.open("rb") as f1, file2.open("rb") as f2: return hashlib.sha256( file1.read_bytes() ).hexdigest() == hashlib.sha256( file2.read_bytes() ).hexdigest()
if hashlib.sha256( f1.read() ).hexdigest() != hashlib.sha256( f2.read() ).hexdigest() :
return False
return True
def get_argparser(): def get_argparser():
@ -39,15 +39,15 @@ def get_argparser():
group.add_argument('-tbup', '--test-batch-unpack-pack', action='store_true', help=""" group.add_argument('-tbup', '--test-batch-unpack-pack', action='store_true', help="""
-tbup source_pzz_folder -tbup source_pzz_folder
source_pzz_folder : put all pzz in this folder source_pzz_folder : put all pzz in this folder
pzzu : will be created with all unpacked pzz from pzz folder unpack_path : will be created with all unpacked pzz from pzz folder
pzz2 : will be created with all packed pzz from pzzu folder repack_path : will be created with all packed pzz from unpack_path folder
print file_name when sha256 is different between source_pzz_folder and pzz2 folder""") print file_name when sha256 is different between source_pzz_folder and repack_path folder""")
group.add_argument('-tbunpzzpzz', '--test-batch-unpzz-pzz', action='store_true', help=""" group.add_argument('-tbunpzzpzz', '--test-batch-unpzz-pzz', action='store_true', help="""
-tbunpzzpzz source_pzz_folder -tbunpzzpzz source_pzz_folder
source_pzz_folder : put all pzz in this folder source_pzz_folder : put all pzz in this folder
pzzu : will be created with all unpzz pzz from pzz folder unpack_path : will be created with all unpzz pzz from pzz folder
pzz2 : will be created with all pzz(pzz_folder) from pzzu folder repack_path : will be created with all pzz(pzz_folder) from unpack_path folder
print file_name when sha256 is different between source_pzz_folder and pzz2 folder""") print file_name when sha256 is different between source_pzz_folder and repack_path folder""")
group.add_argument('-tctplh', '--test-check-tpl-headers', action='store_true', help="-tctplh afs_data_folder : check all files headers in the afs_data and print those who have the tpl magicfile") group.add_argument('-tctplh', '--test-check-tpl-headers', action='store_true', help="-tctplh afs_data_folder : check all files headers in the afs_data and print those who have the tpl magicfile")
group.add_argument('-tcd', '--test-check-decompress', action='store_true', help=""" group.add_argument('-tcd', '--test-check-decompress', action='store_true', help="""
pzz : put all pzz in this folder pzz : put all pzz in this folder
@ -66,9 +66,8 @@ if __name__ == '__main__':
print("# TEST : DECOMPRESS COMPRESS") print("# TEST : DECOMPRESS COMPRESS")
listofinvalid = [] listofinvalid = []
for filename in os.listdir(args.input_path): for pzzp_path in p_input.glob('*'):
file = open(os.path.join(args.input_path, filename), 'rb') original_bytes = pzzp_path.read_bytes()
original_bytes = file.read()
decomp_bytes = pzztool.pzz_decompress(original_bytes) decomp_bytes = pzztool.pzz_decompress(original_bytes)
recomp_bytes = pzztool.pzz_compress(decomp_bytes) recomp_bytes = pzztool.pzz_compress(decomp_bytes)
@ -76,68 +75,74 @@ if __name__ == '__main__':
recomp_digest = hashlib.sha256(recomp_bytes).hexdigest() recomp_digest = hashlib.sha256(recomp_bytes).hexdigest()
if original_digest != recomp_digest: if original_digest != recomp_digest:
listofinvalid.append(f"{filename} : ({original_digest}) ({recomp_digest})") print(f"Invalid sha256 for {pzzp_path} : ({original_digest}) ({recomp_digest})")
file.close()
for invalid in listofinvalid:
print(invalid)
elif args.test_batch_unpack_pack: elif args.test_batch_unpack_pack:
print("# TEST : BATCH UNPACK PACK") print("# TEST : BATCH UNPACK PACK")
# Remove unpack_path and repack_path
if unpack_path.is_dir():
shutil.rmtree(unpack_path)
if repack_path.is_dir():
shutil.rmtree(repack_path)
os.system(f"python pzztool.py -bu {args.input_path} pzzu") if os.system(f"python pzztool.py -bu {p_input} {unpack_path}") != 0:
os.system("python pzztool.py -bp pzzu pzz2") raise Exception("Error while batch unpack.")
verify_sha256(p_input, Path("pzz2")) if os.system(f"python pzztool.py -bp {unpack_path} {repack_path}") != 0:
raise Exception("Error while batch pack.")
verify_sha256(p_input, repack_path)
elif args.test_batch_unpzz_pzz: elif args.test_batch_unpzz_pzz:
os.system(f"python pzztool.py -bunpzz {args.input_path} pzzu") # Remove unpack_path and repack_path
os.system("python pzztool.py -bpzz pzzu pzz2") if unpack_path.is_dir():
verify_sha256(p_input, Path("pzz2")) shutil.rmtree(unpack_path)
if repack_path.is_dir():
shutil.rmtree(repack_path)
# Clean du dossier pzz2 généré par le script if os.system(f"python pzztool.py -bunpzz {p_input} {unpack_path}") != 0:
shutil.rmtree("pzz2") raise Exception("Error while batch unpzz.")
if os.system(f"python pzztool.py -bpzz {unpack_path} {repack_path}") != 0:
raise Exception("Error while batch pzz.")
verify_sha256(p_input, repack_path)
""" """
si pzz : U -> decomp / testé sur les fichiers car l'unpzz décompresse par défaut if pzz : U -> decomp / already tested because unpzz let it decompressed by default
si pzz : U -> comp / à tester if pzz : U -> comp / has to be tested
si pzz : C -> decomp / testé sur les fichiers car l'unpzz décompresse par défaut if pzz : C -> decomp / already tested because unpzz decompress by default
si pzz : C -> comp / à tester if pzz : C -> comp / has to be tested
""" """
# On parcours tous les dossiers : si U -> comp ; si C -> comp : compression de tous les fichiers peu importe le type # Remove repack_path
for pzz_folder in os.listdir("pzzu"): shutil.rmtree(repack_path)
for pzz_file_part_name in os.listdir("pzzu/"+pzz_folder):
# créé un nouveau fichier compressé, à côté de l'original # For all unpack_path folder we compress the file (if U -> comp ; if C -> comp)
os.system(f"python pzztool.py -c pzzu/{pzz_folder}/{pzz_file_part_name}") for pzzpart_path in unpack_path.glob('*/*'):
# supprime l'original # create a new compressed file without removing the original file
os.remove(f"pzzu/{pzz_folder}/{pzz_file_part_name}") if os.system(f"python pzztool.py -c {pzzpart_path}") != 0:
os.system("python pzztool.py -bpzz pzzu pzz2") raise Exception("Error while compress.")
verify_sha256(p_input, Path("pzz2")) # remove the original
elif args.test_check_tpl_headers: os.remove(f"{pzzpart_path}")
# Démontre que SEUL les TPLs ont ce magicnumber
# TEST OK if os.system(f"python pzztool.py -bpzz {unpack_path} {repack_path}") != 0:
print("# TEST : CHECK TPLs HEADERS") raise Exception("Error while batch pzz.")
for afs_data_filename in os.listdir(p_input): verify_sha256(p_input, repack_path)
with open(p_input / afs_data_filename, "rb") as afs_data_file:
if TPL_MAGIC_NUMBER == afs_data_file.read(4) and Path(afs_data_filename).suffix != ".tpl":
print(f"TPL magicfile found : afs_data.afs/{afs_data_filename}")
elif args.test_check_decompress: elif args.test_check_decompress:
print("# TEST : CHECK DECOMPRESS") print("# TEST : CHECK DECOMPRESS")
os.system(f"python pzztool.py -bunpzz {args.input_path} pzzu") if os.system(f"python pzztool.py -bunpzz {p_input} {unpack_path}") != 0:
raise Exception("Error while batch unpzz.")
invalid_files_count = 0 invalid_files_count = 0
total = 0 total = 0
# check that all TPLs length is a multiple of 32 # check that all TPLs length is a multiple of 32
for p in Path("pzzu").glob("**/*.tpl"): for tpl_path in unpack_path.glob("**/*.tpl"):
if p.is_file(): if p.is_file():
#print(Path(p).stat().st_size, end=' ')
total+=1 total+=1
if (Path(p).stat().st_size % 32) != 0: if (tpl_path.stat().st_size % 32) != 0:
print(f"Invalid TPL file length modulo 32 ({Path(p).stat().st_size % 32}) - {p}") print(f"Invalid TPL file length modulo 32 ({tpl_path.stat().st_size % 32}) - {tpl_path}")
invalid_files_count += 1 invalid_files_count += 1
print(f"Invalid files : {invalid_files_count}/{total}") print(f"Invalid files : {invalid_files_count}/{total}")
elif args.test_compare_position: elif args.test_compare_position:
# FULL_AFS_FILE_DUMP contient tous les fichiers de l'afs_data.afs et pzzu le résultat de pzztool.py -bunpzz sur l'ensemble des pzz # FULL_AFS_FILE_DUMP contains all unpacked files from afs_data.afs
# Comparaisons à effectuer : # unpack_path contains result of pzztool.py -bunpzz on all pzz files
# What you have to compare (prove that files of borgs (plxxxx.pzz) are positional and same as pl files in the root of afs_data) :
# pzztest.py -tcp 0 data.bin # pzztest.py -tcp 0 data.bin
# Les fichiers de l'afs_data sont parfois data2 / data3 ou absents # Some afs_data files are named data2 or data3 and it's sometime absent
# pzztest.py -tcp 2 hit.bin # pzztest.py -tcp 2 hit.bin
# pzztest.py -tcp 3 mot.bin # pzztest.py -tcp 3 mot.bin
# pzztest.py -tcp 4 _mdl.arc # pzztest.py -tcp 4 _mdl.arc
@ -147,8 +152,8 @@ if __name__ == '__main__':
# pzztest.py -tcp 8 c_mdl.arc # pzztest.py -tcp 8 c_mdl.arc
# pzztest.py -tcp 9 k_mdl.arc # pzztest.py -tcp 9 k_mdl.arc
for pzzpart_path in Path("pzzu").glob("**/00"+args.input_path+"*"): for pzzpart_path in unpack_path.glob(f"**/00{p_input}*"):
file_path = Path("FULL_AFS_FILE_DUMP/"+pzzpart_path.parent.name+args.output_path) file_path = afsdump_path / pzzpart_path.parent.name / p_output
if pzzpart_path.parent.name[:2] == "pl": if pzzpart_path.parent.name[:2] == "pl":
if not file_path.is_file(): if not file_path.is_file():