#!/usr/bin/env python3 from configparser import ConfigParser from datetime import datetime import logging from math import ceil import os from pathlib import Path import re import time __version__ = "0.2.0" __author__ = "rigodron, algoflash, GGLinnk" __license__ = "MIT" __status__ = "developpement" # Creating afstool.py was a challenge because there is many implementations # and configurations possible. In one hand allowing to configure the rebuild # was usefull but in another it was also important to control errors generated # by those conf files. Using custom exceptions was necessary. # Not tested by afstest.py: class AfsInvalidFileLenError(Exception): pass class AfsEmptyAfsError(Exception): pass class AfsInvalidFilenameDirectoryLengthError(Exception): pass class AfsInvalidAfsFolderError(Exception): pass # Tested by afstest.py: class AfsInvalidMagicNumberError(Exception): pass class AfsInvalidFilesRebuildStrategy(Exception): pass class AfsFilenameDirectoryValueError(Exception): pass class AfsInvalidFilePathError(Exception): pass class AfsInvalidFieldsCountError(Exception): pass class AfsIndexValueError(Exception): pass class AfsIndexOverflowError(Exception): pass class AfsIndexCollisionError(Exception): pass class AfsOffsetValueError(Exception): pass class AfsOffsetAlignError(Exception): pass class AfsOffsetCollisionError(Exception): pass class AfsFdOffsetOffsetValueError(Exception): pass class AfsFdOffsetValueError(Exception): pass class AfsFdLastAttributeTypeValueError(Exception): pass class AfsFdOffsetCollisionError(Exception): pass class AfsEmptyBlockValueError(Exception): pass class AfsEmptyBlockAlignError(Exception): pass def normalize_parent(path_str:str): "Normalize the parent of a path to avoid out of extract folder access." if Path(path_str).parent == Path("."): return path_str parent_str = str(Path(path_str).parent).replace(".", "") while parent_str[0] == "/" or parent_str[0] == "\\": parent_str = parent_str[1:] return parent_str + "/" + Path(path_str).name class FilenameResolver: """ Constructor: system path of the unpack folder DESCRIPTION Use sys/filename_resolver.csv to resolve filename to their index in the TOC. Allow also to rename files since the FD and the TOC are not rebuild during pack. The resolver is necessary in multiple cases: * When multiple packed files have the same name in the FD * When there is no FD * When names contains invalid path operator (not implemented yet) """ __sys_path = None # names_dict: {unpacked_filename: toc_index, ... } __names_dict = None __resolve_buffer = "" __separator = '?' def __init__(self, sys_path:Path): self.__sys_path = sys_path self.__names_dict = {} self.__load() def __load(self): "Load names_dict if there is a csv" if (self.__sys_path / "filename_resolver.csv").is_file(): self.__resolve_buffer = (self.__sys_path / "filename_resolver.csv").read_text() for line in self.__resolve_buffer.split('\n'): name_tuple = line.split(self.__separator) self.__names_dict[name_tuple[1]] = int(name_tuple[0]) def save(self): "Save the resolve_buffer containing formated names_dict to the csv if not empty" if len(self.__resolve_buffer) > 0: logging.info(f"Writting {Path('sys/filename_resolver.csv')}") (self.__sys_path / "filename_resolver.csv").write_text(self.__resolve_buffer[:-1]) def resolve_new(self, file_index:int, filename:str): """ Resolve generate a unique filename when unpacking input: file_index = int input: filename = string return the filename or new generated filename if duplicated """ normalized_str = normalize_parent(filename) if filename != normalized_str: filename = normalized_str if filename not in self.__names_dict: self.__names_dict[filename] = file_index self.__resolve_buffer += f"{file_index}{self.__separator}{filename}\n" return filename if filename in self.__names_dict: i = 1 new_filename = f"{Path(filename).parent / Path(filename).stem} ({i}){Path(filename).suffix}" while new_filename in self.__names_dict: i+=1 new_filename = f"{Path(filename).parent / Path(filename).stem} ({i}){Path(filename).suffix}" self.__names_dict[new_filename] = file_index self.__resolve_buffer += f"{file_index}{self.__separator}{new_filename}\n" return new_filename self.__names_dict[filename] = file_index return filename def add(self, file_index:int, unpacked_filename:str): "Add new entry forcing the unpacked_filename" self.__names_dict[unpacked_filename] = file_index self.__resolve_buffer += f"{file_index}{self.__separator}{unpacked_filename}\n" def resolve_from_index(self, file_index:int, filename:str): """ input: file_index = int input: filename = str return previously generated filename using the index of the file in the TOC else return filename """ for filename_key, fileindex_value in self.__names_dict.items(): if fileindex_value == file_index: return filename_key return filename class Afs: """ DESCRIPTION Afs handle all operations needed by the command parser http://wiki.xentax.com/index.php/GRAF:AFS_AFS """ MAGIC_00 = b"AFS\x00" MAGIC_20 = b"AFS\x20" # The header and each files are aligned to 0x800 ALIGN = 0x800 # magic number and number of files HEADER_LEN = 8 # Each entry in the FD have 32 chars for filename and the rest for date and last_fd_attribute FILENAMEDIRECTORY_ENTRY_LEN = 0x30 __file_count = None # this offset is at the end of the TOC and sometimes there is pad __filenamedirectory_offset_offset = None # if there is a FD at the end of the AFS __filenamedirectory_offset = None __filenamedirectory_len = None __filenamedirectory = None __tableofcontent = None def __get_magic(self): return bytes(self.__tableofcontent[0:4]) def __get_file_count(self): return int.from_bytes(self.__tableofcontent[4:8], "little") def __get_filenamedirectory_offset(self): return int.from_bytes(self.__tableofcontent[self.__filenamedirectory_offset_offset:self.__filenamedirectory_offset_offset+4], "little") def __get_filenamedirectory_len(self): return int.from_bytes(self.__tableofcontent[self.__filenamedirectory_offset_offset+4:self.__filenamedirectory_offset_offset+8], "little") def __get_file_offset(self, fileindex:int): return int.from_bytes(self.__tableofcontent[Afs.HEADER_LEN+fileindex*8:Afs.HEADER_LEN+fileindex*8+4], "little") def __get_file_len(self, fileindex:int): return int.from_bytes(self.__tableofcontent[Afs.HEADER_LEN+fileindex*8+4:Afs.HEADER_LEN+fileindex*8+8], "little") def __get_file_name(self, fileindex:int): return self.__filenamedirectory[fileindex*Afs.FILENAMEDIRECTORY_ENTRY_LEN:fileindex*Afs.FILENAMEDIRECTORY_ENTRY_LEN+32].split(b"\x00")[0].decode("utf-8") def __get_file_fdlast(self, fileindex:int): return int.from_bytes(self.__filenamedirectory[fileindex*Afs.FILENAMEDIRECTORY_ENTRY_LEN+44:fileindex*Afs.FILENAMEDIRECTORY_ENTRY_LEN+48], "little") def __get_file_mtime(self, fileindex:int): mtime_data = self.__filenamedirectory[fileindex*Afs.FILENAMEDIRECTORY_ENTRY_LEN+32:fileindex*Afs.FILENAMEDIRECTORY_ENTRY_LEN+44] year = int.from_bytes(mtime_data[0:2], "little") month = int.from_bytes(mtime_data[2:4], "little") day = int.from_bytes(mtime_data[4:6], "little") hour = int.from_bytes(mtime_data[6:8], "little") minute = int.from_bytes(mtime_data[8:10], "little") second = int.from_bytes(mtime_data[10:12], "little") return time.mktime(datetime(year=year, month=month, day=day, hour=hour, minute=minute, second=second).timetuple()) def __patch_file_len(self, fileindex:int, file_len:int): # Patch file_len in the TOC self.__tableofcontent[Afs.HEADER_LEN+fileindex*8+4:Afs.HEADER_LEN+fileindex*8+8] = file_len.to_bytes(4, "little") def __patch_file_mtime(self, fileindex:int, mtime): mtime = datetime.fromtimestamp(mtime) self.__filenamedirectory[Afs.FILENAMEDIRECTORY_ENTRY_LEN*fileindex+32:Afs.FILENAMEDIRECTORY_ENTRY_LEN*fileindex+44] = \ mtime.year.to_bytes(2,"little")+ \ mtime.month.to_bytes(2,"little")+ \ mtime.day.to_bytes(2,"little")+ \ mtime.hour.to_bytes(2,"little")+ \ mtime.minute.to_bytes(2,"little")+\ mtime.second.to_bytes(2,"little") def __patch_fdlasts(self, fileindex:int, fd_last_attribute_type): "Patch FD last attributes according to the type" if type(fd_last_attribute_type) == int: # every entry has the same const value self.__filenamedirectory[fileindex*Afs.FILENAMEDIRECTORY_ENTRY_LEN+44:fileindex*Afs.FILENAMEDIRECTORY_ENTRY_LEN+48] = fd_last_attribute_type.to_bytes(4, "little") elif fd_last_attribute_type == "length": # self.__filenamedirectory[fileindex*Afs.FILENAMEDIRECTORY_ENTRY_LEN+44:fileindex*Afs.FILENAMEDIRECTORY_ENTRY_LEN+48] = self.__get_file_len(fileindex).to_bytes(4, "little") elif fd_last_attribute_type == "offset-length": # every odd index is changed according to the TOC lengths values with the serie: 0->updated_index=1 1->updated_index=3 2->updated_index=5 # updated_index = index*2+1 with index*2+1 < self.__file_count updated_fdlast_index = fileindex*2+1 if updated_fdlast_index < self.__file_count: self.__filenamedirectory[updated_fdlast_index*Afs.FILENAMEDIRECTORY_ENTRY_LEN+44:updated_fdlast_index*Afs.FILENAMEDIRECTORY_ENTRY_LEN+48] = self.__get_file_len(fileindex).to_bytes(4, "little") # fd_last_attribute_type == unknown def __pad(self, data:bytes): "Add padding to align datas to next block" if len(data) % Afs.ALIGN != 0: data += b"\x00" * (Afs.ALIGN - (len(data) % Afs.ALIGN)) return data def __clean_filenamedirectory(self): """ We can't know if there is a FD without searching and loading data for it So we have to clean loaded data if values are invalid """ self.__filenamedirectory = None self.__filenamedirectory_offset = None self.__filenamedirectory_len = None def __loadsys_from_afs(self, afs_file, afs_len:int): """ Load the TOC and the FD from an AFS file this operation is difficult because there are many cases possible: is there or not a FD? is there padding at the end of files offset/length list in the TOC? So we have to search and control values and test it for errors If there is no FD self.__filename_directory is None return True if there is a FD else None """ self.__tableofcontent = afs_file.read(Afs.HEADER_LEN) if self.__get_magic() not in [Afs.MAGIC_00, Afs.MAGIC_20]: raise AfsInvalidMagicNumberError("Error - Invalid AFS magic number.") self.__file_count = self.__get_file_count() self.__tableofcontent += afs_file.read(self.__file_count*8) tableofcontent_len = len(self.__tableofcontent) offset = tableofcontent_len # Now we have read the TOC and seeked to the end of it # next values could be FD offset and length if there is one # So we read 4 bytes to test if there is padding or not tmp_block = int.from_bytes(afs_file.read(4), "little") if tmp_block != 0: self.__filenamedirectory_offset_offset = offset self.__filenamedirectory_offset = tmp_block # Here it could be padding # If filenamedirectory_offset is not directly after the files offsets and lens # --> we search the next uint32 != 0 else: offset += 4 # We read by 0x800 blocks for better performances block_len = 0x800 tmp_block = afs_file.read(block_len) while tmp_block: match = re.search(b"^(?:\x00{4})*(?!\x00{4})(.{4})", tmp_block) # match next uint32 if match: self.__filenamedirectory_offset_offset = offset + match.start(1) self.__filenamedirectory_offset = int.from_bytes(match[1], "little") break offset += block_len tmp_block = afs_file.read(block_len) # This because we retrieve an int valid or not into fd offset if self.__filenamedirectory_offset is None: raise AfsEmptyAfsError("Error - Empty AFS.") afs_file.seek(self.__filenamedirectory_offset_offset+4) self.__filenamedirectory_len = int.from_bytes(afs_file.read(4), "little") # Test if offset of filenamedirectory is valid and if number of entries match between filenamedirectory and tableofcontent if self.__filenamedirectory_offset + self.__filenamedirectory_len > afs_len or \ self.__filenamedirectory_offset < self.__filenamedirectory_offset_offset or \ (tableofcontent_len - self.HEADER_LEN) / 8 != self.__filenamedirectory_len / Afs.FILENAMEDIRECTORY_ENTRY_LEN: self.__clean_filenamedirectory() return False afs_file.seek(self.__filenamedirectory_offset) self.__filenamedirectory = afs_file.read(self.__filenamedirectory_len) # Test if filename is correct by very basic pattern matching pattern = re.compile(b"^(?=.{32}$)[^\x00]+\x00+$") for i in range(self.__file_count): if not pattern.fullmatch(self.__filenamedirectory[i*Afs.FILENAMEDIRECTORY_ENTRY_LEN:i*Afs.FILENAMEDIRECTORY_ENTRY_LEN+32]): self.__clean_filenamedirectory() return False afs_file.seek(tableofcontent_len) # Here FD is valid and we read it's length self.__tableofcontent += afs_file.read(self.__filenamedirectory_offset_offset+8 - tableofcontent_len) return True def __loadsys_from_folder(self, sys_path:Path): "Load the TOC and FD from an unpacked afs. This time it's easier" self.__tableofcontent = bytearray( (sys_path / "tableofcontent.bin").read_bytes() ) self.__file_count = self.__get_file_count() # If there is a filenamedirectory we load it if (sys_path / "filenamedirectory.bin").is_file(): self.__filenamedirectory = bytearray((sys_path / "filenamedirectory.bin").read_bytes()) self.__filenamedirectory_offset_offset = len(self.__tableofcontent) - 8 self.__filenamedirectory_offset = self.__get_filenamedirectory_offset() self.__filenamedirectory_len = self.__get_filenamedirectory_len() if self.__filenamedirectory_len != len(self.__filenamedirectory): raise AfsInvalidFilenameDirectoryLengthError("Error - Tableofcontent filenamedirectory length does not match real filenamedirectory length.") def __print(self, title:str, lines_tuples, columns:list = list(range(7)), infos:str = ""): "Print is used for stats" stats_buffer = "#"*100+f"\n# {title}\n"+"#"*100+f"\n{infos}|"+"-"*99+"\n" if 0 in columns: stats_buffer += "| Index "; if 1 in columns: stats_buffer += "| b offset "; if 2 in columns: stats_buffer += "| e offset "; if 3 in columns: stats_buffer += "| length "; if 4 in columns: stats_buffer += "| YYYY-mm-dd HH:MM:SS "; if 5 in columns: stats_buffer += "| FD last "; if 6 in columns: stats_buffer += "| Filename"; stats_buffer += "\n|"+"-"*99+"\n" for line in lines_tuples: stats_buffer += line if type(line) == str else "| "+" | ".join(line)+"\n" print(stats_buffer, end='') def __get_offsets_map(self): """ This method is used to check the next file offset and control if there is overlapping during pack end offset not included (0,1) -> len=1 return a list of offsets where files and sys files begin """ # offsets_map is used to check next used offset when updating files # we also check if there is intersect between files offsets_map = [(0, len(self.__tableofcontent))] for i in range(self.__file_count): file_offset = self.__get_file_offset(i) offsets_map.append( (file_offset, file_offset + self.__get_file_len(i)) ) if self.__filenamedirectory: filenamedirectory_offset = self.__get_filenamedirectory_offset() offsets_map.append( (filenamedirectory_offset, filenamedirectory_offset + self.__get_filenamedirectory_len()) ) offsets_map.sort(key=lambda x: x[0]) # Check if there is problems in file memory mapping last_tuple = (-1, -1) for i, offsets_tuple in enumerate(offsets_map): if offsets_tuple[0] < last_tuple[1]: raise AfsOffsetCollisionError(f"Error - Multiple files use same file offsets ranges.") last_tuple = offsets_tuple offsets_map[i] = offsets_tuple[0] return offsets_map def __get_formated_map(self): """ This method is used for stats command end offset not included (0,1) -> len=1 """ files_map = [("SYS TOC ", "00000000", f"{len(self.__tableofcontent):08x}", f"{len(self.__tableofcontent):08x}", "SYS TOC"+' '*12, "SYS TOC ", "SYS TOC")] for i in range(self.__file_count): file_offset = self.__get_file_offset(i) file_len = self.__get_file_len(i) file_date = datetime.fromtimestamp(self.__get_file_mtime(i)).strftime("%Y-%m-%d %H:%M:%S") if self.__filenamedirectory else " "*19 filename = self.__get_file_name(i) if self.__filenamedirectory else f"{i:08}" fdlast = f"{self.__get_file_fdlast(i):08x}" if self.__filenamedirectory else " "*8 files_map.append((f"{i:08x}", f"{file_offset:08x}", f"{file_offset + file_len:08x}", f"{file_len:08x}", file_date, fdlast, filename)) if self.__filenamedirectory: files_map.append(("SYS FD ", f"{self.__filenamedirectory_offset:08x}", \ f"{self.__filenamedirectory_offset + len(self.__filenamedirectory):08x}", \ f"{len(self.__filenamedirectory):08x}", "SYS FD"+' '*13, "SYS FD ", "SYS FD")) return files_map def __get_fdlast_type(self): """ At the end of the FD there is 4 bytes used for different purposes To keep data we search what kind of data it is: return one of this values: * length * offset-length * 0x123 # (hex constant) * unknwon """ # Try to get the type of FD last attribute length_type = True offset_length_type = True constant_type = self.__get_file_fdlast(0) for i in range(self.__file_count): fd_last_attribute = self.__get_file_fdlast(i) if fd_last_attribute != self.__get_file_len(i): length_type = None if fd_last_attribute != self.__tableofcontent[8+i*4:8+i*4+4]: offset_length_type = None if fd_last_attribute != constant_type: constant_type = None if length_type: return "length" if offset_length_type: return "offset-length" if constant_type: return f"0x{constant_type:x}" logging.info("Unknown FD last attribute type.") return "unknown" def __write_rebuild_config(self, sys_path:Path, resolver:FilenameResolver): """ At the end of unpack we use this function to write the 2 files: * "sys/afs_rebuild.conf" * "sys/afs_rebuild.csv" this file will contains every parameters of the AFS to allow exact pack copy when possible (fd_last_atribute != unknown) see documentation for further informations """ config = ConfigParser(allow_no_value=True) # allow_no_value to allow adding comments config.optionxform = str # makes options case sensitive config.add_section("Default") config.set("Default", "# Documentation available here: https://github.com/Virtual-World-RE/NeoGF/tree/main/afstool#afs_rebuildconf") config.set("Default", "AFS_MAGIC", f"0x{self.__get_magic().hex()}") config.set("Default", "files_rebuild_strategy", "mixed") config.set("Default", "filename_directory", "True" if self.__filenamedirectory else "False") if self.__filenamedirectory: config.add_section("FilenameDirectory") config.set("FilenameDirectory", "toc_offset_of_fd_offset", f"0x{self.__filenamedirectory_offset_offset:x}") config.set("FilenameDirectory", "fd_offset", f"0x{self.__filenamedirectory_offset:x}") config.set("FilenameDirectory", "fd_last_attribute_type", self.__get_fdlast_type()) config.write((sys_path / "afs_rebuild.conf").open("w")) rebuild_csv = "" # generate and save afs_rebuild.csv for i in range(self.__file_count): filename = self.__get_file_name(i) if self.__filenamedirectory else f"{i:08}" unpacked_filename = resolver.resolve_from_index(i, filename) if self.__filenamedirectory else f"{i:08}" rebuild_csv += f"{unpacked_filename}?0x{i:x}?0x{self.__get_file_offset(i):x}?{filename}\n" if len(rebuild_csv) > 0: (sys_path / "afs_rebuild.csv").write_text(rebuild_csv[:-1]) def unpack(self, afs_path:Path, folder_path:Path): "Method used to unpack an AFS inside a folder" sys_path = folder_path / "sys" root_path = folder_path / "root" sys_path.mkdir(parents=True) root_path.mkdir() resolver = None with afs_path.open("rb") as afs_file: if not self.__loadsys_from_afs(afs_file, afs_path.stat().st_size): logging.info("There is no filename directory. Creating new names and dates for files.") else: logging.debug(f"filenamedirectory_offset:0x{self.__filenamedirectory_offset:x}, filenamedirectory_len:0x{self.__filenamedirectory_len:x}.") logging.info("Writting sys/filenamedirectory.bin") (sys_path / "filenamedirectory.bin").write_bytes(self.__filenamedirectory) resolver = FilenameResolver(sys_path) logging.info("Writting sys/tableofcontent.bin") (sys_path / "tableofcontent.bin").write_bytes(self.__tableofcontent) logging.info(f"Extracting {self.__file_count} files.") for i in range(self.__file_count): file_offset = self.__get_file_offset(i) file_len = self.__get_file_len(i) filename = resolver.resolve_new(i, self.__get_file_name(i)) if self.__filenamedirectory else f"{i:08}" if Path(filename).parent != Path("."): (root_path / Path(filename).parent).mkdir(parents=True, exist_ok=True) logging.debug(f"Writting {root_path / filename} 0x{file_offset:x}:0x{file_offset + file_len:x}") afs_file.seek(file_offset) (root_path / filename).write_bytes(afs_file.read(file_len)) if self.__filenamedirectory: mtime = self.__get_file_mtime(i) os.utime(root_path / filename, (mtime, mtime)) if self.__filenamedirectory: resolver.save() self.__write_rebuild_config(sys_path, resolver) def pack(self, folder_path:Path, afs_path:Path = None): """ Methood used to pack un unpacked folder inside a new AFS file for a file pack will use the next file offset as max file length an raise an exception if the length overflow pack keep FD and TOC inchanged except for file length, FD dates, fd_last_attribute updates """ if afs_path is None: afs_path = folder_path / Path(folder_path.name).with_suffix(".afs") elif afs_path.suffix != ".afs": logging.warning("Dest file should have .afs file extension.") sys_path = folder_path / "sys" root_path = folder_path / "root" self.__loadsys_from_folder(sys_path) resolver = FilenameResolver(sys_path) offsets_map = self.__get_offsets_map() if self.__filenamedirectory: fd_last_attribute_type = self.__get_fdlast_type() if fd_last_attribute_type[:2] == "0x": fd_last_attribute_type = int(fd_last_attribute_type, 16) try: with afs_path.open("wb") as afs_file: # We update files for i in range(self.__file_count): file_offset = self.__get_file_offset(i) file_len = self.__get_file_len(i) filename = resolver.resolve_from_index(i, self.__get_file_name(i) if self.__filenamedirectory else f"{i:08}") file_path = root_path / filename new_file_len = file_path.stat().st_size if new_file_len != file_len: # If no FD, we can raise AFS length without constraint if offsets_map.index(file_offset) + 1 < len(offsets_map): next_offset = offsets_map[offsets_map.index(file_offset)+1] if file_offset + new_file_len > next_offset: raise AfsInvalidFileLenError(f"File {file_path} as a new file_len giving an end offset (0x{file_offset + new_file_len:x}) > next file offset (0x{next_offset:x}). "\ "This means that we have to rebuild the AFS using -r and changing offset of all next files and this could lead to bugs if the main dol use AFS relative file offsets.") self.__patch_file_len(i, new_file_len) if self.__filenamedirectory: self.__patch_fdlasts(i, fd_last_attribute_type) # If there is a filenamedirectory we update mtime: if self.__filenamedirectory: self.__patch_file_mtime(i, round(file_path.stat().st_mtime)) logging.debug(f"Packing {file_path} 0x{file_offset:x}:0x{file_offset+new_file_len:x} in AFS.") afs_file.seek(file_offset) afs_file.write(self.__pad(file_path.read_bytes())) if self.__filenamedirectory: afs_file.seek(self.__filenamedirectory_offset) afs_file.write(self.__pad(self.__filenamedirectory)) logging.debug(f"Packing {sys_path / 'tableofcontent.bin'} at the beginning of the AFS.") afs_file.seek(0) afs_file.write(self.__tableofcontent) except AfsInvalidFileLenError: afs_path.unlink() raise def rebuild(self, folder_path:Path): """ Rebuild will use following config files: * "sys/afs_rebuild.conf" * "sys/afs_rebuild.csv" It will rebuild the unpacked AFS sys files (TOC and FD) in the sys folder """ config = ConfigParser() root_path = folder_path / "root" sys_path = folder_path / "sys" config.read(sys_path / "afs_rebuild.conf") if config["Default"]["AFS_MAGIC"] not in ["0x41465300", "0x41465320"]: raise AfsInvalidMagicNumberError("Error - Invalid [Default] AFS_MAGIC: must be 0x41465300 or 0x41465320.") if config["Default"]["files_rebuild_strategy"] not in ["index", "offset", "mixed", "auto"]: raise AfsInvalidFilesRebuildStrategy("Error - Invalid [Default] files_rebuild_strategy: must be index, offset, mixed or auto.") if config["Default"]["filename_directory"] not in ["True", "False"]: raise AfsFilenameDirectoryValueError("Error - Invalid [Default] filename_directory: must be True or False.") for path in [sys_path / "tableofcontent.bin", sys_path / "filenamedirectory.bin", sys_path / "filename_resolver.csv"]: if path.is_file(): logging.info(f"Removing {path}.") path.unlink() files_paths = [path for path in root_path.glob("**/*") if path.is_file()] self.__file_count = len(files_paths) max_offset = None if config["Default"]["filename_directory"] == "True": if config["FilenameDirectory"]["toc_offset_of_fd_offset"] != "auto": if config["FilenameDirectory"]["toc_offset_of_fd_offset"][:2] != "0x" or len(config["FilenameDirectory"]["toc_offset_of_fd_offset"]) < 3: raise AfsFdOffsetOffsetValueError("Error - Invalid [FilenameDirectory] toc_offset_of_fd_offset: must use hex format 0xabcdef or auto.") self.__filenamedirectory_offset_offset = int(config["FilenameDirectory"]["toc_offset_of_fd_offset"][2:], 16) else: self.__filenamedirectory_offset_offset = self.__file_count*8 + 8 max_offset = int(ceil((self.__filenamedirectory_offset_offset + 8) / Afs.ALIGN)) * Afs.ALIGN # TOC length self.__filenamedirectory_len = self.__file_count * Afs.FILENAMEDIRECTORY_ENTRY_LEN if config["FilenameDirectory"]["fd_offset"] != "auto": if config["FilenameDirectory"]["fd_offset"][:2] != "0x" or len(config["FilenameDirectory"]["fd_offset"]) < 3: raise AfsFdOffsetValueError("Error - Invalid [FilenameDirectory] fd_offset: must use hex format 0xabcdef or auto.") self.__filenamedirectory_offset = int(config["FilenameDirectory"]["fd_offset"][2:], 16) if config["FilenameDirectory"]["fd_last_attribute_type"] not in ["length", "offset-length", "unknown"]: if config["FilenameDirectory"]["fd_last_attribute_type"][0:2] != "0x" or len(config["FilenameDirectory"]["fd_last_attribute_type"]) < 3: raise AfsFdLastAttributeTypeValueError("Error - Invalid [FilenameDirectory] fd_last_attribute_type: must be length, offset-length, 0xabcdef offset or unknown.") else: max_offset = int(ceil((self.__file_count*8 + 8) / Afs.ALIGN)) * Afs.ALIGN # TOC length self.__tableofcontent = bytearray.fromhex( config["Default"]["AFS_MAGIC"][2:] ) + self.__file_count.to_bytes(4, "little") files_rebuild_strategy = config["Default"]["files_rebuild_strategy"] csv_files_lists = [] reserved_indexes = [] empty_blocks_list = [] # We parse the file csv and verify entries retrieving length for files if (sys_path / "afs_rebuild.csv").is_file(): for line in (sys_path / "afs_rebuild.csv").read_text().split('\n'): line_splited = line.split('?') if len(line_splited) == 4: unpacked_filename = line_splited[0] index = None if files_rebuild_strategy in ["index", "mixed"]: if line_splited[1] != "auto": index = line_splited[1] if index[:2] != "0x" or len(index) < 3: raise AfsIndexValueError(f"Error - Invalid entry index in afs_rebuild.csv: {index} - \"{line}\"") index = int(index[2:], 16) if index >= self.__file_count: raise AfsIndexOverflowError(f"Error - Invalid entry index in afs_rebuild.csv: 0x{index:x} - \"{line}\" - index must be < files_count.") if index in reserved_indexes: raise AfsIndexCollisionError("Error - Multiple files using same index: 0x{index:x}") reserved_indexes.append( index ) file_path = root_path / unpacked_filename if not file_path.is_file(): raise AfsInvalidFilePathError(f"Error - File {file_path} doesn't exist.") file_length = file_path.stat().st_size offset = None if files_rebuild_strategy in ["offset", "mixed"]: if line_splited[2] != "auto": offset = line_splited[2] if offset[:2] != "0x" or len(offset) < 3: raise AfsOffsetValueError(f"Error - Invalid entry offset in afs_rebuild.csv: {offset} - \"{line}\"") offset = int(offset[2:], 16) if offset % Afs.ALIGN > 0: raise AfsOffsetAlignError(f"Error - Invalid entry offset in afs_rebuild.csv: 0x{offset:x} - \"{line}\" - offset must be aligned to 0x800.") csv_files_lists.append( [unpacked_filename, index, offset, line_splited[3], file_length] ) files_paths.remove( root_path / unpacked_filename ) elif len(line_splited) == 2: # empty block if line_splited[0][:2] != "0x" or line_splited[1][:2] != "0x" or len(line_splited[0]) < 3 or len(line_splited[1]) < 3: raise AfsEmptyBlockValueError(f"Error - Invalid empty block values: \"{line}\"") offset = int(line_splited[0][2:], 16) length = int(line_splited[1][2:], 16) if offset % Afs.ALIGN > 0 or length % Afs.ALIGN > 0: raise AfsEmptyBlockAlignError(f"Error - Invalid empty block offset or length in afs_rebuild.csv: \"{line}\" - offset and length must be aligned to 0x800.") empty_blocks_list.append([None, None, offset, None, length]) else: raise AfsInvalidFieldsCountError(f"Error - Invalid entry fields count in afs_rebuild.csv: \"{line}\"") # We generate file memory map with offsets: # available_space_ranges is then used to put files that have an adapted length # max_offset is used here to find memory collisions between files and next available space available_space_ranges = [] tmp_ranges = empty_blocks_list if files_rebuild_strategy in ["offset", "mixed"]: tmp_ranges += csv_files_lists # We have to sort offsets before merging to avoid complex algorithm # TOC is already present with max_offset for file_tuple in sorted(tmp_ranges, key=lambda x: (x[2] is not None, x[2])): offset = file_tuple[2] if offset is None: continue if offset < max_offset: raise AfsOffsetCollisionError(f"Error - Offsets collision with offset \"0x{offset:x}\".") elif offset > max_offset: available_space_ranges.append( [max_offset, offset] ) max_offset = int(ceil((offset + file_tuple[4]) / Afs.ALIGN)) * Afs.ALIGN for file_path in files_paths: csv_files_lists.append( [file_path.name, None, None, file_path.name, file_path.stat().st_size] ) # sort by filename csv_files_lists.sort(key=lambda x: x[3]) current_offset = max_offset # if index==None -> Assign an index not in reserved_indexes reserved_indexes.sort() next_index = 0 for i in range(len(csv_files_lists)): if csv_files_lists[i][1] is None and files_rebuild_strategy in ["index", "mixed"] or files_rebuild_strategy in ["auto", "offset"]: for j in range(next_index, len(csv_files_lists)): if j not in reserved_indexes: next_index = j + 1 csv_files_lists[i][1] = j break # sort by index csv_files_lists.sort(key=lambda x: x[1]) # if offset==None -> Assign an offset in available_space_ranges or at the end of file allocated space for i in range(len(csv_files_lists)): if files_rebuild_strategy in ["offset", "mixed"] and csv_files_lists[i][2] is None or files_rebuild_strategy in ["auto", "index"]: block_len = int(ceil(csv_files_lists[i][4] / Afs.ALIGN)) * Afs.ALIGN for j in range(len(available_space_ranges)): available_block_len = int(ceil((available_space_ranges[j][1] - available_space_ranges[j][0]) / Afs.ALIGN)) * Afs.ALIGN if block_len <= available_block_len: csv_files_lists[i][2] = available_space_ranges[j][0] if block_len == available_block_len: del available_space_ranges[j] else: available_space_ranges[j][0] += block_len break else: # Here we have a bigger file than available ranges so we pick current_offset at the end of allocated space csv_files_lists[i][2] = current_offset current_offset += block_len if self.__filenamedirectory_offset_offset: self.__filenamedirectory = b"" fd_last_attribute_type = config["FilenameDirectory"]["fd_last_attribute_type"] if fd_last_attribute_type[:2] == "0x": fd_last_attribute_type = int(fd_last_attribute_type[2:], 16) # Have to be sorted by index # current_offset contains now fd offset if not already set resolver = FilenameResolver(sys_path) for i in range(len(csv_files_lists)): self.__tableofcontent += csv_files_lists[i][2].to_bytes(4, "little") + csv_files_lists[i][4].to_bytes(4, "little") # unpacked_filename, index, offset, filename, file_length if self.__filenamedirectory_offset_offset: mtime = b"\x00" * 12 # will be patched next pack fd_last_attribute = None if type(fd_last_attribute_type) == int: fd_last_attribute = fd_last_attribute_type.to_bytes(4, "little") elif fd_last_attribute_type == "length": fd_last_attribute = csv_files_lists[i][4].to_bytes(4, "little") elif fd_last_attribute_type == "offset-length": fd_last_attribute = self.__tableofcontent[8+i*4:8+i*4+4] else: # == unknown fd_last_attribute = b"\x00"*4 self.__filenamedirectory += bytes(csv_files_lists[i][3], "utf-8").ljust(32, b"\x00") + mtime + fd_last_attribute # if unpacked_filename != filename we store it into the resolver if csv_files_lists[i][0] != csv_files_lists[i][3] or not self.__filenamedirectory_offset_offset: resolver.add(i, csv_files_lists[i][0]) resolver.save() if self.__filenamedirectory: if not self.__filenamedirectory_offset: self.__filenamedirectory_offset = current_offset elif self.__filenamedirectory_offset < current_offset: raise AfsFdOffsetCollisionError(f"Error - Invalid FD offset 0x{self.__filenamedirectory_offset:x} < last used file block end 0x{current_offset:x}.") self.__tableofcontent = self.__tableofcontent.ljust(self.__filenamedirectory_offset_offset+8, b"\x00") # Add pad if needed self.__tableofcontent[self.__filenamedirectory_offset_offset:self.__filenamedirectory_offset_offset+8] = self.__filenamedirectory_offset.to_bytes(4, "little") + self.__filenamedirectory_len.to_bytes(4, "little") logging.info(f"Writting {Path('sys/filenamedirectory.bin')}") (sys_path / "filenamedirectory.bin").write_bytes(self.__filenamedirectory) logging.info(f"Writting {Path('sys/tableofcontent.bin')}") (sys_path / "tableofcontent.bin").write_bytes(self.__tableofcontent) def stats(self, path:Path): """ Stats will print the AFS stats: Get full informations about header, TOC, FD, full memory mapping sorted by offsets (files and sys files), addresses space informations, and duplicated filenames grouped by filenames. """ if path.is_file(): with path.open("rb") as afs_file: self.__loadsys_from_afs(afs_file, path.stat().st_size) else: self.__loadsys_from_folder(path / "sys") files_map = self.__get_formated_map() files_map.sort(key=lambda x: x[1]) # sort by offset (str with fixed len=8) # Offsets intersect dup_offsets_tuples = [] last_tuple = (-1, "-1", "0") # empty space search init new_set = True # Filenames duplicates dup_names_dict = {} # tmp dict for grouping by filename dup_names_tuples = [] # For empty blocks empty_space_tuples = [] for file_tuple in files_map: # Filenames duplicates if not file_tuple[6] in dup_names_dict: dup_names_dict[file_tuple[6]] = [file_tuple] else: dup_names_dict[file_tuple[6]].append(file_tuple) # Offsets intersect if file_tuple[1] < last_tuple[1]: if new_set: dup_offsets_tuples.append("Files sharing same offsets:\n") new_set = False dup_offsets_tuples.append(file_tuple) else: new_set = True # Empty blocks last_block_end = ceil(int(last_tuple[2], base=16) / Afs.ALIGN) * Afs.ALIGN if int(file_tuple[1], base=16) - last_block_end >= Afs.ALIGN: empty_space_tuples.append( (last_tuple[2], file_tuple[1], f"{int(file_tuple[1], base=16) - int(last_tuple[2], base=16):08x}", file_tuple[6]) ) last_tuple = file_tuple for filename in dup_names_dict: if len(dup_names_dict[filename]) > 1: dup_names_tuples += ["Files sharing same name:\n"] + [file_tuple for file_tuple in dup_names_dict[filename]] dup_offsets = "Yes" if len(dup_offsets_tuples) > 1 else "No" dup_names = "Yes" if len(dup_names_tuples) > 1 else "No" empty_space = "Yes" if len(empty_space_tuples) > 1 else "No" files_info = f"AFS Magic/Version : {str(self.__get_magic())[2:-1]}\n" files_info += f"TOC offset of the FD offset : 0x{self.__filenamedirectory_offset_offset:x}\n" if self.__filenamedirectory else "" files_info += f"Multiple files using same offsets: {dup_offsets}\n" files_info += f"Multiple files using same name : {dup_names}\n" if self.__filenamedirectory else "" files_info += f"Empty blocks : {empty_space}\n" self.__print("Global infos and AFS space mapping:", files_map, infos=files_info) if dup_offsets_tuples: self.__print("Files sharing same AFS offsets:", dup_offsets_tuples) if dup_names_tuples: self.__print("Files using same filenames:", dup_names_tuples) if empty_space_tuples: self.__print("Empty blocks between files (filename = name of the previous file):", empty_space_tuples, columns=[1,2,3,6]) def get_argparser(): import argparse parser = argparse.ArgumentParser(description='AFS packer & unpacker - [GameCube] v' + __version__) parser.add_argument('--version', action='version', version='%(prog)s ' + __version__) parser.add_argument('-v', '--verbose', action='store_true', help='verbose mode') parser.add_argument('input_path', metavar='INPUT', help='') parser.add_argument('output_path', metavar='OUTPUT', help='', nargs='?', default="") group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-p', '--pack', action='store_true', help="-p source_folder (dest_file.afs): Pack source_folder in new file source_folder.afs or dest_file.afs if specified.") group.add_argument('-u', '--unpack', action='store_true', help="-u source_afs.afs (dest_folder): Unpack the AFS in new folder source_afs or dest_folder if specified.") group.add_argument('-s', '--stats', action='store_true', help="-s source_afs.afs or source_folder: Get stats about AFS, files, memory, lengths and offsets.") group.add_argument('-r', '--rebuild', action='store_true', help="-r source_folder: Rebuild AFS tableofcontent (TOC) and filenamedirectory (FD) using afs_rebuild.conf file and afs_rebuild.csv.") return parser if __name__ == '__main__': logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) args = get_argparser().parse_args() p_input = Path(args.input_path) p_output = Path(args.output_path) afs = Afs() if args.verbose: logging.getLogger().setLevel(logging.DEBUG) if args.pack: logging.info("### Pack in new AFS") if(p_output == Path(".")): p_output = Path(p_input.with_suffix(".afs")) logging.info(f"packing folder {p_input} in {p_output}") afs.pack( p_input, p_output ) elif args.unpack: logging.info("### Unpack AFS in new folder") if p_output == Path("."): p_output = p_input.parent / p_input.stem logging.info(f"unpacking AFS {p_input} in {p_output}") afs.unpack( p_input, p_output ) elif args.stats: afs.stats(p_input) elif args.rebuild: if not (p_input / "sys").is_dir(): raise AfsInvalidAfsFolderError(f"Error - Invalid unpacked AFS: {p_input}.") logging.info(f"rebuilding {p_input}") afs.rebuild(p_input)