From c592fbbb188b0810316e8c3de9673d15d1f39d18 Mon Sep 17 00:00:00 2001 From: tmpz23 <28760271+tmpz23@users.noreply.github.com> Date: Wed, 16 Feb 2022 23:04:41 +0100 Subject: [PATCH] Create mdltool.py Global concept about models parsing. Has to be tested... --- mdltool.py | 450 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 450 insertions(+) create mode 100644 mdltool.py diff --git a/mdltool.py b/mdltool.py new file mode 100644 index 0000000..97b50ec --- /dev/null +++ b/mdltool.py @@ -0,0 +1,450 @@ +#!/usr/bin/env python3 +from pathlib import Path + +__version__ = "0.0.0" +__author__ = "rigodron, algoflash, GGLinnk" +__license__ = "MIT" +__status__ = "developpement" + + +class HSDModelEmptyFileError(Exception) : pass +class HSDModelInvalidModelHeaderError(Exception) : pass + + +class HSDNode: + __data_block_offset = None + def __init__(self, offset:int): + self.__data_block_offset = offset + + +class Rotation: + __x = None + __y = None + __z = None + def __init__(self, x:float, y:float, z:float): + self.__x = x + self.__y = y + self.__z = z + + +class Scale: + __x = None + __y = None + __z = None + def __init__(self, x:float, y:float, z:float): + self.__x = x + self.__y = y + self.__z = z + + +class Translation: + __x = None + __y = None + __z = None + def __init__(self, x:float, y:float, z:float): + self.__x = x + self.__y = y + self.__z = z + + +class MObj: + SERIALIZED_LEN = 0x18 + __unknown = None + __flags = None + __tobj_offset = None + __material_offset = None + __unknown_2 = None + def __init__(self, datas:bytes): + if len(datas) != self.SERIALIZED_LEN: + raise Exception(f"Invalid data length in MObj constructor: {len(datas)}") + self.__unknown = int.from_bytes(datas[:4], "big") + self.__flags = int.from_bytes(datas[4:8], "big") + self.__tobj_offset = int.from_bytes(datas[8:12], "big") + self.__material_offset = int.from_bytes(datas[12:16], "big") + self.__unknown_2 = int.from_bytes(datas[16:24], "big") + def __str__(self, depth:int): + return f"{' '*depth + '|-'}MObj:{self.__unknown:08x}:{self.__flags:08x}:{self.__tobj_offset:08x}:{self.__material_offset:08x}:{self.__unknown_2:016x}\n" + + +class PObj: + SERIALIZED_LEN = 0x18 + __unknown = None + __next_offset = None + __vertex_attr_list_offset = None + __flags = None + __display_list_length = None # * 0x20 for getting length + __display_list_offset = None + __weight_list_offset = None + # datas & objects + __next_pobj = None + __display_list = None + def __init__(self, datas:bytes): + if len(datas) != self.SERIALIZED_LEN: + raise Exception(f"Invalid data length in PObj constructor: {len(datas)}") + self.__unknown = int.from_bytes(datas[:4], "big") + self.__next_offset = int.from_bytes(datas[4:8], "big") + self.__vertex_attr_list_offset = int.from_bytes(datas[8:12], "big") + self.__flags = int.from_bytes(datas[12:14], "big") + self.__display_list_length = int.from_bytes(datas[14:16], "big") + self.__display_list_offset = int.from_bytes(datas[16:20], "big") + self.__weight_list_offset = int.from_bytes(datas[20:24], "big") + def __str__(self, depth:int): + buffer_str = f"{' '*depth + '|-'}PObj:{self.__unknown:08x}:{self.__next_offset:08x}:{self.__vertex_attr_list_offset:08x}:{self.__flags:04x}:{self.__display_list_length:04x}:{self.__display_list_offset:08x}:{self.__weight_list_offset:08x}\n" + if self.has_next(): + buffer_str += self.__next_pobj.__str__(depth) + return buffer_str + def has_next(self): return self.__next_offset != 0 + def next_offset(self): return self.__next_offset + def display_list_length(self): return self.__display_list_length + def display_list_offset(self): return self.__display_list_offset + def set_next_pobj(self, pobj): self.__next_pobj = pobj + def set_display_list(self, display_list:list): self.__display_list = display_list + + +class DObj: + SERIALIZED_LEN = 0x10 + __unknown = None + __next_offset = None + __mobj_offset = None + __pobj_offset = None + # Objects + __next_dobj = None + __mobj = None + __pobj = None + def __init__(self, datas:bytes): + if len(datas) != self.SERIALIZED_LEN: + raise Exception(f"Invalid data length in DObj constructor: {len(datas)}") + self.__unknown = int.from_bytes(datas[:4], "big") + self.__next_offset = int.from_bytes(datas[4:8], "big") + self.__mobj_offset = int.from_bytes(datas[8:12], "big") + self.__pobj_offset = int.from_bytes(datas[12:16], "big") + def __str__(self, depth:int): + buffer_str = f"{' '*depth + '|-'}DObj:{self.__unknown:08x}:{self.__next_offset:08x}:{self.__mobj_offset:08x}:{self.__pobj_offset:08x}\n" + if self.has_mobj(): + buffer_str += self.__mobj.__str__(depth + 1) + if self.has_pobj(): + buffer_str += self.__pobj.__str__(depth + 1) + if self.has_next_dobj(): + buffer_str += self.__next_dobj.__str__(depth) + return buffer_str + def has_next_dobj(self): return self.__next_offset != 0 + def has_mobj(self): return self.__mobj_offset != 0 + def has_pobj(self): return self.__pobj_offset != 0 + def next_offset(self): return self.__next_offset + def mobj_offset(self): return self.__mobj_offset + def pobj_offset(self): return self.__pobj_offset + def set_next_dobj(self, dobj): self.__next_dobj = dobj + def set_mobj(self, mobj:MObj): self.__mobj = mobj + def set_pobj(self, pobj:PObj): self.__pobj = pobj + + +class JObj:#(HSDNode): + SERIALIZED_LEN = 0x40 + __unknown = None + __flags = None + __child_offset = None + __next_offset = None + __dobj_offset = None + __rotation = None + __scale = None + __translation = None + # Objects + __child_jobj = None + __next_jobj = None + __dobj = None + def __init__(self, datas:bytes): + if len(datas) != JObj.SERIALIZED_LEN: + raise Exception(f"Invalid data length in JObj constructor: {len(datas)}") + self.__unknown = int.from_bytes(datas[:4], "big") + self.__flags = int.from_bytes(datas[4:8], "big") + self.__child_offset = int.from_bytes(datas[8:12], "big") + self.__next_offset = int.from_bytes(datas[12:16], "big") + self.__dobj_offset = int.from_bytes(datas[16:20], "big") + self.__rotation = Rotation( + int.from_bytes(datas[20:24], "big"), + int.from_bytes(datas[24:28], "big"), + int.from_bytes(datas[28:32], "big")) + self.__scale = Scale( + int.from_bytes(datas[32:36], "big"), + int.from_bytes(datas[40:44], "big"), + int.from_bytes(datas[44:48], "big")) + self.__translation = Translation( + int.from_bytes(datas[48:52], "big"), + int.from_bytes(datas[52:56], "big"), + int.from_bytes(datas[56:60], "big")) + def __str__(self, depth:int): + buffer_str = f"{' '*depth + '|-'}JObj:{self.__unknown:08x}:{self.__flags:08x}:{self.__child_offset:08x}:{self.__next_offset:08x}:{self.__dobj_offset:08x}\n" + if self.has_child(): + buffer_str += self.__child_jobj.__str__(depth + 1) + if self.has_next(): + buffer_str += self.__next_jobj.__str__(depth) + if self.has_dobj(): + buffer_str += self.__dobj.__str__(depth + 1) + return buffer_str + def has_child(self): return self.__child_offset != 0 + def has_next(self): return self.__next_offset != 0 + def has_dobj(self): return self.__dobj_offset != 0 + def child_offset(self): return self.__child_offset + def next_offset(self): return self.__next_offset + def dobj_offset(self): return self.__dobj_offset + def set_child_jobj(self, jobj): self.__child_jobj = jobj + def set_next_jobj(self, jobj): self.__next_jobj = jobj + def set_dobj(self, dobj): self.__dobj = dobj + + +class JObjDesc: + SERIALIZED_LEN = 0x10 + __jobj_offset = None + __joint_animations_offset = None + __material_animations_offset = None + __shape_animations_offset = None + # Objects + __jobj = None + def __init__(self, datas:bytes): + self.__jobj_offset = int.from_bytes(datas[:4], "big") + self.__joint_animations_offset = int.from_bytes(datas[4:8], "big") + self.__material_animations_offset = int.from_bytes(datas[8:12], "big") + self.__shape_animations_offset = int.from_bytes(datas[12:16], "big") + def __str__(self, depth:int): + return f"{' '*depth + '|-'}JObjDesc:{self.__jobj_offset:08x}:{self.__joint_animations_offset:08x}:{self.__material_animations_offset:08x}:{self.__shape_animations_offset:08x}\n{self.__jobj.__str__(depth + 1)}" + def jobj_offset(self): return self.__jobj_offset + def set_jobj(self, jobj:JObj): self.__jobj = jobj + + +# RootNode contains SObj offset when parsing scene_data +class SObj: + SERIALIZED_LEN = 0x10 + __jobj_descs_list_offset = None + __cameras_list_offset = None + __lights_list_offset = None + __fog = None + # Objects + __jobj_descs_list = [] + __cameras_list = [] + __lights_list = [] + def __init__(self, datas:bytes): + if len(datas) != SObj.SERIALIZED_LEN: + raise Exception("Invalid SObj length.") + self.__jobj_descs_list_offset = int.from_bytes(datas[:4], "big") + self.__cameras_list_offset = int.from_bytes(datas[4:8], "big") + self.__lights_list_offset = int.from_bytes(datas[8:12], "big") + self.__fog = int.from_bytes(datas[12:16], "big") + + """if self.__fog != 0: + print(self.__fog) + raise Exception("Fog prensent in SObj.") + """ + def __str__(self, depth:int): + buffer_str = f"{' '*depth + '|-'}SObj:{self.__jobj_descs_list_offset:08x}:{self.__cameras_list_offset:08x}:{self.__lights_list_offset:08x}:{self.__fog:08x}\n" + for jobj_desc in self.__jobj_descs_list: + buffer_str += jobj_desc.__str__(depth + 1) + for camera in self.__cameras_list: + buffer_str += camera.__str__(depth + 1) + for ligth in self.__lights_list: + buffer_str += ligth.__str__(depth + 1) + return buffer_str + def jobj_descs_list_offset(self): return self.__jobj_descs_list_offset + def cameras_list_offset(self): return self.__cameras_list_offset + def lights_list_offset(self): return self.__lights_list_offset + def set_jobj_descs_list(self, jobj_descs_list:list): self.__jobj_descs_list = jobj_descs_list + def set_cameras_list(self, cameras_list:list): self.__cameras_list = cameras_list + def set_lights_list(self, lights_list:list): self.__lights_list = lights_list + + +class RootNode: + SERIALIZED_LEN = 8 + __sobj_offset = None + __name_offset = None + # Objects + __name = None + __sobj = None + def __init__(self, datas:bytes):# offset:int, string_offset:int, name:str, sobj:SObj): + self.__sobj_offset = int.from_bytes(datas[:4], "big") + self.__name_offset = int.from_bytes(datas[4:8], "big") + def __str__(self): + return f"RootNode:{self.__name}:{self.__sobj_offset:08x}:{self.__name_offset:08x}\n{self.__sobj.__str__(1)}" + def name_offset(self): return self.__name_offset + def sobj_offset(self): return self.__sobj_offset + def set_name(self, name:str ): self.__name = name + def set_sobj(self, sobj:SObj): self.__sobj = sobj + + +class HSDTree: + __data_block = None + __relocation_table = None # not used yet + __root0_node = None + __string_table = None + def __init__(self, data_block:bytes, relocation_table:list, root0_node_data:bytes, string_table:bytes): + self.__data_block = data_block + self.__relocation_table = relocation_table + self.__string_table = string_table + + self.__root0_node = RootNode(root0_node_data) + self.__root0_node.set_name(self.__string_table[self.__root0_node.name_offset():].split(b"\x00")[0].decode("utf-8")) + self.__root0_node.set_sobj(self.__add_sobj(self.__root0_node.sobj_offset())) + def __str__(self): + return str(self.__root0_node)[:-1] + def __add_sobj(self, offset:int): + sobj = SObj(self.__data_block[offset:offset+SObj.SERIALIZED_LEN]) + + jobj_descs_list = [] + list_offset = sobj.jobj_descs_list_offset() + jobj_desc_offset = int.from_bytes(self.__data_block[list_offset:list_offset+4], "big") + + while jobj_desc_offset != 0: + jobj_descs_list.append( self.__add_jobj_desc(jobj_desc_offset) ) + list_offset += 4 + jobj_desc_offset = int.from_bytes(self.__data_block[list_offset:list_offset+4], "big") + + sobj.set_jobj_descs_list(jobj_descs_list) + return sobj + def __add_jobj_desc(self, offset:int): + jobj_desc = JObjDesc(self.__data_block[offset: offset+JObjDesc.SERIALIZED_LEN]) + jobj_desc.set_jobj(self.__add_jobj(jobj_desc.jobj_offset())) + return jobj_desc + def __add_jobj(self, offset:int): + new_jobj = JObj(self.__data_block[offset:offset+JObj.SERIALIZED_LEN]) + if new_jobj.has_child(): + new_jobj.set_child_jobj( self.__add_jobj(new_jobj.child_offset()) ) + if new_jobj.has_next(): + new_jobj.set_next_jobj( self.__add_jobj(new_jobj.next_offset()) ) + if new_jobj.has_dobj(): + new_jobj.set_dobj( self.__add_dobj(new_jobj.dobj_offset()) ) + return new_jobj + def __add_dobj(self, offset:int): + new_dobj = DObj(self.__data_block[offset: offset+DObj.SERIALIZED_LEN]) + if new_dobj.has_next_dobj(): + new_dobj.set_next_dobj( self.__add_dobj(new_dobj.next_offset()) ) + if new_dobj.has_mobj(): + new_dobj.set_mobj( self.__add_mobj(new_dobj.mobj_offset()) ) + if new_dobj.has_pobj(): + new_dobj.set_pobj( self.__add_pobj(new_dobj.pobj_offset()) ) + return new_dobj + def __add_mobj(self, offset:int): + return MObj(self.__data_block[offset: offset+MObj.SERIALIZED_LEN]) + def __add_pobj(self, offset:int): + new_pobj = PObj(self.__data_block[offset: offset+PObj.SERIALIZED_LEN]) + if new_pobj.has_next(): + new_pobj.set_next_pobj( self.__add_pobj(new_pobj.next_offset()) ) + new_pobj.set_display_list( bytearray(self.__data_block[new_pobj.display_list_offset(): new_pobj.display_list_offset() + new_pobj.display_list_length()*0x20]) ) + return new_pobj + + +class HSDModel: + # Header (0x20 octets or 0x120) + __content_length = None # length without firsts 0x100 bytes + __data_block_length = None + __relocations_count = None + __root0_nodes_count = None + __root1_nodes_count = None + # data_block + __data_block = None + # Relocation table + __relocation_table = None + # root_nodes + __root0_nodes_data = None + __root1_nodes_data = None + # string table + __string_table = None + + def __init__(self, mdl_path:Path): + mdl_data = mdl_path.read_bytes() + if not mdl_data: + raise HSDModelEmptyFileError("Empty file.") + + # Header format check + header_len = 0 + if int.from_bytes(mdl_data[:4], "big") == 0x100: + header_len = 0x120 + if int.from_bytes(mdl_data[4:8], "big") != 0x20 or\ + int.from_bytes(mdl_data[8:0xc], "big") != 0xc0 or\ + mdl_data[0xc:0x20] != b"\x00"*0x14 or\ + mdl_data[0xa0:0xc0] != b"\x00"*0x20 or\ + mdl_data[0xe0:0x100] != b"\x00"*0x20: + raise HSDModelInvalidModelHeaderError(f"Invalid header Error {mdl_path}") + elif int.from_bytes(mdl_data[:4], "big") == mdl_path.stat().st_size: + header_len = 0x20 + else: + raise HSDModelInvalidModelHeaderError(f"Invalid header Error {mdl_path}") + + hsd_header_offset = header_len - 0x20 + + if mdl_data[hsd_header_offset+0x14:hsd_header_offset+0x20] != b"\x00"*12: + raise HSDModelInvalidModelHeaderError(f"Invalid header Error {mdl_path}") + + # Header parsing + self.__content_length = int.from_bytes(mdl_data[hsd_header_offset :hsd_header_offset+0x4], "big") + self.__data_block_length = int.from_bytes(mdl_data[hsd_header_offset+ 0x4:hsd_header_offset+0x8], "big") + self.__relocations_count = int.from_bytes(mdl_data[hsd_header_offset+ 0x8:hsd_header_offset+0xc], "big") + self.__root0_nodes_count = int.from_bytes(mdl_data[hsd_header_offset+ 0xc:hsd_header_offset+0x10], "big") + self.__root1_nodes_count = int.from_bytes(mdl_data[hsd_header_offset+0x10:hsd_header_offset+0x14], "big") + + relocation_table_offset = header_len + self.__data_block_length + root0_nodes_offset = relocation_table_offset + 4*self.__relocations_count + root1_nodes_offset = root0_nodes_offset + 8*self.__root0_nodes_count + string_table_offset = root1_nodes_offset + 8*self.__root1_nodes_count + file_length = hsd_header_offset + self.__content_length + + if string_table_offset > file_length: + raise Exception(f"Invalid string_table_offset: {mdl_path}.") + if file_length != mdl_path.stat().st_size: + raise Exception(f"Invalid file_length: {mdl_path}.") + + # Root Nodes raw parsing + if self.__root0_nodes_count != 1 and self.__root1_nodes_count != 0: + raise Exception("Invalid root nodes count.") + self.__root0_nodes_data = mdl_data[root0_nodes_offset:root0_nodes_offset+8*self.__root0_nodes_count] + + self.__data_block = mdl_data[header_len:relocation_table_offset] + self.__relocation_table = [int.from_bytes(mdl_data[i:i+4], "big") for i in range(relocation_table_offset, root0_nodes_offset, 4)] + self.__relocation_table.sort() + self.__string_table = mdl_data[string_table_offset:] + if self.__string_table != b"scene_data\x00": + raise Exception(f"Model not handled {self.__string_table} - expecting scene_data") + + """ logging.debug + """ + print(f"header : 0x00000000->0x{header_len:08x}") + print(f"datas : 0x{header_len:08x}->0x{relocation_table_offset:08x}") + print(f"relocs : 0x{relocation_table_offset:08x}->0x{root0_nodes_offset:08x}") + print(f"root0_nodes : 0x{root0_nodes_offset:08x}->0x{root1_nodes_offset:08x}") + print(f"root1_nodes : 0x{root1_nodes_offset:08x}->0x{string_table_offset:08x}") + print(f"string_table : 0x{string_table_offset:08x}->0x{file_length:08x}") + hsd_tree = HSDTree(self.__data_block, self.__relocation_table, self.__root0_nodes_data, self.__string_table) + print(hsd_tree) + def stats(self): + print("#"*90+"\n# Header infos\n"+"#"*90) + print(f"content_length : {self.__content_length}") + print(f"data_block_length: {self.__data_block_length}") + print(f"relocations_count: {self.__relocations_count}") + print(f"self.__root0_nodes_count : {self.__root0_nodes_count}") + print(f"self.__root1_nodes_count : {self.__root1_nodes_count}") + print("#"*90+"\n# Relocation Table infos\n"+"#"*90) + reloc_str_buffer = "" + for i in range(len(self.__relocation_table)): + reloc_str_buffer += f"{self.__relocation_table[i]:08x} | " + if (i+1) % 8 == 0: + reloc_str_buffer = reloc_str_buffer[:-3] + "\n" + print(reloc_str_buffer) + + +file_path = Path("mdl_arc") +hsd_mdl = HSDModel( file_path ) + +""" + +with Path("invalid_header.txt").open("w") as ih_file: + for file_path in list(Path("afs_data/root").glob("*_mdl*"))+\ + list(Path("pzzu").glob("**/*_mdl*"))+\ + list(Path("arzd").glob("*_mdl*"))+\ + list(Path("afs_data/root").glob("tdc*"))+\ + [Path("afs_data/root/collision.arc")]: + if file_path.suffix == ".arz": continue + try: + hsd_mdl = HSDModel( file_path ) + except HSDModelInvalidModelHeaderError: + ih_file.write(f"{file_path}\n") + except HSDModelEmptyFileError: pass +""" +#print(len(list(Path("afs_data/root").glob("*_mdl*")))) +#hsd_mdl.stats()