diff --git a/doltool/doltool.py b/doltool/doltool.py index c42406f..a243c75 100644 --- a/doltool/doltool.py +++ b/doltool/doltool.py @@ -1,9 +1,11 @@ -from pathlib import Path +import copy +import enum import logging +from pathlib import Path import re -__version__ = "0.0.7" +__version__ = "0.0.8" __author__ = "rigodron, algoflash, GGLinnk" __license__ = "MIT" __status__ = "developpement" @@ -17,161 +19,425 @@ class InvalidImgOffsetError(Exception): pass class InvalidVirtualAddressError(Exception): pass # raised when Virtual address + length Overflow out of sections class SectionsOverflowError(Exception): pass +# raised when Virtual address + length is out of main program space memory +class OutOfMemoryError(Exception): pass +# raised when Virtual address of used section is unaligned to 32 bytes +class InvalidSectionAlignError(Exception): pass +# raised when Section offset does not match current file datas +class InvalidSectionOffsetError(Exception): pass -# Get non-overlapping intervals from interval by removing intervals_to_remove -# intervals_to_remove has to be sorted by left val -# return [[a,b], ...] or None -def remove_intervals_from_interval(interval:list, intervals_to_remove:list): - interval = interval[:] - result_intervals = [] - for interval_to_remove in intervals_to_remove: - if interval_to_remove[2] < interval[1]: continue # end before - if interval_to_remove[1] > interval[2]: break # begin after - - if interval_to_remove[1] <= interval[1]: # begin before - if interval_to_remove[2] >= interval[2]: # total overlap - return None - interval[1] = interval_to_remove[2] # begin truncate - elif interval_to_remove[2] >= interval[2]: # end truncate - interval[2] = interval_to_remove[1] - break - else: # middle truncate - result_intervals.append( ["empty", interval[1], interval_to_remove[1]] ) - interval[1] = interval_to_remove[2] - - return result_intervals + [interval] +def align_bottom(address:int, align:int): + if address % align == 0: return address + return address - address % align + + +def align_top(address:int, align:int): + if address % align == 0: return address + return address + align - (address % align) + + +class SectionType(enum.IntFlag): + DATA = 0 + TEXT = 1 + BSS = 2 + SYS = 3 + UNMAPPED = 4 + + +class IntervalDiv(enum.IntFlag): + LEFT = 0 + IN = 1 + RIGHT = 2 + + +class MemoryObject: + __locked_address_space = None + __type = None + __name = None + __address = None + __end_address = None + __length = None + __datas = None + def __init__(self, address:int, section_type:SectionType = SectionType.UNMAPPED, name:str = None, length:int = None, end_address:int = None, locked_address_space = True): + if length is None: + if end_address is None: + raise Exception("Error - length or end_address has to be specified.") + self.__end_address = end_address + self.__length = end_address - address + else: + self.__length = length + self.__end_address = address + length + + if section_type == section_type.SYS or not locked_address_space: + self.__locked_address_space = False + else: + self.__locked_address_space = True + if not 0x80003100 <= address < 0x81200000 or not 0x80003100 < self.__end_address <= 0x81200000: + raise OutOfMemoryError(f"Error - Out of memory address: {address:08x}:{self.__end_address:08x}: should be in 0x80003100:0x81200000.") + + self.__type = section_type + self.__name = name + self.__address = address + def __str__(self): + return f"| {str(self.name()).ljust(11)} | {self.address():08x} | {self.end_address():08x} | {self.length():08x} |" + def __sub__(interval:'MemoryObject', intervals_to_remove:list): + """ + Get non-overlapping intervals from interval by removing intervals_to_remove + input: interval = MemoryObject + input: intervals_to_remove = [ MemoryObject, ... ] + return [MemoryObject, ...] or None + * sorted by address + """ + interval = copy.deepcopy(interval) + intervals_to_remove.sort(key=lambda x: x.address()) + result_memory_objects = [] + for interval_to_remove in intervals_to_remove: + if interval_to_remove < interval: continue # end before + if interval_to_remove > interval: break # begin after + if interval in interval_to_remove: return result_memory_objects if result_memory_objects != [] else None # total overlap + + # begin truncate + if interval_to_remove.address() <= interval.address(): + interval.set_address(interval_to_remove.end_address()) + continue + result_memory_objects.append(MemoryObject(interval.address(), interval.type(), interval.name(), end_address=interval_to_remove.address())) + + # end truncate + if interval_to_remove.end_address() >= interval.end_address(): + return result_memory_objects + # interval.address() < interval_to_remove < interval.end_address() + interval.set_address( interval_to_remove.end_address() ) + continue + if interval.length() > 0: + result_memory_objects.append(interval) + return result_memory_objects if result_memory_objects != [] else None + def __lt__(a, b): return a.end_address() <= b.address() + def __le__(a, b): return b.address() < a.end_address() <= b.end_address() and a.address() < b.address() + def __ge__(a, b): return b.address() <= a.address() < b.end_address() and a.end_address() > b.end_address() + def __gt__(a, b): return a.address() >= b.end_address() + def __contains__(a, b): return b.address() >= a.address() and b.end_address() <= a.end_address() + def __and__(a, b): return a.address() < b.end_address() and a.end_address() > b.address() # Intersect + def __truediv__(a, b): + """ + Description: Split a using b by creating before_b, in_b, after_b intervals + input: a = MemoryObject or inherited class + input: b = MemoryObject or inherited class + return: {IntervalDiv: splited_copy, ... } or None + """ + if not a & b: return None + result = {} + + if a.address() < b.address(): + new_left = copy.deepcopy(a) + + new_left.set_end_address(b.address()) + new_left.set_datas( new_left.datas()[:new_left.length()] ) + + a.set_address(b.address()) + a.set_datas( a.datas()[-a.length():] ) + + result[IntervalDiv.LEFT] = new_left + + if a.end_address() > b.end_address(): + new_right = copy.deepcopy(a) + new_right.set_address(b.end_address()) + new_right.set_datas( new_right.datas()[-new_right.length():] ) + + a.set_end_address(b.end_address()) + a.set_datas( a.datas()[:a.length()] ) + + result[IntervalDiv.RIGHT] = new_right + + result[IntervalDiv.IN] = a + return result if len(result) > 0 else None + #__eq__(a, b) + def type(self): return self.__type + def name(self): return self.__name + def address(self): return self.__address + def end_address(self): return self.__end_address + def length(self): return self.__length + def datas(self): return self.__datas + def set_name(self, name:str): self.__name = name + def set_address(self, address:int): + if self.__locked_address_space and not 0x80003100 <= address < 0x81200000: + raise OutOfMemoryError(f"Error - Out of memory address: {address:08x} should be 0x80003100 <= address < 0x81200000.") + self.__address = address + self.__length = self.__end_address - address + def set_end_address(self, address:int): + if self.__locked_address_space and not 0x80003100 < address <= 0x81200000: + raise OutOfMemoryError(f"Error - Out of memory end_address: {address:08x} should be 0x80003100 < end_address <= 0x81200000.") + self.__end_address = address + self.__length = address - self.__address + def set_datas(self, datas:bytes): + self.__datas = datas + def set_type(self, section_type:SectionType): + self.__type = section_type + def update_datas(self, memory_object:'MemoryObject'): + if not memory_object in self: + raise Exception("Error - Invalid update adresses.") + if len(memory_object.datas()) != memory_object.length(): + raise Exception("Error - length does not match the datas length.") + self.__datas = bytearray(self.__datas) + offset = memory_object.address() - self.address() + self.__datas[offset: offset + memory_object.length()] = memory_object.datas() + def to_memory_object(self): return MemoryObject(self.address(), self.type(), self.name(), length=self.length()) + def align(self): + self.set_address( align_bottom(self.address(), 32) ) + self.set_end_address( align_top(self.end_address(), 32) ) + + +class Section(MemoryObject): + __index = None + __offset = None + __is_used = None + def __init__(self, index:int, offset:int, address:int, length:int, section_type:SectionType = None): + if section_type is None: + section_type = SectionType.TEXT if index < 7 else SectionType.DATA + super().__init__(address, section_type, length=length, locked_address_space=False) + self.__index = index + self.__offset = offset + if self.is_used(): + # Section virtual address has to be aligned to 32 bytes. + if self.address() % 32 != 0: + raise InvalidSectionAlignError(f"Error - Section {index} is not aligned to 32 bytes.") + def index(self): return self.__index + def offset(self): return self.__offset + def set_index(self, index:int): self.__index = index + def set_offset(self, offset:int): self.__offset = offset + def is_used(self): + return (self.__offset != 0) and (self.address() != 0) and (self.length() != 0) + def format_raw(self): + section_raw_name = f"text{self.index()}".ljust(7) if self.type() == SectionType.TEXT else f"data{self.index()}".ljust(7) + return f"| {section_raw_name} | {self.offset():08x} | {self.address():08x} | {self.length():08x} | {str(self.is_used()).ljust(5)} |\n" + def resolve_img2virtual(self, offset:int): + if offset >= self.offset() and offset < self.offset() + self.length(): + return self.address() + offset - self.offset() + return None + def resolve_virtual2img(self, address:int): + if address >= self.address() and address < self.end_address(): + return self.offset() + address - self.address() + return None + + +class Bss(MemoryObject): + # list of memory objects out of sections + __splited = None + def __init__(self, address:int, length:int): + super().__init__(address, SectionType.BSS, "bss", length=length) + def format(self): + return f"bss: address:{self.address():08x} length:{self.length():08x}" + def split(self, memory_objects:list): + self.__splited = self - memory_objects + if self.__splited is not None: # If .bss is mapped + for i, splited in enumerate(self.__splited): + splited.set_name(f".bss{i}") + return self.__splited + def splited(self): return self.__splited + + +def get_unmapped_intervals(merged_intervals:list, memory_objects:list): + """ + Description: This function is usefull to find new sections to create for an .ini file processing + input: merged_intervals = [MemoryObject, ...] + * non overlapping, with length > 0 (There is always sections in dols) + input: memory_objects = [ActionReplayCode, ...] + * could overlap + return [MemoryObject, ...] else None + * unmapped sections intervals where we found ARCodes sorted by address + * it means that this intervals are used but are not in already existing intervals (merged_intervals) + """ + memory_objects.sort(key=lambda x:x.address()) + unoverlapped_list = [] + for memory_object in memory_objects: + unoverlapped = memory_object - merged_intervals + if unoverlapped is not None: + unoverlapped_list += unoverlapped + + if len(unoverlapped_list) == 0: + return None + + merged_intervals = copy.deepcopy(merged_intervals) + unoverlapped_list.sort(key=lambda x:x.address()) + def _get_unmapped_intervals(merged_intervals:list, unoverlapped_list:list): + """ + input: merged_intervals: [MemoryObject, ...] + * contains intervals separated by empty interval + input: unoverlapped_list: [MemoryObject, ...] + * contains intervals < merged_intervals or intervals > merged_intervals + return [MemoryObject, ...] + * each of the returned memory objects describe an unmapped interval used by unoverlapped_list + """ + if len(merged_intervals) == 0: + return [MemoryObject(unoverlapped_list[0].address(), end_address=unoverlapped_list[-1].end_address())] + merged_interval = merged_intervals.pop(0) + new_unmapped = [] + for i, memory_object in enumerate(unoverlapped_list): + if memory_object < merged_interval: + if new_unmapped == []: + new_unmapped = [memory_object] + continue + else: + new_unmapped[0].set_end_address(memory_object.end_address()) + continue + else: + if len(unoverlapped_list[i:]) == 0: return new_unmapped + return new_unmapped + _get_unmapped_intervals(merged_intervals, unoverlapped_list[i:]) + return new_unmapped + return _get_unmapped_intervals(merged_intervals, unoverlapped_list) + + +def get_overlapping_arcodes(action_replay_list:list): + """ + input: action_replay_list = [ActionReplayCode, ...] + return [(ActionReplayCode, ActionReplayCode), ...] else None + Get overlapping action replay code in memory. Return couples of arcodes that patch sames memory addresses. + """ + if len(action_replay_list) < 2: return None + action_replay_list.sort(key=lambda x:x.address()) + + # Find overlaps between ARCodes + overlaps_list = [] + last_arcode = action_replay_list[0] + for action_replay_code in action_replay_list[1:]: + # Intersect + if last_arcode & action_replay_code: + overlaps_list.append( (last_arcode, action_replay_code) ) + last_arcode = action_replay_code + return overlaps_list if overlaps_list != [] else None -# Parse an ini file and return a list of [ [virtual_address:int, value:bytes], ... ] -# All ARCodes present in the ini will be enabled without taking care of [ActionReplay_Enabled] section -# raise an Exception if lines are in invalid format: -# * empty lines are removed -# * lines beginning with $ are concidered as comments and are removed -# * lines beginning with [ are concidered as comments and are removed -# * others lines have to be in format: "0AXXXXXX XXXXXXXX" with A in [2,3,4,5] and X in [0-9a-fA-F] def parse_action_replay_ini(path:Path): - action_replay_lines = path.read_text().splitlines() + """ + input: path of ini + return [ActionReplayCode, ...] + Parse an ini file. All ARCodes present in the ini will be enabled without taking care of [ActionReplay_Enabled] section. + * empty lines are removed + * lines beginning with $ are concidered as comments and are removed + * lines beginning with [ are concidered as comments and are removed + * others lines have to be in format: "0AXXXXXX XXXXXXXX" with A in [2,3,4,5] and X in [0-9a-fA-F] + """ + return [ActionReplayCode(action_replay_line, i) for i, action_replay_line in enumerate(path.read_text().splitlines()) if len(action_replay_line) != 0 and action_replay_line[0] not in ["$", "["]] - # address = (first 4 bytes & 0x01FFFFFF) | 0x80000000 - # opcode = first byte & 0xFE - pattern = re.compile("^(0[2345][0-9a-zA-Z]{6}) ([0-9a-zA-Z]{8})$") - result_list = [] - for action_replay_line in action_replay_lines: - if len(action_replay_line) == 0: - continue - if action_replay_line[0] in ["$", "["]: - continue - res = pattern.fullmatch(action_replay_line) +class ActionReplayCode(MemoryObject): + __PATTERN = re.compile("^(0[2345][0-9a-zA-Z]{6}) ([0-9a-zA-Z]{8})$") # class variable give better perfs for regex processing + __line_number = None + __opcode = None + def __init__(self, action_replay_code:str, line_number:int): + self.__line_number = line_number + res = ActionReplayCode.__PATTERN.fullmatch(action_replay_code) if res is None: - raise InvalidIniFileEntryError(f"Error - Arcode has to be in format: '0AXXXXXX XXXXXXXX' with A in [2,3,4,5] and X in [0-9a-fA-F] line \"{action_replay_line}\".") + raise InvalidIniFileEntryError(f"Error - Arcode has to be in format: '0AXXXXXX XXXXXXXX' with A in [2,3,4,5] and X in [0-9a-fA-F] line {line_number} \"{action_replay_code}\".") - virtual_address = (int(res[1], base=16) & 0x01FFFFFF) | 0x80000000 - opcode = int(res[1][:2], base=16) & 0xFE - bytes_value = None + # address = (first 4 bytes & 0x01FFFFFF) | 0x80000000 + address = (int(res[1], base=16) & 0x01FFFFFF) | 0x80000000 - if opcode == 0x04: - bytes_value = int(res[2], 16).to_bytes(4, "big") - elif opcode == 0x02: - bytes_value = (int(res[2][:4], 16) + 1) * int(res[2][4:], 16).to_bytes(2, "big") - else: - raise InvalidIniFileEntryError("Error - Arcode has to be in format: '0AXXXXXX XXXXXXXX' with A in [2,3,4,5] and X in [0-9a-fA-F] line \"{action_replay_line}\".") + # opcode = first byte & 0xFE + self.__opcode = int(res[1][:2], base=16) & 0xFE + if self.__opcode not in [2, 4]: + raise InvalidIniFileEntryError(f"Error - ARCode has to be in format: '0AXXXXXX XXXXXXXX' with A in [2,3,4,5] and X in [0-9a-fA-F] line {line_number} \"{action_replay_code}\".") - result_list.append( (virtual_address, bytes_value) ) - return result_list + datas = int(res[2], 16).to_bytes(4, "big") if self.__opcode == 0x04 else (int(res[2][:4], 16) + 1) * int(res[2][4:], 16).to_bytes(2, "big") + length = len(datas) + + try: + super().__init__(address, SectionType.UNMAPPED, action_replay_code, length=length) + except OutOfMemoryError: + raise OutOfMemoryError(f"Error - Out of memory address line {line_number}: {address:08x}:{address + length} should be in 0x80003100:0x81200000.") + self.set_datas(datas) + def __str__(self): + return f"| {str(self.__line_number).rjust(8)} | {self.name()} | {self.address():08x} | {self.end_address():08x} | {self.length():08x} |" + def __eq__(a, b): return a.name() == b.name() and a.address() == b.address() and a.end_address() == b.end_address() and a.__line_number == b.__line_number and a.__opcode == b.__opcode and a.datas() == b.datas() + def __ne__(a, b): return a.name() != b.name() or a.address() != b.address() or a.end_address() != b.end_address() or a.__line_number != b.__line_number or a.__opcode != b.__opcode or a.datas() != b.datas() + def line_number(self): return self.__line_number class Dol: - HEADER_LEN = 0x100 + #HEADER_LEN = 0x100 __path = None - __header = None - __data = None - # List of 18 tuples [(offset, address, length, is_used), ] that describe all sections of the dol - __sections_info = None - # (address, length) - __bss_info = None + # [Section, ...] with length = 18 + __sections = None + # Bss object + __bss = None __entry_point = None def __init__(self, path:Path): self.__path = path - data = path.read_bytes() - self.__header = data[:Dol.HEADER_LEN] - self.__data = data[Dol.HEADER_LEN:] + datas = path.read_bytes() - self.__bss_info = ( int.from_bytes(data[0xd8:0xdc], "big"), int.from_bytes(data[0xdc:0xe0], "big") ) - self.__entry_point = int.from_bytes(data[0xe0:0xe4], "big") + self.__bss = Bss( int.from_bytes(datas[0xd8:0xdc], "big"), int.from_bytes(datas[0xdc:0xe0], "big") ) + self.__entry_point = int.from_bytes(datas[0xe0:0xe4], "big") - self.__sections_info = [] + current_section = 0 + sections = [] for i in range(18): - offset = int.from_bytes(data[i*4:i*4+4], "big") - address = int.from_bytes(data[0x48+i*4:0x48+i*4+4], "big") - length = int.from_bytes(data[0x90+i*4:0x90+i*4+4], "big") - is_used = (offset != 0) and (address != 0) and (length != 0) - self.__sections_info.append( (offset, address, length, is_used) ) - # print a table with each sections + section = Section( + i, # index + int.from_bytes(datas[i*4:i*4+4], "big"), # offset + int.from_bytes(datas[0x48+i*4:0x48+i*4+4], "big"), # address + int.from_bytes(datas[0x90+i*4:0x90+i*4+4], "big")) # length + + if section.is_used(): + if i == 7: current_section = 0 + + section.set_datas(datas[section.offset():section.offset()+section.length()]) + section.set_name( f".text{current_section}" if i < 7 else f".data{current_section}" ) + + current_section += 1 + sections.append(section) + # Make a tuple to lock from sorting + self.__sections = tuple(sections) def __str__(self): - res = f"Entry point: {self.__entry_point:08x}\n\n|" - res += "-"*50 + "|\n| Section | Offset | Address | Length | Used |\n|" + "-"*9 + ("|"+"-"*10)*3 + "|" + "-"*7 + "|\n" - i = 0 - for section in self.__sections_info: - res+= "| text"+str(i) if i < 7 else "| data"+str(i) - if i < 10: res += " " - res += f" | {section[0]:08x} | {section[1]:08x} | {section[2]:08x} | {str(section[3]).ljust(5)} |\n" - i += 1 - res += "|"+"-"*50+f"|\n\nbss: address:{self.__bss_info[0]:08x} length:{self.__bss_info[1]:08x}" - return res - """ - # search_raw: bytecode - # we could also identify text sections to improve search - def search_raw(self, bytecode:bytes): - if len(bytecode) == 0: - raise Exception("Error - No bytecode.") - offsets = [] - for i in range(len(self.__data) - len(bytecode) + 1): - if self.__data[i:i+len(bytecode)] == bytecode: - offsets.append(self.resolve_img2virtual(i + Dol.HEADER_LEN)) - return offsets if len(offsets) > 0 else None - """ - # When patching a section we could overflow on the next section - # Return list of [ [virtual_address:int, value:bytes], ... ] - # raise SectionsOverflowError if part of the bytecode is out of the existing sections - # raise InvalidVirtualAddressError if the base virtual address is out of the existing sections - def __get_section_mapped_values(self, virtual_address:int, bytes_value:bytes): - for section_info in self.__sections_info: - if not section_info[3]: continue - # first byte out of section: - section_end_address = section_info[1] + section_info[2] - if virtual_address >= section_info[1] and virtual_address < section_end_address: - if virtual_address + len(bytes_value) <= section_end_address: - return [ [section_info[0] + virtual_address - section_info[1], bytes_value] ] # dol offset and value - # Here we have to split the value to find where in the dol is the second part - splited_len = section_end_address - virtual_address - splited_result = [[section_info[0] + virtual_address - section_info[1], bytes_value[:splited_len]]] - try: - splited_result += self.__get_section_mapped_values(virtual_address + splited_len, bytes_value[splited_len:]) - return splited_result - except InvalidVirtualAddressError: - raise SectionsOverflowError(f"Error - Value Overflow in an inexistant dol initial section: {virtual_address:08x}:{bytes_value.hex()}") - raise InvalidVirtualAddressError(f"Error - Not found in dol initial sections: {virtual_address:08x}") - # Resolve a dol absolute offset to a virtual memory address + 'Print a table with each sections from 0 to 17.' + str_buffer = f"Entry point: {self.__entry_point:08x}\n\n|" + str_buffer += "-"*50 + "|\n| Section | Offset | Address | Length | Used |\n|" + "-"*9 + ("|"+"-"*10)*3 + "|" + "-"*7 + "|\n" + for section in self.__sections: + str_buffer += section.format_raw() + return str_buffer + "|"+"-"*50+f"|\n\n{self.__bss.format()}" + def __get_used_sections(self): return [section for section in self.__sections if section.is_used()] + def __get_merged_mapped_memory(self): + """ + Get sorted intervals where there is datas or text. + return [MemoryObject, ...] + * Merged and sorted + private [Section, ...] + * Don't overlap, section >= 1 + """ + memory_objects = [section.to_memory_object() for section in self.__get_used_sections()] + memory_objects.sort(key=lambda x:x.address()) + + merged_intervals = [memory_objects[0]] + for memory_object in memory_objects[1:]: + if merged_intervals[-1].end_address() == memory_object.address(): + merged_intervals[-1].set_end_address( memory_object.end_address() ) + else: + merged_intervals.append(memory_object) + return merged_intervals def resolve_img2virtual(self, offset:int): + """ + input: dol_absolute_offset + return virtual_memory_address + """ memory_address = None - for section_info in self.__sections_info: - if not section_info[3]: continue - if offset >= section_info[0] and offset < section_info[0] + section_info[2]: - return section_info[1] + offset - section_info[0] + for section in self.__sections: + if section.is_used(): + virtual_address = section.resolve_img2virtual(offset) + if virtual_address is not None: + return virtual_address raise InvalidImgOffsetError(f"Error - Invalid dol image offset: {offset:08x}") - # Resolve a virtual memory address to a dol absolute offset def resolve_virtual2img(self, address:int): - for section_info in self.__sections_info: - if not section_info[3]: continue - if address >= section_info[1] and address < section_info[1] + section_info[2]: - return section_info[0] + address - section_info[1] + """ + input: virtual_memory_address + return dol_absolute_offset + """ + for section in self.__sections: + if section.is_used(): + offset = section.resolve_virtual2img(address) + if offset is not None: + return offset raise InvalidVirtualAddressError(f"Error - Not found in dol initial sections: {address:08x}") def stats(self): - print(self) - # https://www.gc-forever.com/yagcd/chap4.html#sec4 # system: 0x80000000 -> 0x80003100 # available: 0x80003100 -> 0x81200000 @@ -180,58 +446,173 @@ class Dol: # Now we have to generate a memory map with splited bss and empty spaces # [ [section_name, beg_addr, end_addr, length], ... ] - result_intervals = [ - ["system", 0x80000000, 0x80003100, 0x3100], - ["apploader", 0x81200000, 0x81300000, 0x100000], - ["Bootrom/IPL", 0x81300000, 0x81800000, 0x500000]] + memory_objects = [ + MemoryObject(0x80000000, SectionType.SYS, "System", length=0x3100), + MemoryObject(0x81200000, SectionType.SYS, "Apploader", length=0x100000), + MemoryObject(0x81300000, SectionType.SYS, "Bootrom/IPL", length=0x500000)] + self.__get_used_sections() - i = 0 - for section in self.__sections_info[:7]: - if section[3]: - result_intervals.append( [f".text{i}", section[1], section[1] + section[2], section[2]] ) - i += 1 + splited = self.__bss.split(memory_objects) + if splited is not None: + memory_objects += splited - i = 0 - for section in self.__sections_info[7:18]: - if section[3]: - result_intervals.append( [f".data{i}", section[1], section[1] + section[2], section[2]] ) - i += 1 + # We search now unmapped program space + memory_objects += MemoryObject(0x80003100, SectionType.UNMAPPED, "Empty", end_address=0x81200000) - memory_objects - result_intervals.sort(key=lambda x: x[1]) - i = 0 - for bss_interval in remove_intervals_from_interval([None, self.__bss_info[0], self.__bss_info[0] + self.__bss_info[1]], result_intervals): - result_intervals.append( [f".bss{i}", bss_interval[1], bss_interval[2], bss_interval[2] - bss_interval[1]] ) - i += 1 - - # We search now available program space - result_intervals.sort(key=lambda x: x[1]) - empty_intervals = remove_intervals_from_interval(["empty", 0x80003100, 0x81200000], result_intervals) - for empty_interval in empty_intervals: - result_intervals += [[empty_interval[0], empty_interval[1], empty_interval[2], empty_interval[2] - empty_interval[1]]] - - result_intervals.sort(key=lambda x: x[1]) + memory_objects.sort(key=lambda x: x.address()) str_buffer = "\n|"+"-"*46+"|\n| Section | beg_addr | end_addr | length |\n|" + "-"*13 + ("|"+"-"*10)*3 + "|\n" - for interval in result_intervals: - str_buffer += f"| {interval[0].ljust(11)} | {interval[1]:08x} | {interval[2]:08x} | {interval[3]:08x} |\n" - - print(str_buffer + "|"+"-"*46+"|") - def extract(self, filename:str, section_index:int): + for memory_object in memory_objects: + str_buffer += str(memory_object)+"\n" + print(f"{self}{str_buffer}|"+"-"*46+"|") + def extract(self, filename:str, section_index:int, output_path:Path): if section_index > 17: raise Exception("Error - Section index has to be in 0 - 17") - begin_offset = self.__sections_info[section_index][0] - self.HEADER_LEN - end_offset = begin_offset + self.__sections_info[section_index][2] + output_path.write_bytes(self.__sections[section_index].datas()) + def analyse_action_replay(self, action_replay_list:list): + merged_intervals = self.__get_merged_mapped_memory() - section_type = "text" if section_index < 7 else "data" - Path(f"{filename}_{section_type}{section_index}").write_bytes(self.__data[begin_offset:end_offset]) - # [ [virtual_address:int, value:bytes], ... ] - def patch_action_replay(self, virtualaddress_bytes_list:list): - self.__data = bytearray(self.__data) - for virtualaddress_bytes in virtualaddress_bytes_list: - offset = self.resolve_virtual2img(virtualaddress_bytes[0]) - for mapped_list in self.__get_section_mapped_values(virtualaddress_bytes[0], virtualaddress_bytes[1]): - print(f"Patching {virtualaddress_bytes[0]:08x} at dol offset {offset:08x} with value {virtualaddress_bytes[1].hex()}") - self.__data[offset: offset + len(virtualaddress_bytes[1])] = virtualaddress_bytes[1] + overlaps_list = get_overlapping_arcodes(action_replay_list) + + # Get unmapped groups splited by sections intervals: + # each group contains intervals to patch grouped by data sections to add + unmapped_memory_objects = get_unmapped_intervals(merged_intervals, action_replay_list) + + if overlaps_list is not None: + str_buffer = "Found overlapping ARCodes:\n" + str_buffer += "|"+"-"*127+"|\n| Line | ActionReplayCode1 | beg_addr | end_addr | length | Line | ActionReplayCode2 | beg_addr | end_addr | length |\n|" + ("-"*10 + "|" + "-"*19 + ("|"+"-"*10)*3 + "|")*2 + "\n" + for [arcode0, arcode1] in overlaps_list: + str_buffer += str(arcode0)[-1] + str(arcode1) + "\n" + print(str_buffer+"|"+"-"*127+"|") + else: + print(f"No overlapping ARCodes found.") + + if unmapped_memory_objects is not None: + str_buffer = "\nUnmapped virtual addresses intervals used by ARCodes:\n"+"|"+"-"*32+"|\n| beg_addr | end_addr | length |\n"+("|"+"-"*10)*3 +"|\n" + for unmapped_memory_object in unmapped_memory_objects: + unmapped_memory_object.align() + str_buffer += f"| {unmapped_memory_object.address():08x} | {unmapped_memory_object.end_address():08x} | {unmapped_memory_object.length():08x} |\n" + print(str_buffer+"|"+"-"*32+"|") + #print("Use -par file.dol -ini arcodes.ini \"-auto\" to remap sections and allow complete processing of the ARCodes in this ini file. Else the patching process will be interupted for out of dol ARCodes.") + else: + print(f"No out of sections ARCodes found.\n") + def patch_memory_objects(self, output_path:Path, memory_objects:list): + """ + input: [MemoryObject, ... ] + return True + raise SectionsOverflowError if part of the bytecode is out of the existing sections + raise InvalidVirtualAddressError if the base virtual address is out of the existing sections + """ + sections = self.__get_used_sections() + sections.sort(key=lambda x: x.address()) + def split_and_patch(sections:list, memory_object:MemoryObject): + """ + When patching a section we could overflow on the next section or in the previous. + input: ActionReplayCode + return True + raise SectionsOverflowError if part of the bytecode is out of the existing sections + raise InvalidVirtualAddressError if the base virtual address is out of the existing sections + """ + for section in sections: + try: + # Intersection + if not memory_object & section: continue + + # Split left_interval, in, right_interval + splited = memory_object / section + + if IntervalDiv.LEFT in splited: + split_and_patch(sections, splited[IntervalDiv.LEFT]) + + logging.debug(f"----> offset:{section.offset() + splited[IntervalDiv.IN].address() - section.address():08x} val:{splited[IntervalDiv.IN].datas().hex()}") + section.update_datas( splited[IntervalDiv.IN] ) + + if IntervalDiv.RIGHT in splited: + split_and_patch(sections, splited[IntervalDiv.RIGHT]) + + return True + except InvalidVirtualAddressError: + raise SectionsOverflowError(f"Error - Value Overflow in an inexistant dol initial section: {memory_object.address():08x}:{memory_object.datas().hex()}") + raise InvalidVirtualAddressError(f"Error - Not found in dol initial sections: {memory_object.address():08x}:{memory_object.end_address():08x}") + + for memory_object in memory_objects: + logging.debug(f"Processing {memory_object.name()} address:{memory_object.address():08x}") + split_and_patch(sections, memory_object) + self.__save(output_path) + def remap_sections(self, action_replay_list:list): + merged_intervals = self.__get_merged_mapped_memory() + unmapped_memory_objects = get_unmapped_intervals(merged_intervals, action_replay_list) + + if unmapped_memory_objects is None: + return True + + text_sections = [] + data_sections = [] + for section in self.__sections: + if section.is_used(): + section.set_offset(0) + section.set_index(None) + if section.type() == SectionType.TEXT: + text_sections.append(section) + else: + data_sections.append(section) + self.__sections = None + + if len(unmapped_memory_objects) + len(data_sections) > 11: + raise Exception("Error - Not enought empty data sections available for remapping.") + + for unmapped_memory_object in unmapped_memory_objects: + unmapped_memory_object.align() + new_section = Section(None, 0, unmapped_memory_object.address(), unmapped_memory_object.length(), section_type=SectionType.UNMAPPED) + new_section.set_datas( bytearray(b"\x00" * new_section.length()) ) + data_sections.append( new_section ) + + text_sections.sort(key=lambda x: x.address()) + data_sections.sort(key=lambda x: x.address()) + + sections = [] + current_offset = 0x100 + i = 0 + for text_section in text_sections: + sections.append( text_section ) + text_section.set_index(i) + text_section.set_offset(current_offset) + text_section.set_type(SectionType.TEXT) + current_offset += text_section.length() + i += 1 + while i < 7: + sections.append( Section(i, 0, 0, 0) ) + i += 1 + for data_section in data_sections: + sections.append( data_section ) + data_section.set_index(i) + data_section.set_offset(current_offset) + data_section.set_type(SectionType.DATA) + current_offset += data_section.length() + i += 1 + while i < 18: + sections.append( Section(i, 0, 0, 0) ) + i += 1 + self.__sections = tuple(sections) + def __save(self, output_path:Path): + offsets = b"" + addresses = b"" + lengths = b"" + for section in self.__sections: + offsets += section.offset().to_bytes(4, "big") + addresses += section.address().to_bytes(4, "big") + lengths += section.length().to_bytes(4, "big") + datas = offsets + addresses + lengths +\ + self.__bss.address().to_bytes(4, "big") + self.__bss.length().to_bytes(4, "big") +\ + self.__entry_point.to_bytes(4, "big") + datas = datas.ljust(0x100, b"\x00") + for section in sorted(self.__sections, key=lambda x: x.offset()): + if section.is_used(): + if len(datas) != section.offset(): + raise InvalidSectionOffsetError(f"Error - Section {section.index()} has an offset that does'nt match the previous datas length.") + if len(section.datas()) != section.length(): + raise Exception(f"Error - Invalid datas length.") + datas += section.datas() + output_path.write_bytes(datas) def get_argparser(): @@ -240,14 +621,26 @@ def get_argparser(): parser.add_argument('--version', action='version', version='%(prog)s ' + __version__) parser.add_argument('-v', '--verbose', action='store_true', help='verbose mode') parser.add_argument('input_path', metavar='INPUT', help='') - parser.add_argument('arg2', metavar='arg2', help='', nargs='?', default=None) + parser.add_argument('-o', '--output-path', type=str, help='-o path: output path.', default=None) + parser.add_argument('-ini', '--ini-path', type=str, help='-ini path: ini path.', default=None) + parser.add_argument('-sr', '--sections-remap', action='store_true', help="-sr: remap the data sections of the dol to allow full ARCodes ini" + " file processing.", default=None) group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-v2i', '--virtual2image', action='store_true', help="-v2i source.dol virtual_address: Translate a virtual address into a dol offset if this was originaly mapped from data or text. virtual_address has to be in hexadecimal: 80003100.") - group.add_argument('-i2v', '--image2virtual', action='store_true', help="-i2b source.dol dol_offset: Translate a dol offset to a virtual address mapped from data or text. dol_offset has to be in hexadecimal: 2000.") - group.add_argument('-s', '--stats', action='store_true', help="-s source.dol: Get stats about entry point, sections, bss and unused virtual address space.") - group.add_argument('-e', '--extract', action='store_true', help="-e source.dol section_index: Extract a section. index must be between 0 and 17") - group.add_argument('-par', '--patch-action-replay', action='store_true', help="-p source.dol action_replay.ini: Patch initialised data inside the dol with an ini file containing a list of [write] directives. Handle only ARCodes beginning with 04.") + group.add_argument('-v2i', '--virtual2image', type=str, help="-v2i source.dol virtual_address: Translate a virtual address into " + "a dol offset if this was originaly mapped from data or text. virtual_address has to be in hexadecimal: 80003100.") + group.add_argument('-i2v', '--image2virtual', type=str, help="-i2v source.dol dol_offset: Translate a dol offset to a virtual ad" + "dress mapped from data or text. dol_offset has to be in hexadecimal: 2000.") + group.add_argument('-s', '--stats', action='store_true', help="-s source.dol: Get stats about entry point, sections, bss and unu" + "sed virtual address space.") + group.add_argument('-e', '--extract', type=int, help="-e source.dol section_index [-o output_path]: Extract a section. index mus" + "t be between 0 and 17") + group.add_argument('-aar', '--analyse-action-replay', action='store_true', help="-aar source.dol action_replay.ini: Analyse an i" + "ni file containing a list of [write] directives to show unmapped sections to add for processing all ARCodes including thoos" + "e who are in inexistant sections. Handle only ARCodes beginning with [02, 03, 04, 05].") + group.add_argument('-par', '--patch-action-replay', action='store_true', help="-par source.dol -ini action_replay.ini [-o output" + "_path] [-sr]: Patch initialised data inside the dol with an ini file containing a list of [write] directives. Handle only A" + "RCodes beginning with [02, 03, 04, 05]. If -sr is specified then add or update .data sections to allow full ini processing.") return parser @@ -266,18 +659,14 @@ if __name__ == '__main__': dol = Dol(p_input) if args.virtual2image: - if args.arg2 is None: - raise Exception("Error - Virtual address has to be specified in hexadecimal: 80003000.") - virtual_address = int(args.arg2, 16) + virtual_address = int(args.virtual2image, 16) try: offset = dol.resolve_virtual2img(virtual_address) print(f"Virtual address {virtual_address:08x} is at dol offset {offset:08x}") except InvalidVirtualAddressError: print("This virtual address is not in the dol.") elif args.image2virtual: - if args.arg2 is None: - raise Exception("Error - dol offset has to be specified in hexadecimal: 1234.") - offset = int(args.arg2, 16) + offset = int(args.image2virtual, 16) try: virtual_address = dol.resolve_img2virtual(offset) print(f"Dol offset {offset:08x} is at virtual address {virtual_address:08x}") @@ -287,20 +676,40 @@ if __name__ == '__main__': dol.stats() elif args.extract: logging.info("### Extract section") - if args.arg2 is None: - raise Exception("Error - Section index has to be specified.") - index = int(args.arg2) + index = args.extract section_type = "text" if index < 7 else "data" - logging.info(f"Extracting section {index} in file {p_input.name}_{section_type}{index}...") - dol.extract(p_input.name, index) + output_path = Path(args.output_path) if args.output_path is not None else Path(f"{p_input.name}_{section_type}{index}") + logging.info(f"Extracting section {index} in file {output_path}...") + + dol.extract(p_input.name, index, output_path) + elif args.analyse_action_replay: + logging.info("### Analyse Action Replay ini file") + if args.ini_path is None: + raise Exception("Error - Action Replay ini file has to be specified.") + action_replay_ini_path = Path(args.ini_path) + if not action_replay_ini_path.is_file(): + raise Exception("Error - Invalid action replay ini file path.") + dol.analyse_action_replay(parse_action_replay_ini(action_replay_ini_path)) elif args.patch_action_replay: logging.info("### Patch dol using Action Replay ini file") - if args.arg2 is None: + if args.ini_path is None: raise Exception("Error - Action Replay ini file has to be specified.") - action_replay_ini_path = Path(args.arg2) + action_replay_ini_path = Path(args.ini_path) if not action_replay_ini_path.is_file(): raise Exception("Error - Invalid action replay ini file path.") - logging.info(f"Patching dol {p_input} using .ini file {action_replay_ini_path}...") - dol.patch_action_replay(parse_action_replay_ini(action_replay_ini_path)) + if not args.output_path: + raise Exception("Error - Output path has to be specified.") + output_path = Path(args.output_path) + if output_path.is_file(): + raise Exception(f"Error - Please remove {output_path}.") + + logging.info(f"Patching dol {p_input} in {output_path} using {action_replay_ini_path} ini file...") + + action_replay_list = parse_action_replay_ini(action_replay_ini_path) + if args.sections_remap != None: + logging.info(f"Sections remapping using action replay ini file...") + dol.remap_sections(action_replay_list) + + dol.patch_memory_objects(output_path, action_replay_list)