From cc435d98cbf435d51b3deae1cba0784c9dcabb53 Mon Sep 17 00:00:00 2001 From: Steven Garcia Date: Wed, 17 Jan 2024 22:21:06 -0600 Subject: [PATCH 01/16] Halo 1 MCC map extraction support --- refinery/core.py | 10 ++++++---- refinery/main.py | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/refinery/core.py b/refinery/core.py index 1c47809..c4ed751 100644 --- a/refinery/core.py +++ b/refinery/core.py @@ -30,6 +30,7 @@ from reclaimer.meta.wrappers.halo1_map import Halo1Map from reclaimer.meta.wrappers.halo1_yelo import Halo1YeloMap from reclaimer.meta.wrappers.halo1_anni_map import Halo1AnniMap +from reclaimer.meta.wrappers.halo1_mcc_map import Halo1MccMap from reclaimer.meta.wrappers.halo1_rsrc_map import Halo1RsrcMap from reclaimer.meta.wrappers.halo2_map import Halo2Map from reclaimer.meta.wrappers.halo3_map import Halo3Map @@ -91,6 +92,7 @@ halo_map_wrappers_by_engine[name] = Halo2Map # Use a different wrapper for this so we can use a different set of tag defs halo_map_wrappers_by_engine["halo1yelo"] = Halo1YeloMap +halo_map_wrappers_by_engine["halo1mcc"] = Halo1MccMap def get_halo_map_section_ends(halo_map): @@ -380,7 +382,7 @@ def save_map(self, save_path=None, map_name=ACTIVE_INDEX, raise KeyError("No map loaded and none provided.") elif halo_map.is_resource: raise TypeError("Cannot save resource maps.") - elif halo_map.engine not in ("halo1ce", "halo1yelo", + elif halo_map.engine not in ("halo1ce", "halo1yelo", "halo1mcc", "halo1pc", "halo1vap"): raise TypeError("Cannot save this kind of map.") elif is_path_empty(save_path): @@ -604,7 +606,7 @@ def load_resource_maps(self, halo_map=None, maps_dir="", map_paths=(), **kw): def deprotect_all(self, **kw): for engine in self.maps_by_engine: - if engine not in ("halo1ce", "halo1yelo", "halo1pc", "halo1vap"): + if engine not in ("halo1ce", "halo1yelo", "halo1pc", "halo1vap", "halo1mcc"): continue maps = self.maps_by_engine[engine] @@ -625,7 +627,7 @@ def deprotect(self, save_path=None, map_name=ACTIVE_INDEX, raise KeyError("No map loaded.") elif halo_map.is_resource: raise TypeError("Cannot deprotect resource maps.") - elif halo_map.engine not in ("halo1ce", "halo1yelo", + elif halo_map.engine not in ("halo1ce", "halo1yelo", "halo1mcc", "halo1pc", "halo1vap"): raise TypeError("Cannot deprotect this kind of map.") @@ -867,7 +869,7 @@ def repair_tag_classes(self, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX): if not halo_map: return repaired - if halo_map.engine not in ("halo1ce", "halo1yelo", + if halo_map.engine not in ("halo1ce", "halo1yelo", "halo1mcc", "halo1pc", "halo1vap"): raise TypeError('Cannot repair tag classes in "%s" maps' % halo_map.engine) diff --git a/refinery/main.py b/refinery/main.py index e9c42b7..b958c4c 100644 --- a/refinery/main.py +++ b/refinery/main.py @@ -1272,7 +1272,7 @@ def save_map_as(self, e=None): elif halo_map.is_resource: print("Cannot save resource maps.") return Path("") - elif halo_map.engine not in ("halo1ce", "halo1yelo", + elif halo_map.engine not in ("halo1ce", "halo1yelo", "halo1mcc", "halo1pc", "halo1vap"): print("Cannot save this kind of map.") return Path("") @@ -1317,7 +1317,7 @@ def save_map(self, save_path=None, map_name=ACTIVE_INDEX, elif halo_map.is_resource: print("Cannot save resource maps.") return Path("") - elif halo_map.engine not in ("halo1ce", "halo1yelo", + elif halo_map.engine not in ("halo1ce", "halo1yelo", "halo1mcc", "halo1pc", "halo1vap"): print("Cannot save this kind of map.") return Path("") From 52ebbff9a26fac8a2c7f52a37ab9ad814e833b3e Mon Sep 17 00:00:00 2001 From: Steven Garcia Date: Sun, 21 Jan 2024 22:59:33 -0600 Subject: [PATCH 02/16] bump version info --- refinery/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/refinery/__init__.py b/refinery/__init__.py index c43e338..0338f8a 100644 --- a/refinery/__init__.py +++ b/refinery/__init__.py @@ -12,8 +12,8 @@ # ############## __author__ = "Sigmmma" # YYYY.MM.DD -__date__ = "2020.03.13" -__version__ = (2, 6, 0) +__date__ = "2024.01.16" +__version__ = (2, 7, 0) __website__ = "https://github.com/Sigmmma/refinery" __all__ = ( 'defs', 'heuristic_deprotection', 'repl', 'tag_index', 'widgets', 'windows', From ed9ad8ebecafe5a5cec0c6874596a08fd993a4d2 Mon Sep 17 00:00:00 2001 From: Steven Garcia Date: Tue, 23 Jan 2024 16:19:40 -0600 Subject: [PATCH 03/16] Change how map wrapper gets defined --- refinery/core.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/refinery/core.py b/refinery/core.py index c4ed751..a847303 100644 --- a/refinery/core.py +++ b/refinery/core.py @@ -72,27 +72,26 @@ # and what tag defintions to use # Note the loops after this that handle engines that use the same map format halo_map_wrappers_by_engine = { - "stubbs": StubbsMap, - "stubbspc": StubbsMap, - "shadowrun_proto": ShadowrunMap, - "halo1anni": Halo1AnniMap, - "halo2": Halo2Map, - "halo3beta": Halo3BetaMap, - "halo3": Halo3Map, - "halo3odst": Halo3OdstMap, - "haloreachbeta": HaloReachBetaMap, - "haloreach": HaloReachMap, - "halo4beta": Halo4BetaMap, - "halo4": Halo4Map, - "halo5": Halo5Map, + "stubbs": StubbsMap, + "stubbspc": StubbsMap, + "shadowrun_proto": ShadowrunMap, + "halo1anni": Halo1AnniMap, + "halo1yelo": Halo1YeloMap, + "halo1mcc": Halo1MccMap, + "halo2": Halo2Map, + "halo3beta": Halo3BetaMap, + "halo3": Halo3Map, + "halo3odst": Halo3OdstMap, + "haloreachbeta": HaloReachBetaMap, + "haloreach": HaloReachMap, + "halo4beta": Halo4BetaMap, + "halo4": Halo4Map, + "halo5": Halo5Map, } for name in GEN_1_HALO_ENGINES: - halo_map_wrappers_by_engine[name] = Halo1Map + halo_map_wrappers_by_engine.setdefault(name, Halo1Map) for name in GEN_2_ENGINES: - halo_map_wrappers_by_engine[name] = Halo2Map -# Use a different wrapper for this so we can use a different set of tag defs -halo_map_wrappers_by_engine["halo1yelo"] = Halo1YeloMap -halo_map_wrappers_by_engine["halo1mcc"] = Halo1MccMap + halo_map_wrappers_by_engine.setdefault(name, Halo2Map) def get_halo_map_section_ends(halo_map): From 1765fced917561702401c46db8b00509cc540bd5 Mon Sep 17 00:00:00 2001 From: Steven Garcia Date: Tue, 23 Jan 2024 16:20:07 -0600 Subject: [PATCH 04/16] bump version --- refinery/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/refinery/__init__.py b/refinery/__init__.py index 0338f8a..5512ec2 100644 --- a/refinery/__init__.py +++ b/refinery/__init__.py @@ -12,8 +12,8 @@ # ############## __author__ = "Sigmmma" # YYYY.MM.DD -__date__ = "2024.01.16" -__version__ = (2, 7, 0) +__date__ = "2024.01.23" +__version__ = (2, 7, 1) __website__ = "https://github.com/Sigmmma/refinery" __all__ = ( 'defs', 'heuristic_deprotection', 'repl', 'tag_index', 'widgets', 'windows', From 14f382daa7ecf6f4d5edce08e3639f28f8da6ad6 Mon Sep 17 00:00:00 2001 From: Steven Garcia Date: Thu, 25 Jan 2024 23:59:59 -0600 Subject: [PATCH 05/16] Calculate tag crc and handle 64bit Stubbs --- refinery/__init__.py | 4 ++-- refinery/core.py | 47 +++++++++++++++++++++++++++++++------------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/refinery/__init__.py b/refinery/__init__.py index 5512ec2..b7928b2 100644 --- a/refinery/__init__.py +++ b/refinery/__init__.py @@ -12,8 +12,8 @@ # ############## __author__ = "Sigmmma" # YYYY.MM.DD -__date__ = "2024.01.23" -__version__ = (2, 7, 1) +__date__ = "2024.01.25" +__version__ = (2, 7, 2) __website__ = "https://github.com/Sigmmma/refinery" __all__ = ( 'defs', 'heuristic_deprotection', 'repl', 'tag_index', 'widgets', 'windows', diff --git a/refinery/core.py b/refinery/core.py index a847303..4994c18 100644 --- a/refinery/core.py +++ b/refinery/core.py @@ -29,6 +29,7 @@ from reclaimer.hek import hardcoded_ce_tag_paths from reclaimer.meta.wrappers.halo1_map import Halo1Map from reclaimer.meta.wrappers.halo1_yelo import Halo1YeloMap +from reclaimer.meta.wrappers.halo1_xbox_map import Halo1XboxMap from reclaimer.meta.wrappers.halo1_anni_map import Halo1AnniMap from reclaimer.meta.wrappers.halo1_mcc_map import Halo1MccMap from reclaimer.meta.wrappers.halo1_rsrc_map import Halo1RsrcMap @@ -42,14 +43,16 @@ from reclaimer.meta.wrappers.halo4_beta_map import Halo4BetaMap from reclaimer.meta.wrappers.halo5_map import Halo5Map from reclaimer.meta.wrappers.stubbs_map import StubbsMap +from reclaimer.meta.wrappers.stubbs_map_64bit import StubbsMap64Bit from reclaimer.meta.wrappers.shadowrun_map import ShadowrunMap -from reclaimer.meta.halo_map import get_map_header, get_map_version,\ +from reclaimer.meta.halo_map import get_map_header, get_engine_name,\ get_tag_index, get_map_magic from reclaimer.meta.class_repair import class_repair_functions,\ get_tagc_refs from reclaimer.meta.rawdata_ref_editing import rawdata_ref_move_functions from reclaimer.meta.halo1_map_fast_functions import class_bytes_by_fcc +from reclaimer.util import calc_halo_crc32 from refinery.constants import INF, ACTIVE_INDEX, MAP_TYPE_ANY,\ MAP_TYPE_REGULAR, MAP_TYPE_RESOURCE @@ -74,7 +77,10 @@ halo_map_wrappers_by_engine = { "stubbs": StubbsMap, "stubbspc": StubbsMap, + "stubbspc64bit": StubbsMap64Bit, "shadowrun_proto": ShadowrunMap, + "halo1xbox": Halo1XboxMap, + "halo1xboxdemo": Halo1XboxMap, "halo1anni": Halo1AnniMap, "halo1yelo": Halo1YeloMap, "halo1mcc": Halo1MccMap, @@ -98,7 +104,7 @@ def get_halo_map_section_ends(halo_map): head = halo_map.map_header index = halo_map.tag_index raw_data_end = index.model_data_offset - vertex_data_end = index.vertex_data_size + raw_data_end + vertex_data_end = index.index_parts_offset + raw_data_end index_data_end = index.model_data_size + raw_data_end meta_data_end = head.tag_data_size + head.tag_index_header_offset return raw_data_end, vertex_data_end, index_data_end, meta_data_end @@ -130,9 +136,9 @@ def expand_halo_map(halo_map, raw_data_expansion=0, vertex_data_expansion=0, meta_ptr_diff = diff - meta_data_expansion # update the map_header and tag_index_header's offsets and sizes - tag_index.model_data_offset += raw_data_expansion - tag_index.vertex_data_size += vertex_data_expansion - tag_index.model_data_size += vertex_data_expansion + triangle_data_expansion + tag_index.model_data_offset += raw_data_expansion + tag_index.index_parts_offset += vertex_data_expansion + tag_index.model_data_size += vertex_data_expansion + triangle_data_expansion halo_map.map_magic -= meta_ptr_diff map_header.tag_index_header_offset += meta_ptr_diff map_header.decomp_len = map_end @@ -351,7 +357,8 @@ def unload_maps(self, map_type=MAP_TYPE_REGULAR, self.set_active_map() def unload_map(self, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX, **kw): - halo_map = self.maps_by_engine.get(engine, {}).get(map_name) + maps = self.maps_by_engine.get(engine, {}) + halo_map = maps.get(map_name) if halo_map is None: return @@ -359,6 +366,12 @@ def unload_map(self, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX, **kw): self.active_map_name = "" halo_map.unload_map() + # pop the map out in case the map uses its own separate maps dict + # due to conflicting resources being referenced. This can occur + # due to multiple yelo maps being loaded that each use different + # resource maps, or mcc maps that can do the same thing. + maps.pop(map_name, None) + maps.pop(halo_map.map_name, None) def save_map(self, save_path=None, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX, **kw): @@ -537,7 +550,7 @@ def load_map(self, map_path, replace_if_same_name=False, **kw): with get_rawdata_context(filepath=map_path, writable=False) as f: head_sig = unpack(" Date: Sat, 10 Feb 2024 13:50:47 -0600 Subject: [PATCH 06/16] Add additional debug info --- refinery/core.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/refinery/core.py b/refinery/core.py index 4994c18..c4555f3 100644 --- a/refinery/core.py +++ b/refinery/core.py @@ -1234,8 +1234,9 @@ def process_queue(self, **kw): self.process_queue_item(item, **extract_kw) if kw.get("do_printout", self.do_printout): print() # print a new line to separate operations - except RefineryError: - print(format_exc(0)) # only last line for RefineryErrors + except RefineryError as e: + print(" Could not process queue item: %s" % + (e.args[0] if e.args else "Unspecified reason.")) except Exception: print(format_exc()) @@ -1448,8 +1449,9 @@ def extract_tags(self, tag_ids, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX, **kw tag_path = "%s.%s" % (PureWindowsPath(tag_index_ref.path), tag_index_ref.class_1.enum_name) tagslist += "%s: %s\n" % (extract_mode, tag_path) - except RefineryError: - print(format_exc(0)) # only last line for RefineryErrors + except RefineryError as e: + print(" Could not extract tag: %s" % + (e.args[0] if e.args else "Unspecified reason.")) except Exception: print(format_exc()) From 1f1beeb50a975a1bd9e5cbc662ef4df63e2b65f3 Mon Sep 17 00:00:00 2001 From: Steven Garcia Date: Sat, 10 Feb 2024 13:51:20 -0600 Subject: [PATCH 07/16] bump version --- refinery/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/refinery/__init__.py b/refinery/__init__.py index b7928b2..6b9d553 100644 --- a/refinery/__init__.py +++ b/refinery/__init__.py @@ -12,8 +12,8 @@ # ############## __author__ = "Sigmmma" # YYYY.MM.DD -__date__ = "2024.01.25" -__version__ = (2, 7, 2) +__date__ = "2024.02.10" +__version__ = (2, 7, 3) __website__ = "https://github.com/Sigmmma/refinery" __all__ = ( 'defs', 'heuristic_deprotection', 'repl', 'tag_index', 'widgets', 'windows', From 4150415208ea73d4e3c53ff3c81045f90539d734 Mon Sep 17 00:00:00 2001 From: Steven Garcia Date: Sat, 10 Feb 2024 21:27:58 -0600 Subject: [PATCH 08/16] Check extension for tags log --- refinery/windows/actions_window.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/refinery/windows/actions_window.py b/refinery/windows/actions_window.py index 70b73d9..4e9b271 100644 --- a/refinery/windows/actions_window.py +++ b/refinery/windows/actions_window.py @@ -284,15 +284,23 @@ def tags_list_browse(self): init_dir = os.path.dirname(self.tagslist_path.get()) except Exception: init_dir = None - dirpath = asksaveasfilename( + filepath = asksaveasfilename( initialdir=init_dir, parent=self, title="Select where to save the tag list log", - filetypes=(("text log", "*.txt"), ("All", "*"))) + filetypes=( + ("text file", "*.txt"), + ("text log", "*.log"), + ("All", "*") + )) - if not dirpath: + if not filepath: return - self.tagslist_path.set(str(Path(dirpath))) + filepath = Path(filepath) + if not filepath.suffix: + filepath = filepath.with_suffix(".txt") + + self.tagslist_path.set(str(filepath)) def extract_to_browse(self): dirpath = askdirectory( From e765ec4e967c820c29b51dd7bdf8318989ddbe1f Mon Sep 17 00:00:00 2001 From: Steven Garcia Date: Sat, 10 Feb 2024 21:28:22 -0600 Subject: [PATCH 09/16] bump version --- refinery/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/refinery/__init__.py b/refinery/__init__.py index 6b9d553..3153a00 100644 --- a/refinery/__init__.py +++ b/refinery/__init__.py @@ -13,7 +13,7 @@ __author__ = "Sigmmma" # YYYY.MM.DD __date__ = "2024.02.10" -__version__ = (2, 7, 3) +__version__ = (2, 7, 4) __website__ = "https://github.com/Sigmmma/refinery" __all__ = ( 'defs', 'heuristic_deprotection', 'repl', 'tag_index', 'widgets', 'windows', From 0201250d58db8b38c89af0d0563f5697b8872383 Mon Sep 17 00:00:00 2001 From: Steven Garcia Date: Tue, 20 Feb 2024 07:33:39 -0600 Subject: [PATCH 10/16] deprotector improvements --- refinery/arbytmap_ext.py | 160 ++++++++++ refinery/core.py | 158 ++++++---- refinery/defs/config_def.py | 11 +- refinery/heuristic_deprotection/constants.py | 16 +- refinery/heuristic_deprotection/functions.py | 308 +++++++++++-------- refinery/heuristic_deprotection/util.py | 95 +++--- refinery/main.py | 70 +++-- refinery/tag_index/tag_path_handler.py | 68 +++- refinery/util.py | 17 +- refinery/widgets/explorer_hierarchy_tree.py | 13 +- refinery/windows/settings_window.py | 70 ++++- 11 files changed, 693 insertions(+), 293 deletions(-) create mode 100644 refinery/arbytmap_ext.py diff --git a/refinery/arbytmap_ext.py b/refinery/arbytmap_ext.py new file mode 100644 index 0000000..6947520 --- /dev/null +++ b/refinery/arbytmap_ext.py @@ -0,0 +1,160 @@ +''' +This module adds an new format to arbytmap, which allows it +to read new formats. Specifically it adds A16R16G16B16_F +and R9G9B9E5, which is are floating-point formats +''' + +import arbytmap +import array +import types + +FORMAT_R9G9B9E5_SHAREDEXP = "R9G9B9E5_SHAREDEXP" +FORMAT_A16R16G16B16F = "A16R16G16B16F" + +MAX_R9G9B9E5_UNPACK_VALUE = 64 +AVG_R9G9B9E5_UNPACK_SCALE = 10 + + +def _apply_exponent_scales_to_pixels(pixels, scales): + # scale the pixels to floats + float_pixels = map(int(511).__rtruediv__, pixels) + + # multiply the float pixels by their scales and return them + return list(map(float.__mul__, float_pixels, scales)) + + +def unpack_r9g9b9e5(arby, bitmap_index, width, height, depth=1): + # unpack the pixels so we can easily manipulate them + unpacked_pixels = arby._unpack_raw_4_channel( + arby.texture_block[bitmap_index], + _R9G9B9E5_OFFSETS, _R9G9B9E5_MASKS, + [_UPSCALE_5BIT, _UPSCALE_9BIT, _UPSCALE_9BIT, _UPSCALE_9BIT] + ) + + # slice the eponents out, and apply map operations to get an exponent + # scale for each exponent using the following algorithm: v = 2^(e-15) + exps = map(int(15).__rsub__, unpacked_pixels[0::4]) + exp_scales = list(map(int(2).__pow__, exps)) + + # apply the exponent to each channel + channel_values = [(0,)] + for i in (1, 2, 3): + channel_values.append(_apply_exponent_scales_to_pixels( + unpacked_pixels[i::4], exp_scales, + )) + + # NOTE: this part is a hack. we're scaling the color range values + # to take up as much of the [0, 65536] value range as possible + average_vals = list(sum(vals)/len(vals) for vals in channel_values) + max_val = min( + MAX_R9G9B9E5_UNPACK_VALUE, + AVG_R9G9B9E5_UNPACK_SCALE * max(average_vals) + ) + max_clamp = types.MethodType(min, max_val) + for i in (1, 2, 3): + # clamp the values that are just too bright to the max + channel_values[i] = map(max_clamp, channel_values[i]) + # scale the values to the range we defined + channel_values[i] = map(float(0xFFff/max_val).__mul__, channel_values[i]) + + # create a new array to hold the exponent applied pixels + exp_applied_pixels = array.array( + unpacked_pixels.typecode, b'\xFF' * (len(unpacked_pixels) * unpacked_pixels.itemsize) + ) + for i in (1, 2, 3): + # convert the floats to ints and slice them into + # the new array(leaving the alpha white) + exp_applied_pixels[i::4] = array.array("H", map(int, channel_values[i])) + + return exp_applied_pixels + + +def unpack_a16r16g16b16_f(arby, bitmap_index, width, height, depth=1): + offsets = arby.channel_offsets + masks = arby.channel_masks + upscalers = arby.channel_upscalers + chan_ct = len(offsets) + + blank_channels = [] + for i in range(chan_ct): + if len(set(upscalers[i])) == 1: + blank_channels.append(i) + upscalers[i] = _UPSCALE_16BIT_ERASE + + # convert the pixel channels to uint16's so we can easily manipulate them + packed_channels = arby._unpack_raw_4_channel( + arby.texture_block[bitmap_index], offsets, masks, upscalers, + ) + + channel_values = array.array("I", b'\x00' * (4 * len(packed_channels))) + # pad the half-floats to regular floats + for i, half_float in enumerate(packed_channels): + # sign, exponent, and mantissa + s = half_float >> 15 + e = (half_float >> 10) & 0x1F + m = half_float & 0x3FF + + if e == 0x1F: + e = 0xFF + elif e: + e += 0x70 + elif m: + e = 0x71 + while not m & 0x400: + m <<= 1 + e -= 1 + m &= ~0x400 + + channel_values[i] = (s << 31) | (e << 23) | (m << 13) + + channel_values = array.array("f", bytearray(channel_values)) + + # clamp the values to the [0.0, 1.0] bounds + channel_values = map(types.MethodType(max, 0.0), channel_values) + channel_values = map(types.MethodType(min, 1.0), channel_values) + # scale the values to the range we defined + channel_values = map(float(0x7FFF).__mul__, channel_values) + + # convert the floats to ints and slice them into + # the new array(leaving the alpha white) + unpacked_pixels = array.array("H", map(int, channel_values)) + + # replace any empty channels with their + for chan in blank_channels: + if chan == 0: + unpacked_pixels[::chan_ct] = array.array( + "H", b'\xFF\xFF' * (len(unpacked_pixels) // chan_ct) + ) + + return unpacked_pixels + + +def pack_r9g9b9e5(arby, unpacked, width, height, depth=1): + raise NotImplementedError("Not Implemented") + + +def pack_a16r16g16b16_f(arby, unpacked, width, height, depth=1): + raise NotImplementedError("Not Implemented") + + +arbytmap.format_defs.register_format( + FORMAT_R9G9B9E5_SHAREDEXP, 1, + depths=(9,9,9,5), offsets=(18,9,0,27), + unpacker=unpack_r9g9b9e5, packer=pack_r9g9b9e5 + ) + +arbytmap.format_defs.register_format( + FORMAT_A16R16G16B16F, 1, + depths=(16,16,16,16), offsets=(16,32,48,0), + unpacker=unpack_a16r16g16b16_f, packer=pack_a16r16g16b16_f + ) + +_R9G9B9E5_OFFSETS = array.array( + "B", arbytmap.format_defs.CHANNEL_OFFSETS[FORMAT_R9G9B9E5_SHAREDEXP] + ) +_R9G9B9E5_MASKS = array.array( + "H", arbytmap.format_defs.CHANNEL_MASKS[FORMAT_R9G9B9E5_SHAREDEXP] + ) +_UPSCALE_9BIT = array.array("H", list(range(2**9))) +_UPSCALE_5BIT = array.array("H", list(range(2**5))) +_UPSCALE_16BIT_ERASE = array.array("H", b'\x00\x00' * (2**16)) diff --git a/refinery/core.py b/refinery/core.py index c4555f3..ea190f2 100644 --- a/refinery/core.py +++ b/refinery/core.py @@ -23,7 +23,9 @@ from supyr_struct.util import path_normalize -from reclaimer.constants import GEN_1_HALO_ENGINES, GEN_2_ENGINES +from reclaimer.constants import GEN_1_HALO_GBX_ENGINES, GEN_1_HALO_XBOX_ENGINES,\ + GEN_1_STUBBS_ENGINES, GEN_1_SHADOWRUN_ENGINES, GEN_1_HALO_CUSTOM_ENGINES,\ + GEN_1_HALO_ENGINES, GEN_1_ENGINES, GEN_2_ENGINES, GEN_3_ENGINES from reclaimer.halo_script.hsc import get_hsc_data_block, HSC_IS_GLOBAL,\ get_h1_scenario_script_object_type_strings from reclaimer.hek import hardcoded_ce_tag_paths @@ -73,20 +75,16 @@ # Map the different map wrappers that decide how to read the map # and what tag defintions to use -# Note the loops after this that handle engines that use the same map format halo_map_wrappers_by_engine = { - "stubbs": StubbsMap, - "stubbspc": StubbsMap, - "stubbspc64bit": StubbsMap64Bit, - "shadowrun_proto": ShadowrunMap, - "halo1xbox": Halo1XboxMap, - "halo1xboxdemo": Halo1XboxMap, - "halo1anni": Halo1AnniMap, - "halo1yelo": Halo1YeloMap, - "halo1mcc": Halo1MccMap, - "halo2": Halo2Map, - "halo3beta": Halo3BetaMap, + **{engine: Halo1Map for engine in GEN_1_HALO_GBX_ENGINES}, + **{engine: Halo1XboxMap for engine in GEN_1_HALO_XBOX_ENGINES}, + **{engine: StubbsMap for engine in GEN_1_STUBBS_ENGINES}, + **{engine: ShadowrunMap for engine in GEN_1_SHADOWRUN_ENGINES}, + **{engine: Halo2Map for engine in GEN_2_ENGINES}, "halo3": Halo3Map, + # below are unsupported, and are simply here to allow + # loading the map and showing the tag index and header data + "halo3beta": Halo3BetaMap, "halo3odst": Halo3OdstMap, "haloreachbeta": HaloReachBetaMap, "haloreach": HaloReachMap, @@ -94,10 +92,13 @@ "halo4": Halo4Map, "halo5": Halo5Map, } -for name in GEN_1_HALO_ENGINES: - halo_map_wrappers_by_engine.setdefault(name, Halo1Map) -for name in GEN_2_ENGINES: - halo_map_wrappers_by_engine.setdefault(name, Halo2Map) +# override the above with a few engine-specific map wrappers +halo_map_wrappers_by_engine.update({ + "stubbspc64bit": StubbsMap64Bit, + "halo1anni": Halo1AnniMap, + "halo1yelo": Halo1YeloMap, + "halo1mcc": Halo1MccMap, + }) def get_halo_map_section_ends(halo_map): @@ -190,12 +191,15 @@ class RefineryCore: skip_seen_tags_during_queue_processing = True disable_safe_mode = False disable_tag_cleaning = False + treat_ce_map_as_yelo = False # deprotection settings fix_tag_classes = True fix_tag_index_offset = False use_minimum_priorities = True + disable_minimum_equal_priorities = False use_heuristics = True + root_dir_prefix = "" valid_tag_paths_are_accurate = True scrape_tag_paths_from_scripts = True limit_tag_path_lengths = True @@ -261,6 +265,8 @@ def map_loaded(self): return self.active_map is not None def active_maps(self): return self.maps_by_engine.get(ACTIVE_INDEX, {}) @property def active_map(self): return self.active_maps.get(ACTIVE_INDEX) + @property + def safe_mode(self): return not self.disable_safe_mode def enqueue(self, operation="extract_tags", **kwargs): if isinstance(operation, RefineryQueueItem): @@ -394,8 +400,7 @@ def save_map(self, save_path=None, map_name=ACTIVE_INDEX, raise KeyError("No map loaded and none provided.") elif halo_map.is_resource: raise TypeError("Cannot save resource maps.") - elif halo_map.engine not in ("halo1ce", "halo1yelo", "halo1mcc", - "halo1pc", "halo1vap"): + elif halo_map.engine not in GEN_1_HALO_GBX_ENGINES: raise TypeError("Cannot save this kind of map.") elif is_path_empty(save_path): save_path = halo_map.filepath @@ -409,7 +414,7 @@ def save_map(self, save_path=None, map_name=ACTIVE_INDEX, save_dir.mkdir(exist_ok=True, parents=True) try: - map_file = halo_map.map_data + map_file, out_file = halo_map.map_data, None if save_path == halo_map.filepath: out_file = map_file = halo_map.get_writable_map_data() else: @@ -511,7 +516,8 @@ def save_map(self, save_path=None, map_name=ACTIVE_INDEX, # set the size of the map in the header to 0 to fix a bug where # halo will leak file handles for very large maps. Also removes # the map size limitation so halo can load stupid big maps. - if halo_map.engine in ("halo1ce", "halo1yelo", "halo1vap"): + if (halo_map.engine != "halo1mcc" and + halo_map.engine in GEN_1_HALO_CUSTOM_ENGINES): map_header.decomp_len = 0 # write the map header so the calculate_ce_checksum can read it @@ -533,7 +539,7 @@ def save_map(self, save_path=None, map_name=ACTIVE_INDEX, halo_map.filepath = halo_map.decomp_filepath = save_path except Exception: - if halo_map.map_data is not out_file: + if out_file and halo_map.map_data is not out_file: out_file.close() raise @@ -565,7 +571,13 @@ def load_map(self, map_path, replace_if_same_name=False, **kw): map_name = map_header.map_name else: raise EngineDetectionError( - 'Could not determine map engine for "%s"' % map_path) + 'Could not determine map engine for "%s". Got "%s"' % + (map_path, engine) + ) + + engine_override = None + if self.treat_ce_map_as_yelo and engine == "halo1ce": + engine = engine_override = "halo1yelo" maps = self.maps_by_engine.setdefault(engine, {}) if not(maps.get(map_name) is None or replace_if_same_name): @@ -592,6 +604,10 @@ def load_map(self, map_path, replace_if_same_name=False, **kw): 'Could not determine map engine for "%s"' % map_path) new_map.load_map(map_path, **kw) + if engine_override: + # map will have corrected its engine based on what we + # passed in, and we'll have to override it again here + new_map.engine = engine_override if make_active: maps[ACTIVE_INDEX] = new_map @@ -618,7 +634,7 @@ def load_resource_maps(self, halo_map=None, maps_dir="", map_paths=(), **kw): def deprotect_all(self, **kw): for engine in self.maps_by_engine: - if engine not in ("halo1ce", "halo1yelo", "halo1pc", "halo1vap", "halo1mcc"): + if engine not in GEN_1_HALO_GBX_ENGINES: continue maps = self.maps_by_engine[engine] @@ -639,8 +655,7 @@ def deprotect(self, save_path=None, map_name=ACTIVE_INDEX, raise KeyError("No map loaded.") elif halo_map.is_resource: raise TypeError("Cannot deprotect resource maps.") - elif halo_map.engine not in ("halo1ce", "halo1yelo", "halo1mcc", - "halo1pc", "halo1vap"): + elif halo_map.engine not in GEN_1_HALO_GBX_ENGINES: raise TypeError("Cannot deprotect this kind of map.") if is_path_empty(save_path): @@ -648,8 +663,9 @@ def deprotect(self, save_path=None, map_name=ACTIVE_INDEX, save_path = Path(save_path) - do_printout = kw.pop("do_printout", self.do_printout) - print_errors = kw.pop("print_errors", self.print_errors) + do_printout = kw.pop("do_printout", self.do_printout) + print_changes = kw.pop("print_heuristic_name_changes", self.print_heuristic_name_changes) + print_errors = kw.pop("print_errors", self.print_errors) use_heuristics = kw.pop( "use_heuristics", self.use_heuristics) @@ -686,6 +702,7 @@ def deprotect(self, save_path=None, map_name=ACTIVE_INDEX, print(format_exc()) tag_path_handler = TagPathHandler(tag_index_array) + tag_path_handler.root_dir_prefix = self.root_dir_prefix tag_path_handler.set_path_by_priority( halo_map.tag_index.scenario_tag_id & 0xFFff, @@ -735,18 +752,23 @@ def deprotect(self, save_path=None, map_name=ACTIVE_INDEX, if valid_tag_paths_are_accurate: for b in tag_index_array: - if not b.path.lower().startswith("protected"): + for tag_path_part in b.path.lower().split("\\"): + if tag_path_part.startswith("protected"): + b = None + break + + if b is not None: tag_path_handler.set_overwritable(b.id, False) tag_path_handler.set_priority(b.id, VERY_HIGH_PRIORITY) try: tagc_names = self.detect_tag_collection_names(map_name, engine) - if do_printout: + if print_changes: print("Renaming tag collections\n" "tag_id\ttag_path\n") for tag_id, tag_path in tagc_names.items(): - if do_printout: + if print_changes: print(tag_id, tag_path, sep="\t") tag_path_handler.set_path_by_priority( tag_id, tag_path, VERY_HIGH_PRIORITY, True, False) @@ -760,7 +782,7 @@ def deprotect(self, save_path=None, map_name=ACTIVE_INDEX, try: self._script_scrape_deprotect( tag_path_handler, map_name, engine, - do_printout=do_printout, print_errors=print_errors) + do_printout=print_changes, print_errors=print_errors) except Exception: if not print_errors: raise @@ -768,6 +790,7 @@ def deprotect(self, save_path=None, map_name=ACTIVE_INDEX, if use_heuristics: try: + halo_map.safe_mode = self.safe_mode self._heuristics_deprotect( tag_path_handler, map_name, engine, do_printout=do_printout, print_errors=print_errors) @@ -775,10 +798,12 @@ def deprotect(self, save_path=None, map_name=ACTIVE_INDEX, if not print_errors: raise print(format_exc()) + finally: + halo_map.safe_mode = True if limit_tag_path_lengths: try: - tag_path_handler.shorten_paths(254, do_printout=do_printout, + tag_path_handler.shorten_paths(254, do_printout=print_changes, print_errors=print_errors) except Exception: if not print_errors: @@ -843,7 +868,9 @@ def detect_tag_collection_names(self, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX for tag_id, tag_cls in tag_classes_by_id.items(): if tag_cls != "yelo": continue - yelo_meta = halo_map.get_meta(tag_id) + yelo_meta = halo_map.get_meta( + tag_id, disable_safe_mode=self.disable_safe_mode + ) if not yelo_meta: continue if (yelo_meta.scenario_explicit_references.id & 0xFFff) != 0xFFff: @@ -881,8 +908,7 @@ def repair_tag_classes(self, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX): if not halo_map: return repaired - if halo_map.engine not in ("halo1ce", "halo1yelo", "halo1mcc", - "halo1pc", "halo1vap"): + if halo_map.engine not in GEN_1_HALO_GBX_ENGINES: raise TypeError('Cannot repair tag classes in "%s" maps' % halo_map.engine) @@ -891,18 +917,20 @@ def repair_tag_classes(self, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX): # locate the tags to start deprotecting with repair = {} for b in tag_index_array: + tag_ext = b.class_1.enum_name if b.id == halo_map.tag_index.scenario_tag_id: tag_cls = "scnr" - elif b.indexed or b.class_1.enum_name in ( + elif b.indexed or tag_ext in ( supyr_constants.INVALID, "NONE"): continue else: tag_cls = int_to_fourcc(b.class_1.data) - tag_id = b.id & 0xFFff - if tag_cls in ("scnr", "DeLa"): - repair[tag_id] = tag_cls - elif tag_cls == "matg" and b.path == "globals\\globals": + tag_id = b.id & 0xFFff + tag_path = "%s.%s" % (b.path, tag_ext) + if (tag_cls in ("scnr", "DeLa") or tag_path in hardcoded_ce_tag_paths\ + .HARDCODED_TAG_PATHS_BY_TYPE.get(tag_cls, ()) + ): repair[tag_id] = tag_cls # scan the tags that need repairing and repair them @@ -925,21 +953,23 @@ def repair_tag_classes(self, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX): if tag_cls != "sbsp": class_repair_functions[tag_cls]( tag_id, tag_index_array, halo_map.get_writable_map_data(), - halo_map.map_magic, next_repair, halo_map.engine, - not self.disable_safe_mode) + halo_map.map_magic, next_repair, halo_map.engine, self.safe_mode) # replace meta with the deprotected one if tag_cls == "matg": - halo_map.matg_meta = halo_map.get_meta(tag_id) + halo_map.matg_meta = halo_map.get_meta( + tag_id, disable_safe_mode=self.disable_safe_mode + ) elif tag_cls == "scnr": - halo_map.scnr_meta = halo_map.get_meta(tag_id) + halo_map.scnr_meta = halo_map.get_meta( + tag_id, disable_safe_mode=self.disable_safe_mode + ) elif tag_id in halo_map.bsp_headers: class_repair_functions[tag_cls]( halo_map.bsp_headers[tag_id].meta_pointer, tag_index_array, halo_map.get_writable_map_data(), halo_map.bsp_magics[tag_id] - halo_map.bsp_header_offsets[tag_id], - next_repair, halo_map.engine, halo_map.map_magic, - not self.disable_safe_mode) + next_repair, halo_map.engine, halo_map.map_magic, self.safe_mode) except Exception: print(format_exc()) @@ -1029,7 +1059,10 @@ def _script_scrape_deprotect(self, path_handler, map_name=ACTIVE_INDEX, if not halo_map: return - scnr_meta = halo_map.get_meta(halo_map.tag_index.scenario_tag_id) + scnr_meta = halo_map.get_meta( + halo_map.tag_index.scenario_tag_id, + disable_safe_mode=self.disable_safe_mode + ) if not scnr_meta: raise ValueError("Could not get scenario data for script scraping.") @@ -1067,13 +1100,18 @@ def _heuristics_deprotect(self, path_handler, map_name=ACTIVE_INDEX, ids_to_deprotect_by_class = {class_name: [] for class_name in ( "scenario", "globals", "hud_globals", "project_yellow", "vehicle", "actor_variant", "biped", "weapon", "equipment", "tag_collection", - "ui_widget_collection", "ui_widget_definition", "scenario_structure_bsp" + "ui_widget_collection", "ui_widget_definition", "scenario_structure_bsp", + "sound_looping", "sound_scenery" )} do_printout = kw.pop("do_printout", self.do_printout) print_errors = kw.pop("print_errors", self.print_errors) use_minimum_priorities = kw.pop( "use_minimum_priorities", self.use_minimum_priorities) + prioritize_model_names = kw.pop( + "prioritize_model_names", self.prioritize_model_names_over_message_strings) + use_minimum_equal_priorities = not kw.pop( + "disable_minimum_equal_priorities", self.disable_minimum_equal_priorities) shallow_ui_widget_nesting = kw.pop( "shallow_ui_widget_nesting", self.shallow_ui_widget_nesting) print_name_changes = do_printout and kw.pop( @@ -1121,7 +1159,9 @@ def _heuristics_deprotect(self, path_handler, map_name=ACTIVE_INDEX, "scenario_structure_bsp", "vehicle", "weapon", "equipment", "actor_variant", "biped", "ui_widget_collection", "ui_widget_definition", "hud_globals", - "project_yellow", "globals", "scenario", "tag_collection"): + "project_yellow", "globals", "scenario", "tag_collection", + "sound_looping", "sound_scenery" + ): if do_printout: print("Renaming %s tags" % tag_type) if print_name_changes: @@ -1137,7 +1177,9 @@ def _heuristics_deprotect(self, path_handler, map_name=ACTIVE_INDEX, tag_id, halo_map, path_handler, do_printout=print_name_changes, depth=depth, shallow_ui_widget_nesting=shallow_ui_widget_nesting, - use_minimum_priorities=use_minimum_priorities) + prioritize_model_names=prioritize_model_names, + use_minimum_priorities=use_minimum_priorities, + use_minimum_equal_priorities=use_minimum_equal_priorities) except Exception: if not print_errors: raise @@ -1424,10 +1466,8 @@ def extract_tags(self, tag_ids, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX, **kw raise RefineryError( "Cannot extract tags directly from Halo PC resource maps") - is_gen1 = ("halo1" in halo_map.engine or - "stubbs" in halo_map.engine or - "shadowrun" in halo_map.engine) - recursive &= is_gen1 and (extract_mode == "tags") + is_gen1 = halo_map.engine in GEN_1_ENGINES + recursive &= is_gen1 and (extract_mode == "tags") curr_tag_ids = set(tag_ids) while curr_tag_ids: @@ -1549,9 +1589,7 @@ def extract_tag(self, tag_id, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX, **kw): get_dependencies = False # it makes no sense to fill out # dependencies in data extraction - is_gen1 = ("halo1" in halo_map.engine or - "stubbs" in halo_map.engine or - "shadowrun" in halo_map.engine) + is_gen1 = halo_map.engine in GEN_1_ENGINES if tag_cls == "matg" and filepath.is_file() and is_gen1: # determine if we should overwrite the globals tag @@ -1660,8 +1698,8 @@ def extract_tag(self, tag_id, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX, **kw): f.write(data_bytes) except PermissionError: raise RefineryError( - "Refinery does not have permission to save here. " - "Running Refinery as admin could potentially fix this.") + "Refinery does not have permission to save here." + ) except FileNotFoundError: if not e_c.IS_WIN or len(str(filepath)) < 260: raise diff --git a/refinery/defs/config_def.py b/refinery/defs/config_def.py index 1ea863d..1053fb2 100644 --- a/refinery/defs/config_def.py +++ b/refinery/defs/config_def.py @@ -51,6 +51,8 @@ def get(): Bit("debug_mode"), Bit("do_printout"), Bit("autoload_resources"), + Bit("base_10_ids"), + Bit("base_10_offsets"), SIZE=4 ), Bool32("extraction_flags", @@ -64,14 +66,15 @@ def get(): "generate_uncomp_verts", "generate_comp_verts", Pad(3), - "use_tag_index_for_script_names", - "use_scenario_names_for_script_names", + {"NAME": "use_tag_index_for_script_names", "DEFAULT": True}, + {"NAME": "use_scenario_names_for_script_names", "DEFAULT": True}, "bitmap_extract_keep_alpha", Pad(1), # this flag has never been used # upper 16 bits "disable_safe_mode", "disable_tag_cleaning", + "treat_ce_map_as_yelo", ), Bool32("deprotection_flags", "fix_tag_classes", @@ -84,6 +87,8 @@ def get(): "shallow_ui_widget_nesting", "rename_cached_tags", "print_heuristic_name_changes", + "disable_minimum_equal_priorities", + "prioritize_model_names_over_message_strings", ), Bool32("preview_flags", "show_all_fields", @@ -139,7 +144,7 @@ def get(): paths = Array("paths", SUB_STRUCT=path, SIZE=".array_sizes.path_count", - NAME_MAP=("last_dir", "tagslist", "tags_dir", "data_dir"), + NAME_MAP=("last_dir", "tagslist_path", "tags_dir", "data_dir", "root_dir_prefix"), VISIBLE=False ) diff --git a/refinery/heuristic_deprotection/constants.py b/refinery/heuristic_deprotection/constants.py index ce44adc..03b803a 100644 --- a/refinery/heuristic_deprotection/constants.py +++ b/refinery/heuristic_deprotection/constants.py @@ -25,9 +25,19 @@ LOW_PRIORITY = 0.5 -INVALID_MODEL_NAMES = frozenset( - ("", "base", "unnamed", "blur", "unnamed base", - "def", "default", "damaged")) +INVALID_MODEL_NAMES = frozenset(( + "", + # common region names + "base", "unnamed", "unnamed_base", "hull", "body", + "def", "default", "damaged", "fp", + # common root node names + "frame", "pelvis", "root", "bone", + "gun", "hand", "grip", "pole", # <-- flag + # common modeling software names + "box", "cube", "sphere", "cylinder", "geosphere", + # permutations + "blur", "damaged", "primary_blur", "secondary_blur" + )) # directories inside the root_dir diff --git a/refinery/heuristic_deprotection/functions.py b/refinery/heuristic_deprotection/functions.py index 238575a..2d9972d 100644 --- a/refinery/heuristic_deprotection/functions.py +++ b/refinery/heuristic_deprotection/functions.py @@ -11,17 +11,13 @@ from refinery.heuristic_deprotection.constants import * from refinery.heuristic_deprotection.util import MinPriority, sanitize_name,\ sanitize_name_piece, get_tag_id, join_names, get_model_name,\ - get_sound_sub_dir_and_name, get_sound_looping_name, get_sound_scenery_name + get_sound_sub_dir_and_name, get_sound_looping_name, get_sound_scenery_name,\ + sanitize_model_or_sound_name from traceback import format_exc def heuristic_deprotect(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): - # create a copy of this set for each recursion level to prevent - # infinite recursion, but NOT prevent revisiting the - seen = kw.setdefault("seen", set()) - kw.setdefault("depth", INF) - priority = kw.get("priority") if tag_id is None: return INF @@ -29,15 +25,24 @@ def heuristic_deprotect(tag_id, halo_map, tag_path_handler, if tag_index_id not in range(len(halo_map.tag_index.tag_index)): return INF + # create a copy of this set for each recursion level to prevent + # infinite recursion, but NOT prevent revisiting the + seen = kw.setdefault("seen", set()) + kw.setdefault("depth", INF) + priority = kw.get("priority") + min_priority = MinPriority() curr_min_prio = tag_path_handler.get_priority_min(tag_index_id) if tag_index_id in seen or kw["depth"] < 0: # do nothing if tag already seen or already at max depth return curr_min_prio - elif kw.get("use_minimum_priorities") and (priority is not None and - curr_min_prio > priority): + elif not kw.get("use_minimum_priorities") or priority is None or curr_min_prio < priority: + pass + elif curr_min_prio > priority or (kw.get("use_minimum_equal_priorities") and + kw.get("return_on_equal_min_priority")): # do nothing if priority is lower than the lowest priority - # of any tag referenced by this tag or its dependencies + # of any tag referenced by this tag or its dependencies, or + # is equal and return_on_equal_min_priority is True return curr_min_prio kw.update(depth=kw["depth"] - 1, min_priority=min_priority) @@ -70,7 +75,7 @@ def rename_scnr(tag_id, halo_map, tag_path_handler, kw.update(halo_map=halo_map, root_dir=root_dir, tag_path_handler=tag_path_handler) - meta = halo_map.get_meta(tag_id) + meta = halo_map.get_meta(tag_id, reextract=True) if meta is None: return elif not name: @@ -275,7 +280,7 @@ def rename_scnr(tag_id, halo_map, tag_path_handler, kw['priority'] = LOW_PRIORITY sub_id = get_tag_id(b.reference) tag_cls = b.reference.tag_class.enum_name - tag_name = "protected %s" % sub_id + tag_name = "protected_%s" % sub_id if tag_cls == "model_animations": ref_sub_dir = cinematic_anims_dir @@ -330,7 +335,7 @@ def rename_matg(tag_id, halo_map, tag_path_handler, kwargs_no_priority = dict(kw) kwargs_no_priority.pop('priority') - meta = halo_map.get_meta(tag_id) + meta = halo_map.get_meta(tag_id, reextract=True) if meta is None: return elif not name: @@ -566,7 +571,7 @@ def rename_hudg(tag_id, halo_map, tag_path_handler, kw.update(halo_map=halo_map, root_dir=root_dir, tag_path_handler=tag_path_handler) - meta = halo_map.get_meta(tag_id) + meta = halo_map.get_meta(tag_id, reextract=True) if meta is None: return @@ -645,7 +650,7 @@ def rename_sbsp(tag_id, halo_map, tag_path_handler, for b in meta.collision_materials.STEPTREE: min_prio.val = heuristic_deprotect( get_tag_id(b.shader), sub_dir=coll_shdr_dir, - name="protected %s" % get_tag_id(b.shader), + name="protected_%s" % get_tag_id(b.shader), override=True, **kw) # lightmap materials(non-collidable) @@ -654,18 +659,18 @@ def rename_sbsp(tag_id, halo_map, tag_path_handler, for mat in lightmap.materials.STEPTREE: min_prio.val = heuristic_deprotect( get_tag_id(mat.shader), sub_dir=non_coll_shdr_dir, - name="protected %s" % get_tag_id(mat.shader), **kw) + name="protected_%s" % get_tag_id(mat.shader), **kw) # lens flares for b in meta.lens_flares.STEPTREE: min_prio.val = heuristic_deprotect( get_tag_id(b.shader), sub_dir=sub_dir + "lens flares\\", - name="protected %s" % get_tag_id(b.shader), **kw) + name="protected_%s" % get_tag_id(b.shader), **kw) # fog palettes for b in meta.fog_palettes.STEPTREE: min_prio.val = heuristic_deprotect(get_tag_id(b.fog), sub_dir=sub_dir + weather_dir, - name=sanitize_name_piece(b.name, "protected %s" % + name=sanitize_name_piece(b.name, "protected_%s" % get_tag_id(b.fog)), **kw) # weather palettes @@ -673,11 +678,11 @@ def rename_sbsp(tag_id, halo_map, tag_path_handler, min_prio.val = heuristic_deprotect(get_tag_id(b.particle_system), sub_dir=sub_dir + weather_dir, name=sanitize_name_piece( - b.name, "protected weather %s" % + b.name, "protected_weather_%s" % get_tag_id(b.particle_system)), **kw) min_prio.val = heuristic_deprotect(get_tag_id(b.wind), sub_dir=sub_dir + weather_dir, name=sanitize_name_piece( - b.name, "protected wind %s" % + b.name, "protected_wind_%s" % get_tag_id(b.wind)), **kw) # background sounds @@ -685,7 +690,7 @@ def rename_sbsp(tag_id, halo_map, tag_path_handler, min_prio.val = heuristic_deprotect(get_tag_id(b.background_sound), sub_dir=sub_dir + sounds_dir, name=sanitize_name_piece( - b.name, "protected bg sound %s" % + b.name, "protected_bg_sound_%s" % get_tag_id(b.background_sound)), **kw) # sound environments @@ -693,7 +698,7 @@ def rename_sbsp(tag_id, halo_map, tag_path_handler, min_prio.val = heuristic_deprotect(get_tag_id(b.sound_environment), sub_dir=snd_sound_env_dir, name=sanitize_name_piece( - b.name, "protected sound env %s" % + b.name, "protected_sound_env_%s" % get_tag_id(b.sound_environment)), **kw) @@ -718,7 +723,9 @@ def rename_sky_(tag_id, halo_map, tag_path_handler, tag_path_handler=tag_path_handler) kw.setdefault("priority", DEFAULT_PRIORITY) - min_prio.val = heuristic_deprotect(get_tag_id(meta.model), name=name, **kw) + for b in (meta.model, meta.animation_graph): + min_prio.val = heuristic_deprotect(get_tag_id(b), name=name, **kw) + for b in meta.lights.STEPTREE: light_name = sanitize_name(b.global_function_name) if not light_name: @@ -745,32 +752,40 @@ def rename_obje(tag_id, halo_map, tag_path_handler, replace(" source", "").replace(" src", "") i += 1 - if not name: + string_name = model_name = "" + if not name or name.startswith("protected"): if obje_type in ("weap", "eqip"): - name = tag_path_handler.get_item_string( + string_name = tag_path_handler.get_item_string( meta.item_attrs.message_index) eqip_attrs = getattr(meta, "eqip_attrs", None) if eqip_attrs and eqip_attrs.powerup_type.data in (1, 4): - name = eqip_attrs.powerup_type.enum_name.replace("_", " ") + string_name = eqip_attrs.powerup_type.enum_name.replace("_", " ") elif obje_type == "vehi": - name = tag_path_handler.get_icon_string( + string_name = tag_path_handler.get_icon_string( meta.obje_attrs.hud_text_message_index) - if name: - # up the priority if we could detect a name for - # this in the strings for the weapon or vehicles - kw.setdefault('priority', HIGH_PRIORITY) - else: - kw.setdefault('priority', MEDIUM_HIGH_PRIORITY) - name = tag_path_handler.get_basename(get_tag_id(obje_attrs.model)) - if not name or name.startswith("protected"): - name = "protected %s" % tag_id - if obje_type == "ssce": - name = get_sound_scenery_name(meta, halo_map, name) - else: - name = get_model_name( - halo_map, get_tag_id(obje_attrs.model), name) + model_tag_id = get_tag_id(obje_attrs.model) + model_name = tag_path_handler.get_basename(model_tag_id) + if not model_name or model_name.startswith("protected"): + if obje_type == "ssce": + model_name = get_sound_scenery_name(meta, halo_map, name) + else: + model_name = get_model_name( + halo_map=halo_map, tag_id=model_tag_id, name=name + ) + + if kw.get("prioritize_model_names") and model_name: + name = model_name + kw.setdefault('priority', HIGH_PRIORITY) + elif string_name: + # up the priority if we could detect a name for + # this in the strings for the weapon or vehicles + name = string_name + kw.setdefault('priority', HIGH_PRIORITY) + elif not name: + name = model_name or ("protected_%s" % tag_id) + kw.setdefault('priority', MEDIUM_HIGH_PRIORITY) if obje_type == "bipd": obje_dir = characters_dir @@ -802,13 +817,25 @@ def rename_obje(tag_id, halo_map, tag_path_handler, if not sub_dir: sub_dir = obje_dir - if sub_dir.lower().endswith(obje_dir): + orig_sub_dir = sub_dir + is_obje_sub_dir = sub_dir.lower().endswith(obje_dir) + if is_obje_sub_dir: sub_dir += name + "\\" min_prio.val = tag_path_handler.set_path_by_priority( tag_id, root_dir + sub_dir + name, kw.get('priority'), kw.get("override"), kw.get("do_printout")) + assigned_name = tag_path_handler.get_basename(tag_id) + if name != assigned_name: + name = assigned_name + if is_obje_sub_dir: + sub_dir = orig_sub_dir + name + "\\" + + min_prio.val = tag_path_handler.set_path_by_priority( + tag_id, root_dir + sub_dir + name, kw.get('priority'), True, + kw.get("do_printout")) + sub_dir = tag_path_handler.get_sub_dir(tag_id, root_dir) name = tag_path_handler.get_basename(tag_id) @@ -930,62 +957,62 @@ def rename_shdr(tag_id, halo_map, tag_path_handler, senv_attrs = meta.senv_attrs min_prio.val = heuristic_deprotect( get_tag_id(senv_attrs.diffuse.base_map), - name="senv %s diffuse" % name, **kw) + name="%s_senv_diffuse" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(senv_attrs.diffuse.primary_detail_map), - name="senv %s pri detail" % name, **kw) + name="%s_senv_pri_detail" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(senv_attrs.diffuse.secondary_detail_map), - name="senv %s sec detail" % name, **kw) + name="%s_senv_sec_detail" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(senv_attrs.diffuse.micro_detail_map), - name="senv %s micro detail" % name, **kw) + name="%s_senv_micro_detail" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(senv_attrs.bump_properties.map), - name="senv %s bump" % name, **kw) + name="%s_senv_bump" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(senv_attrs.self_illumination.map), - name="senv %s self illum" % name, **kw) + name="%s_senv_self_illum" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(senv_attrs.reflection.cube_map), - name="senv %s reflection" % name, **kw) + name="%s_senv_reflection" % name, **kw) senv_ext = getattr(getattr(senv_attrs, "os_shader_environment_ext", ()), "STEPTREE", ()) for b in senv_ext: min_prio.val = heuristic_deprotect(get_tag_id(b.specular_color_map), - name="senv %s spec color" % name, **kw) + name="%s_senv_spec_color" % name, **kw) elif shdr_type == "soso": soso_attrs = meta.soso_attrs min_prio.val = heuristic_deprotect( get_tag_id(soso_attrs.maps.diffuse_map), - name="soso %s diffuse" % name, **kw) + name="%s_soso_diffuse" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(soso_attrs.maps.multipurpose_map), - name="soso %s multi" % name, **kw) + name="%s_soso_multi" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(soso_attrs.maps.detail_map), - name="soso %s detail" % name, **kw) + name="%s_soso_detail" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(soso_attrs.reflection.cube_map), - name="soso %s reflection" % name, **kw) + name="%s_soso_reflection" % name, **kw) soso_ext = getattr(getattr(soso_attrs, "os_shader_model_ext", ()), "STEPTREE", ()) for b in soso_ext: min_prio.val = heuristic_deprotect( get_tag_id(b.specular_color_map), - name="soso %s spec color" % name, **kw) + name="%s_soso_spec_color" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(b.base_normal_map), - name="soso %s normal" % name, **kw) + name="%s_soso_normal" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(b.detail_normal_1_map), - name="soso %s normal detail 1" % name, **kw) + name="%s_soso_normal_detail_1" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(b.detail_normal_2_map), - name="soso %s normal detail 2" % name, **kw) + name="%s_soso_normal_detail_2" % name, **kw) elif shdr_type in ("sotr", "schi", "scex"): if shdr_type == "scex": @@ -1001,17 +1028,17 @@ def rename_shdr(tag_id, halo_map, tag_path_handler, for maps in maps_list: for map in maps.STEPTREE: - min_prio.val = heuristic_deprotect(get_tag_id(map.bitmap), name="%s %s" % - (shdr_type, name), **kw) + min_prio.val = heuristic_deprotect(get_tag_id(map.bitmap), name="%s_%s" % + (name, shdr_type), **kw) kw.update(sub_dir=sub_dir) i = 0 for extra_layer in extra_layers.STEPTREE: ex_layer_name = name if len(extra_layers.STEPTREE) == 1: - ex_layer_name += " ex " + ex_layer_name += "_ex_" else: - ex_layer_name += " ex %s" % i + ex_layer_name += "_ex_%s" % i min_prio.val = heuristic_deprotect( get_tag_id(extra_layer), name=ex_layer_name, **kw) i += 1 @@ -1020,40 +1047,40 @@ def rename_shdr(tag_id, halo_map, tag_path_handler, water_shader = meta.swat_attrs.water_shader min_prio.val = heuristic_deprotect( get_tag_id(water_shader.base_map), - name="swat %s base" % name, **kw) + name="%s_swat_base" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(water_shader.reflection_map), - name="swat %s reflection" % name, **kw) + name="%s_swat_reflection" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(water_shader.ripple_maps), - name="swat %s ripples" % name, **kw) + name="%s_swat_ripples" % name, **kw) elif shdr_type == "sgla": sgla_attrs = meta.sgla_attrs min_prio.val = heuristic_deprotect( get_tag_id(sgla_attrs.background_tint_properties.map), - name="sgla %s background tint" % name, **kw) + name="%s_sgla_background_tint" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(sgla_attrs.reflection_properties.map), - name="sgla %s reflection" % name, **kw) + name="%s_sgla_reflection" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(sgla_attrs.reflection_properties.bump_map), - name="sgla %s bump" % name, **kw) + name="%s_sgla_bump" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(sgla_attrs.diffuse_properties.map), - name="sgla %s diffuse" % name, **kw) + name="%s_sgla_diffuse" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(sgla_attrs.diffuse_properties.detail_map), - name="sgla %s diffuse detail" % name, **kw) + name="%s_sgla_diffuse_detail" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(sgla_attrs.specular_properties.map), - name="sgla %s specular" % name, **kw) + name="%s_sgla_specular" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(sgla_attrs.specular_properties.detail_map), - name="sgla %s specular detail" % name, **kw) + name="%s_sgla_specular_detail" % name, **kw) elif shdr_type == "smet": smet_attrs = meta.smet_attrs @@ -1065,10 +1092,10 @@ def rename_shdr(tag_id, halo_map, tag_path_handler, meter_name = func_name if not meter_name: - meter_name = name if "meter" in name else name + " meter" + meter_name = name if "meter" in name else name + " meter" min_prio.val = heuristic_deprotect(get_tag_id(smet_attrs.meter_shader.map), - name="smet %s" % name, **kw) + name="%s_smet" % name, **kw) elif shdr_type == "spla": spla_attrs = meta.spla_attrs @@ -1080,10 +1107,10 @@ def rename_shdr(tag_id, halo_map, tag_path_handler, min_prio.val = heuristic_deprotect( get_tag_id(spla_attrs.primary_noise_map.noise_map), - name="spla %s noise" % plasma_name, **kw) + name="%s_spla_noise" % plasma_name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(spla_attrs.primary_noise_map.noise_map), - name="spla %s noise sec" % plasma_name, **kw) + name="%s_spla_noise_sec" % plasma_name, **kw) def rename_item_attrs(meta, tag_id, halo_map, tag_path_handler, @@ -1093,7 +1120,8 @@ def rename_item_attrs(meta, tag_id, halo_map, tag_path_handler, return kw.update(halo_map=halo_map, root_dir=root_dir, - tag_path_handler=tag_path_handler) + tag_path_handler=tag_path_handler, + return_on_equal_min_priority=True) item_attrs = meta.item_attrs eqip_attrs = getattr(meta, "eqip_attrs", None) @@ -1233,7 +1261,8 @@ def rename_unit_attrs(meta, tag_id, halo_map, tag_path_handler, return kw.update(halo_map=halo_map, root_dir=root_dir, - tag_path_handler=tag_path_handler) + tag_path_handler=tag_path_handler, + return_on_equal_min_priority=True) unit_attrs = meta.unit_attrs bipd_attrs = getattr(meta, "bipd_attrs", None) @@ -1329,7 +1358,7 @@ def rename_unit_attrs(meta, tag_id, halo_map, tag_path_handler, min_prio.val = heuristic_deprotect(get_tag_id(b.melee_damage.damage_effect), sub_dir=boarding_dir, name=seat_name + " melee damage", **kw) - min_prio.val = heuristic_deprotect(get_tag_id(b.region_damage.damage_effect), + min_prio.val = heuristic_deprotect(get_tag_id(b.region_targeting.damage_effect), sub_dir=boarding_dir, name=seat_name + " region damage", **kw) @@ -1407,7 +1436,7 @@ def rename_actv(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) if not name: - name = "protected %s" % tag_id + name = "protected_%s" % tag_id if not sub_dir: sub_dir = characters_dir + name + "\\" @@ -1505,9 +1534,21 @@ def rename_mode(tag_id, halo_map, tag_path_handler, if meta is None: return + if not name or name.startswith("protected"): + name = tag_path_handler.get_basename(tag_id) + + model_name = get_model_name(meta=meta) + if (not name or name.startswith("protected")) and model_name: + name = model_name + kw.setdefault('priority', + MEDIUM_HIGH_PRIORITY if kw.get("prioritize_model_names") else + MEDIUM_PRIORITY + ) + min_prio.val = tag_path_handler.set_path_by_priority( tag_id, root_dir + sub_dir + name, kw.get('priority'), kw.get("override"), kw.get("do_printout")) + sub_dir = tag_path_handler.get_sub_dir(tag_id, root_dir) name = tag_path_handler.get_basename(tag_id) @@ -1515,7 +1556,6 @@ def rename_mode(tag_id, halo_map, tag_path_handler, sub_dir=sub_dir + shaders_dir, tag_path_handler=tag_path_handler) - shader_names = {} for i in range(len(meta.regions.STEPTREE)): region = meta.regions.STEPTREE[i] @@ -1526,7 +1566,9 @@ def rename_mode(tag_id, halo_map, tag_path_handler, for j in range(len(region.permutations.STEPTREE)): perm = region.permutations.STEPTREE[j] - shader_name = region_name + perm.name.replace(' ', '').strip("_") + shader_name = sanitize_model_or_sound_name( + region_name + "_" + perm.name + ) or model_name for lod in ("superhigh", "high", "medium", "low", "superlow"): geom_index = getattr(perm, lod + "_geometry_block") @@ -1538,17 +1580,15 @@ def rename_mode(tag_id, halo_map, tag_path_handler, for part_i in range(len(parts)): final_shader_name = shader_name if len(parts) > 1: - final_shader_name += " part%s" % part_i + final_shader_name += "_part%s" % part_i if lod != "superhigh": - final_shader_name += " " + lod + final_shader_name += "_" + lod shader_names.setdefault(parts[part_i].shader_index, final_shader_name) for i in range(len(meta.shaders.STEPTREE)): - shader_name = shader_names.get(i, "").replace("_", " ").\ - replace(".", "_").strip() min_prio.val = heuristic_deprotect(get_tag_id(meta.shaders.STEPTREE[i].shader), - name=shader_name.strip(), **kw) + name=shader_names.get(i, ""), **kw) def rename_coll(tag_id, halo_map, tag_path_handler, @@ -1733,7 +1773,8 @@ def rename_deca(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) kw.update(halo_map=halo_map, root_dir=root_dir, - tag_path_handler=tag_path_handler) + tag_path_handler=tag_path_handler, + return_on_equal_min_priority=True) meta = halo_map.get_meta(tag_id) if meta is None: @@ -1747,10 +1788,10 @@ def rename_deca(tag_id, halo_map, tag_path_handler, sub_dir = tag_path_handler.get_sub_dir(tag_id, root_dir) name = tag_path_handler.get_basename(tag_id) - min_prio.val = heuristic_deprotect(get_tag_id(meta.next_decal_in_chain), name=name + " next", - sub_dir=sub_dir, **kw) min_prio.val = heuristic_deprotect(get_tag_id(meta.shader.shader_map), name=name + " bitmaps", sub_dir=sub_dir + bitmaps_dir, **kw) + min_prio.val = heuristic_deprotect(get_tag_id(meta.next_decal_in_chain), name=name + " next", + sub_dir=sub_dir, **kw) def rename_ant_(tag_id, halo_map, tag_path_handler, @@ -1854,7 +1895,7 @@ def rename_DeLa(tag_id, halo_map, tag_path_handler, seen = kw.get("seen", set()) if not name: - name = "protected %s" % tag_id + name = "protected_%s" % tag_id kw["priority"] = (MEDIUM_HIGH_PRIORITY if kw.get("priority", 0) < MEDIUM_HIGH_PRIORITY @@ -1917,12 +1958,12 @@ def rename_lsnd(tag_id, halo_map, tag_path_handler, if not sub_dir: sub_dir = snd_music_dir - meta = halo_map.get_meta(tag_id) + meta = halo_map.get_meta(tag_id, recache=True) if meta is None: return - if not name: - name = get_sound_looping_name(meta, halo_map, "protected %s" % tag_id) + if not name or name.startswith("protected"): + name = get_sound_looping_name(meta, halo_map, "protected_%s" % tag_id) kw.update(halo_map=halo_map, root_dir=root_dir, tag_path_handler=tag_path_handler) @@ -1940,21 +1981,17 @@ def rename_lsnd(tag_id, halo_map, tag_path_handler, i = 0 tracks_dir = sub_dir + name + " tracks & details\\" for b in meta.tracks.STEPTREE: - min_prio.val = heuristic_deprotect( - get_tag_id(b.start), name="track %s start" % i, - sub_dir=tracks_dir, **kw) - min_prio.val = heuristic_deprotect( - get_tag_id(b.loop), name="track %s loop" % i, - sub_dir=tracks_dir, **kw) - min_prio.val = heuristic_deprotect( - get_tag_id(b.end), name="track %s end" % i, - sub_dir=tracks_dir, **kw) - min_prio.val = heuristic_deprotect( - get_tag_id(b.alternate_loop), name="track %s alt loop" % i, - sub_dir=tracks_dir, **kw) - min_prio.val = heuristic_deprotect( - get_tag_id(b.alternate_end), name="track %s alt end" % i, - sub_dir=tracks_dir, **kw) + for tag_ref, sub_name in (( + [b.start, "track %s start" % i], + [b.loop, "track %s loop" % i], + [b.end, "track %s end" % i], + [b.alternate_loop, "track %s alt loop" % i], + [b.alternate_end, "track %s alt end" % i], + )): + min_prio.val = heuristic_deprotect( + get_tag_id(tag_ref), name=sub_name, + sub_dir=tracks_dir, **kw) + i += 1 i = 0 @@ -2038,7 +2075,7 @@ def rename_soul(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) if not sub_dir: sub_dir = ui_shell_dir - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id meta = halo_map.get_meta(tag_id) if meta is None: @@ -2060,7 +2097,7 @@ def rename_soul(tag_id, halo_map, tag_path_handler, def rename_tagc(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id meta = halo_map.get_meta(tag_id) if meta is None: @@ -2118,7 +2155,7 @@ def rename_devc(tag_id, halo_map, tag_path_handler, def rename_ligh(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effect_lights_dir meta = halo_map.get_meta(tag_id) @@ -2147,7 +2184,7 @@ def rename_ligh(tag_id, halo_map, tag_path_handler, def rename_glw_(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effect_lights_dir meta = halo_map.get_meta(tag_id) @@ -2170,7 +2207,7 @@ def rename_glw_(tag_id, halo_map, tag_path_handler, def rename_lens(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effect_lens_flares_dir meta = halo_map.get_meta(tag_id) @@ -2194,7 +2231,7 @@ def rename_lens(tag_id, halo_map, tag_path_handler, def rename_mgs2(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effect_lens_flares_dir meta = halo_map.get_meta(tag_id) @@ -2217,7 +2254,7 @@ def rename_mgs2(tag_id, halo_map, tag_path_handler, def rename_elec(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effect_lens_flares_dir meta = halo_map.get_meta(tag_id) @@ -2240,7 +2277,7 @@ def rename_elec(tag_id, halo_map, tag_path_handler, def rename_part(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effect_particles_dir meta = halo_map.get_meta(tag_id) @@ -2279,7 +2316,7 @@ def rename_part(tag_id, halo_map, tag_path_handler, def rename_pctl(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effect_contrails_dir meta = halo_map.get_meta(tag_id) @@ -2321,7 +2358,7 @@ def rename_pctl(tag_id, halo_map, tag_path_handler, def rename_cont(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effect_contrails_dir meta = halo_map.get_meta(tag_id) @@ -2352,7 +2389,7 @@ def rename_cont(tag_id, halo_map, tag_path_handler, def rename_jpt_(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effects_dir + "damage\\" meta = halo_map.get_meta(tag_id) @@ -2375,7 +2412,7 @@ def rename_jpt_(tag_id, halo_map, tag_path_handler, def rename_effe(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effects_dir meta = halo_map.get_meta(tag_id) if meta is None: @@ -2389,7 +2426,8 @@ def rename_effe(tag_id, halo_map, tag_path_handler, name = tag_path_handler.get_basename(tag_id) kw.update(halo_map=halo_map, sub_dir=sub_dir + name + " events\\", - root_dir=root_dir, tag_path_handler=tag_path_handler) + root_dir=root_dir, tag_path_handler=tag_path_handler, + return_on_equal_min_priority=True) i = 0 event_name = "" @@ -2445,7 +2483,7 @@ def rename_hud_background(block, name, background_name="", **kw): def rename_grhi(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = ui_hud_dir meta = halo_map.get_meta(tag_id) if meta is None: @@ -2482,7 +2520,7 @@ def rename_grhi(tag_id, halo_map, tag_path_handler, def rename_unhi(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = ui_hud_dir meta = halo_map.get_meta(tag_id) if meta is None: @@ -2534,7 +2572,7 @@ def rename_unhi(tag_id, halo_map, tag_path_handler, def rename_wphi(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = ui_hud_dir meta = halo_map.get_meta(tag_id) if meta is None: @@ -2611,7 +2649,7 @@ def rename_ngpr(tag_id, halo_map, tag_path_handler, return meta_name = sanitize_name(meta.name) - if not name: name = meta_name if meta_name else "protected %s" % tag_id + if not name: name = meta_name if meta_name else "protected_%s" % tag_id if not sub_dir: sub_dir = ui_shell_dir + "netgame_prefs\\" kw.update(halo_map=halo_map, root_dir=root_dir, @@ -2665,7 +2703,7 @@ def rename_yelo(tag_id, halo_map, tag_path_handler, tag_path_handler=tag_path_handler) kw.setdefault('priority', INF) - meta = halo_map.get_meta(tag_id) + meta = halo_map.get_meta(tag_id, reextract=True) if meta is None: return @@ -2712,7 +2750,7 @@ def rename_gelo(tag_id, halo_map, tag_path_handler, tag_path_handler=tag_path_handler) kw.setdefault('priority', INF) - meta = halo_map.get_meta(tag_id) + meta = halo_map.get_meta(tag_id, reextract=True) if meta is None: return elif not name: @@ -2757,7 +2795,7 @@ def rename_gelc(tag_id, halo_map, tag_path_handler, tag_path_handler=tag_path_handler) kw.setdefault('priority', INF) - meta = halo_map.get_meta(tag_id) + meta = halo_map.get_meta(tag_id, reextract=True) if meta is None: return elif not name: @@ -2799,7 +2837,7 @@ def rename_efpc(tag_id, halo_map, tag_path_handler, def rename_efpg(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id sub_dir = "postprocess\\" meta = halo_map.get_meta(tag_id) if meta is None: @@ -2822,7 +2860,7 @@ def rename_efpg(tag_id, halo_map, tag_path_handler, def rename_shpg(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = "postprocess\\" meta = halo_map.get_meta(tag_id) if meta is None: @@ -2865,7 +2903,7 @@ def rename_sppg(tag_id, halo_map, tag_path_handler, def rename_efpp(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id meta = halo_map.get_meta(tag_id) if meta is not None: min_prio.val = tag_path_handler.set_path_by_priority( @@ -2877,7 +2915,7 @@ def rename_efpp(tag_id, halo_map, tag_path_handler, def rename_sily(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = ui_dir meta = halo_map.get_meta(tag_id) if meta is None: @@ -2913,7 +2951,7 @@ def rename_sily(tag_id, halo_map, tag_path_handler, def rename_unic(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = ui_dir meta = halo_map.get_meta(tag_id) if meta is None: @@ -2980,7 +3018,7 @@ def rename_keyframe_action(b, name, **kw): def rename_atvi(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id meta = halo_map.get_meta(tag_id) if meta is None: return @@ -3016,7 +3054,7 @@ def rename_atvi(tag_id, halo_map, tag_path_handler, def rename_atvo(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id meta = halo_map.get_meta(tag_id) if meta is None: return diff --git a/refinery/heuristic_deprotection/util.py b/refinery/heuristic_deprotection/util.py index 17f0053..2b7f9ad 100644 --- a/refinery/heuristic_deprotection/util.py +++ b/refinery/heuristic_deprotection/util.py @@ -7,10 +7,16 @@ # See LICENSE for more information. # +import re from refinery.util import sanitize_win32_path from refinery.heuristic_deprotection import constants as const +INVALID_NAME_CHAR_SUB = re.compile(r'[~\\/]') +REDUCE_ID_DIV_NAME_SUB = re.compile(r'[-_\s.<>{};:\'"|=+`!@#$%^&*()[\]]+') +REMOVE_PERM_AND_TRAIL_SUB = re.compile(r'^_|[\d_]+$') + + class MinPriority: _val = const.INF @property @@ -20,8 +26,9 @@ def val(self, new_val): self._val = min(new_val, self.val) def sanitize_name(name): - return str(sanitize_win32_path(name)).lower()\ - .replace("~", "").replace("\\", " ").strip() + return INVALID_NAME_CHAR_SUB.sub( + '', str(sanitize_win32_path(name)) + ).lower().strip() def sanitize_name_piece(name, default_name): @@ -47,40 +54,52 @@ def join_names(names, max_len=100): return name -def sanitize_tag_name(name, def_name): - name = sanitize_name(name).replace("_", " ").replace("-", " ").strip() - name = " ".join(s for s in name.split(" ") if s) - while name and name[-1] in "0123456789": - name = name[: -1].strip() - - if name not in const.INVALID_MODEL_NAMES: - return name - - return def_name - - -def get_model_name(halo_map, tag_id, model_name=""): - meta = halo_map.get_meta(tag_id) - if not(hasattr(meta, "regions") and meta.regions.STEPTREE): - return model_name - - names = [] - for region in meta.regions.STEPTREE: - for perm in region.permutations.STEPTREE: - names.append(sanitize_tag_name(perm.name, "")) +def sanitize_model_or_sound_name(name, def_name=''): + name = REMOVE_PERM_AND_TRAIL_SUB.sub('', + REDUCE_ID_DIV_NAME_SUB.sub('_', sanitize_name(name) + )) + return def_name if name in const.INVALID_MODEL_NAMES else name - name = "" if not names else names.pop() - if not names and name not in const.INVALID_MODEL_NAMES: - # just one single valid name was found amidst the permutations - return name - if len(meta.regions.STEPTREE) == 1: - # more than 1 perm name found. try to get a valid name from the regions - name = sanitize_tag_name(meta.regions.STEPTREE[0].name, "") - if name not in const.INVALID_MODEL_NAMES: - return name - - return model_name +def get_model_name(meta=None, halo_map=None, tag_id=None, name=""): + if meta is None: + meta = halo_map.get_meta(tag_id) + + regions = getattr(meta, "regions", None) + nodes = getattr(meta, "nodes", None) + node_name = perm_name = region_name = "" + if regions and regions.STEPTREE: + names = [] + for region in regions.STEPTREE: + for perm in region.permutations.STEPTREE: + perm_name = sanitize_model_or_sound_name(perm.name, perm_name) + if perm_name: break + + if len(regions.STEPTREE) == 1: + # couldn't find a valid name amidst the permutations, or there is + # more than 1 permutation. try to get a valid name from the regions + region_name = sanitize_model_or_sound_name(regions.STEPTREE[0].name, "") + + if nodes and nodes.STEPTREE: + for node in nodes.STEPTREE: + node_name = sanitize_model_or_sound_name( + node.name.split("frame", 1)[-1].split("bip01", 1)[-1]. + split("bone24", 1)[-1].strip(" \r\n\t-_") + ) + break + + if len(region_name) <= 2: region_name = "" + if len(perm_name) <= 2: perm_name = "" + if len(node_name) <= 2: node_name = "" + + #print("%s '%s' '%s' '%s' '%s'" % (tag_id, perm_name, region_name, node_name, name)) + + return ( + perm_name if perm_name not in const.INVALID_MODEL_NAMES else + region_name if region_name not in const.INVALID_MODEL_NAMES else + node_name if node_name not in const.INVALID_MODEL_NAMES else + name + ) def get_sound_sub_dir_and_name(snd_meta, sub_dir="", snd_name=""): @@ -113,7 +132,7 @@ def get_sound_sub_dir_and_name(snd_meta, sub_dir="", snd_name=""): for pr in snd_meta.pitch_ranges.STEPTREE: for perm in pr.permutations.STEPTREE: - perm_name = sanitize_tag_name(perm.name, "") + perm_name = sanitize_model_or_sound_name(perm.name) if perm_name: snd_name = perm_name break @@ -129,9 +148,11 @@ def get_sound_looping_name(meta, halo_map, def_name=""): # try and determine a name for this sound_looping from its sound tags for b in meta.tracks.STEPTREE: - for snd_id in (get_tag_id(b.start), get_tag_id(b.loop), get_tag_id(b.end)): + for tag_ref in ( + b.start, b.loop, b.end, b.alternate_loop, b.alternate_end + ): _, snd_name = get_sound_sub_dir_and_name( - halo_map.get_meta(snd_id, ignore_rawdata=True)) + halo_map.get_meta(get_tag_id(tag_ref), ignore_rawdata=True)) snd_name = snd_name.lower() if snd_name not in ("", "in", "start", "begin", "loops", "loop", "lp", "lps", "out", "stop", "end"): diff --git a/refinery/main.py b/refinery/main.py index b958c4c..c3c0831 100644 --- a/refinery/main.py +++ b/refinery/main.py @@ -13,7 +13,7 @@ import sys import webbrowser -from refinery.core import RefineryCore +from refinery.core import RefineryCore, GEN_1_HALO_GBX_ENGINES from pathlib import Path from time import time @@ -40,6 +40,11 @@ from refinery.windows.crc_window import RefineryChecksumEditorWindow from refinery.util import is_path_empty +try: + from refinery import arbytmap_ext +except Exception: + pass + from supyr_struct.defs import constants as supyr_constants from supyr_struct.field_types import FieldType @@ -144,7 +149,9 @@ def __init__(self, *args, **kwargs): self._active_engine_name = tk.StringVar(self) self._autoload_resources = tk.IntVar(self, 1) - self._do_printout = tk.IntVar(self, 1) + self._do_printout = tk.IntVar(self, 1) + self._base_10_ids = tk.IntVar(self, 1) + self._base_10_offsets = tk.IntVar(self, 1) self._force_lower_case_paths = tk.IntVar(self, 1) self._extract_yelo_cheape = tk.IntVar(self) @@ -160,6 +167,7 @@ def __init__(self, *args, **kwargs): self._skip_seen_tags_during_queue_processing = tk.IntVar(self, 1) self._disable_safe_mode = tk.IntVar(self, 0) self._disable_tag_cleaning = tk.IntVar(self, 0) + self._treat_ce_map_as_yelo = tk.IntVar(self, 0) self._globals_overwrite_mode = tk.IntVar(self, 0) self._bitmap_extract_format = tk.StringVar(self) @@ -167,7 +175,10 @@ def __init__(self, *args, **kwargs): self._fix_tag_classes = tk.IntVar(self, 1) self._fix_tag_index_offset = tk.IntVar(self) self._use_minimum_priorities = tk.IntVar(self, 1) + self._disable_minimum_equal_priorities = tk.IntVar(self, 1) + self._prioritize_model_names_over_message_strings = tk.IntVar(self, 1) self._use_heuristics = tk.IntVar(self, 1) + self._root_dir_prefix = tk.StringVar(self) self._valid_tag_paths_are_accurate = tk.IntVar(self, 1) self._scrape_tag_paths_from_scripts = tk.IntVar(self, 1) self._limit_tag_path_lengths = tk.IntVar(self, 1) @@ -188,6 +199,8 @@ def __init__(self, *args, **kwargs): do_printout=self._do_printout, autoload_resources=self._autoload_resources, + base_10_ids=self._base_10_ids, + base_10_offsets=self._base_10_offsets, force_lower_case_paths=self._force_lower_case_paths, extract_yelo_cheape=self._extract_yelo_cheape, @@ -203,6 +216,7 @@ def __init__(self, *args, **kwargs): skip_seen_tags_during_queue_processing=self._skip_seen_tags_during_queue_processing, disable_safe_mode=self._disable_safe_mode, disable_tag_cleaning=self._disable_tag_cleaning, + treat_ce_map_as_yelo=self._treat_ce_map_as_yelo, globals_overwrite_mode=self._globals_overwrite_mode, bitmap_extract_format=self._bitmap_extract_format, @@ -210,7 +224,10 @@ def __init__(self, *args, **kwargs): fix_tag_classes=self._fix_tag_classes, fix_tag_index_offset=self._fix_tag_index_offset, use_minimum_priorities=self._use_minimum_priorities, + disable_minimum_equal_priorities=self._disable_minimum_equal_priorities, + prioritize_model_names_over_message_strings=self._prioritize_model_names_over_message_strings, use_heuristics=self._use_heuristics, + root_dir_prefix=self._root_dir_prefix, valid_tag_paths_are_accurate=self._valid_tag_paths_are_accurate, scrape_tag_paths_from_scripts=self._scrape_tag_paths_from_scripts, limit_tag_path_lengths=self._limit_tag_path_lengths, @@ -437,6 +454,13 @@ def last_dir(self, new_val): new_val = Path(new_val) self._last_dir = new_val + @property + def root_dir_prefix(self): + return self._root_dir_prefix.get() + @root_dir_prefix.setter + def root_dir_prefix(self, new_val): + self._root_dir_prefix.set(new_val) + @property def running(self): return self._running @@ -532,14 +556,16 @@ def apply_config(self): paths = self.config_file.data.paths app_window = self.config_file.data.app_window fonts = self.config_file.data.fonts - - self.tagslist_path = paths.tagslist.path - self.tags_dir = paths.tags_dir.path - self.data_dir = paths.data_dir.path - self.last_dir = paths.last_dir.path + for name in ("root_dir_prefix", "tagslist_path", + "tags_dir", "data_dir", "last_dir"): + try: + setattr(self, name, getattr(paths, name).path) + except (AttributeError, IndexError): + pass self._display_mode = header.flags.display_mode.enum_name - for name in ("do_printout", "autoload_resources"): + for name in ("do_printout", "autoload_resources", + "base_10_ids", "base_10_offsets"): setattr(self, name, bool(getattr(header.flags, name))) for attr_name in header.preview_flags.NAME_MAP: @@ -598,13 +624,15 @@ def update_config(self, config_file=None): if len(paths.NAME_MAP) > len(paths): paths.extend(len(paths.NAME_MAP) - len(paths)) - paths.tagslist.path = "" if is_path_empty(self.tagslist_path) else str(self.tagslist_path) + paths.root_dir_prefix.path = "" if is_path_empty(self.root_dir_prefix) else str(self.root_dir_prefix) + paths.tagslist_path.path = "" if is_path_empty(self.tagslist_path) else str(self.tagslist_path) paths.tags_dir.path = "" if is_path_empty(self.tags_dir) else str(self.tags_dir) paths.data_dir.path = "" if is_path_empty(self.data_dir) else str(self.data_dir) paths.last_dir.path = "" if is_path_empty(self.last_dir) else str(self.last_dir) header.flags.display_mode.set_to(self._display_mode) - for attr_name in ("do_printout", "autoload_resources"): + for attr_name in ("do_printout", "autoload_resources", + "base_10_ids", "base_10_offsets"): setattr(header.flags, attr_name, getattr(self, attr_name)) for attr_name in header.preview_flags.NAME_MAP: @@ -997,11 +1025,13 @@ def unload_maps(self, map_type=False, engines_to_unload=(ACTIVE_INDEX, ), def load_map(self, map_path, make_active=True, ask_close_open=False, **kw): autoload_resources = kw.pop("autoload_resources", self.autoload_resources) + unlink_mismatched = kw.pop("unlink_mismatched_resources", True) new_map = prev_active_engine = prev_active_map = None try: new_map = RefineryCore.load_map( self, map_path, not ask_close_open, make_active=False, - autoload_resources=False, decompress_overwrite=True) + autoload_resources=False, decompress_overwrite=True, + unlink_mismatched_resources=unlink_mismatched) except MapAlreadyLoadedError: if not(ask_close_open and messagebox.askyesno( "A map with that name is already loaded!", @@ -1015,7 +1045,9 @@ def load_map(self, map_path, make_active=True, ask_close_open=False, **kw): prev_active_map = self.active_map_name new_map = RefineryCore.load_map( self, map_path, True, make_active=False, - autoload_resources=False) + autoload_resources=False, + unlink_mismatched_resources=unlink_mismatched + ) if (new_map.engine == prev_active_engine and new_map.map_name == prev_active_map): @@ -1171,8 +1203,7 @@ def deprotect(self, save_path=None, map_name=ACTIVE_INDEX, elif halo_map.is_resource: print("Cannot deprotect resource maps.") return - elif halo_map.engine not in ("halo1ce", "halo1yelo", - "halo1pc", "halo1vap"): + elif halo_map.engine not in GEN_1_HALO_GBX_ENGINES: print("Cannot deprotect this kind of map.") return @@ -1272,8 +1303,7 @@ def save_map_as(self, e=None): elif halo_map.is_resource: print("Cannot save resource maps.") return Path("") - elif halo_map.engine not in ("halo1ce", "halo1yelo", "halo1mcc", - "halo1pc", "halo1vap"): + elif halo_map.engine not in GEN_1_HALO_GBX_ENGINES: print("Cannot save this kind of map.") return Path("") @@ -1317,8 +1347,7 @@ def save_map(self, save_path=None, map_name=ACTIVE_INDEX, elif halo_map.is_resource: print("Cannot save resource maps.") return Path("") - elif halo_map.engine not in ("halo1ce", "halo1yelo", "halo1mcc", - "halo1pc", "halo1vap"): + elif halo_map.engine not in GEN_1_HALO_GBX_ENGINES: print("Cannot save this kind of map.") return Path("") elif save_path is None or is_path_empty(save_path): @@ -1365,7 +1394,10 @@ def save_map(self, save_path=None, map_name=ACTIVE_INDEX, if reload_window and not is_path_empty(save_path): print("Reloading map to apply changes...") - self.load_map(save_path, make_active=True, autoload_resources=False) + self.load_map( + save_path, make_active=True, autoload_resources=False, + unlink_mismatched_resources=False + ) return save_path diff --git a/refinery/tag_index/tag_path_handler.py b/refinery/tag_index/tag_path_handler.py index 792d12c..06aa72b 100644 --- a/refinery/tag_index/tag_path_handler.py +++ b/refinery/tag_index/tag_path_handler.py @@ -8,16 +8,38 @@ # import os +import re from pathlib import PureWindowsPath from refinery.constants import INF from refinery.util import sanitize_win32_path, get_unique_name -from supyr_struct.util import str_to_identifier +from supyr_struct.util import str_to_identifier as orig_str_to_identifier from queue import LifoQueue, Empty as EmptyQueueException +BALL_MATCH_SUB = re.compile('\b(?:ball|skull|oddball)\b') +FLAG_MATCH_SUB = re.compile('\b(?:flag)\b') + +def str_to_identifier(string): + ''' + This version of str_to_identifier won't die if it fails + to sanitize the string to something useful, but will + instead return a single underscore. Useful when dealing + with potentially completely invalid identifier strings. + ''' + name = "" + try: + name = orig_str_to_identifier(string) + except (AssertionError, TypeError): + pass + + return name or "_" + class TagPathHandler(): + # _root_dir_prefix is a directory name to prefix to + # all tags renamed through priority(i.e. deprotection) + _root_dir_prefix = "" _path_map = () _index_map = () _priorities = () @@ -47,6 +69,14 @@ def __init__(self, tag_index_array, **kwargs): self._overwritables[i] = False if ref.indexed else True i += 1 + @property + def root_dir_prefix(self): + return self._root_dir_prefix or "" + @root_dir_prefix.setter + def root_dir_prefix(self, value): + value = str(PureWindowsPath(value)).lower().rstrip("\\") + self._root_dir_prefix = value + "\\" if value else "" + @property def def_priority(self): return self._def_priority @@ -63,14 +93,20 @@ def get_icon_string(self, index): def set_item_strings(self, strings_body): new_strings = [] + for b in strings_body.strings.STEPTREE: string = str_to_identifier(b.data.lower()).\ replace("_", " ").replace(" d ", ' ').\ replace("picked up an ", '').replace("picked up a ", '').\ - replace("picked up the ", '').replace("picked up ", '') + replace("picked up the ", '').replace("picked up ", '').\ + replace("powerup", '') if string == "need a string entry here": string = "" + elif FLAG_MATCH_SUB.match(string): + string = "flag" + elif BALL_MATCH_SUB.match(string): + string = "ball" elif " for " in string: string = string.split(" for ")[-1] + " ammo" elif string.startswith("for "): @@ -164,32 +200,38 @@ def get_will_overwrite(self, index, priority, override=False): return False return True - def set_path(self, index, new_path_no_ext, do_printout=False, ensure_unique_name=True): + def set_path(self, index, new_path_no_ext, do_printout=False, + ensure_unique_name=True, ensure_root_prefixed=False): if index is None: return index &= 0xFFff + new_path_no_ext = (new_path_no_ext or "").lower() + if ensure_root_prefixed and not new_path_no_ext.startswith(self.root_dir_prefix): + new_path_no_ext = self.root_dir_prefix + new_path_no_ext + tag_ref = self.get_index_ref(index) if tag_ref is None: return elif not new_path_no_ext or new_path_no_ext[-1] == "\\": - new_path_no_ext += "protected %s" % index + new_path_no_ext += "protected_%s" % index ext = "." + tag_ref.class_1.enum_name.lower() - new_path_no_ext = str(sanitize_win32_path(new_path_no_ext.lower())).strip() + new_path_no_ext = str(sanitize_win32_path(new_path_no_ext)).strip().lower() if ensure_unique_name and self._path_map.get(new_path_no_ext + ext) not in (None, index): - path_pieces = PureWindowsPath(new_path_no_ext).parts - path_basename = path_pieces[-1] + path_pieces = list(PureWindowsPath(new_path_no_ext).parts) try: - path_basename_pieces = path_basename.split("#") - if len(path_basename_pieces) > 1: - path_basename = "#".join(path_basename_pieces[: -1]) + # remove the digit suffix, but keep it if there's + # more than digits on the end(its not one we made) + basename_pieces = path_pieces[-1].rsplit("#", 1) + if not basename_pieces[-1].strip("0123456789"): + path_pieces[-1] = basename_pieces[0] except Exception: pass - path_pieces = tuple(path_pieces[: -1]) + (path_basename, ) new_path_no_ext = get_unique_name( - self._path_map, str(PureWindowsPath(*path_pieces)), ext, index) + self._path_map, str(PureWindowsPath(*path_pieces)), ext, index + ) old_path = tag_ref.path.lower() + ext new_path = new_path_no_ext + ext @@ -224,7 +266,7 @@ def set_path_by_priority(self, index, new_path_no_ext, priority=None, if not self.get_will_overwrite(index, priority, override): return self.get_priority(index) - self.set_path(index, new_path_no_ext, do_printout) + self.set_path(index, new_path_no_ext, do_printout, ensure_root_prefixed=True) self.set_priority(index, priority) return priority diff --git a/refinery/util.py b/refinery/util.py index 0cbbd57..a334b21 100644 --- a/refinery/util.py +++ b/refinery/util.py @@ -117,10 +117,17 @@ def sanitize_win32_path(name): def get_unique_name(collection, name="", ext="", curr_value=object()): - final_name = name - i = 1 - while collection.get(final_name + ext) not in (None, curr_value): - final_name = "%s #%s" % (name, i) + name_parts = name.split("\\") + basename = name_parts[-1] + # if the folder name is the same as the file name, update it too + update_folder_name = len(name_parts) > 1 and name_parts[-2] == basename + + i = 0 + while collection.get("\\".join(name_parts) + ext) not in (None, curr_value): i += 1 + new_basename = "%s#%s" % (basename, i) + name_parts[-1] = new_basename + if update_folder_name: + name_parts[-2] = new_basename - return final_name + return "\\".join(name_parts) diff --git a/refinery/widgets/explorer_hierarchy_tree.py b/refinery/widgets/explorer_hierarchy_tree.py index 831dac0..a1770cc 100644 --- a/refinery/widgets/explorer_hierarchy_tree.py +++ b/refinery/widgets/explorer_hierarchy_tree.py @@ -58,6 +58,13 @@ def __init__(self, *args, **kwargs): self.tags_tree.bind('', self.activate_item) self.tags_tree.bind('', self.set_sort_mode) + @property + def id_format_str(self): + return '%d' if getattr(self.app_root, 'base_10_ids', False) else '%04X' + @property + def pointer_format_str(self): + return '%d' if getattr(self.app_root, 'base_10_offsets', False) else '%08X' + def set_sort_mode(self, event): if self.tags_tree.identify_region(event.x, event.y) != "heading": return @@ -455,12 +462,12 @@ def add_tag_index_ref(self, parent_dir_parts, tag_path, tag_index_ref): if tag_index_ref.indexed and pointer_converter and not is_h1_rsrc_map: pointer = "not in map" elif pointer_converter is not None: - pointer = '%08X' % pointer_converter.v_ptr_to_f_ptr( + pointer = self.pointer_format_str % pointer_converter.v_ptr_to_f_ptr( tag_index_ref.meta_offset) else: pointer = 0 - meta_offset = '%08X' % tag_index_ref.meta_offset + meta_offset = self.pointer_format_str % tag_index_ref.meta_offset try: cls1 = cls2 = cls3 = "" @@ -478,7 +485,7 @@ def add_tag_index_ref(self, parent_dir_parts, tag_path, tag_index_ref): parent_iid, 'end', iid=str(tag_id), tags=("item", ), text=tag_path, values=(cls1, cls2, cls3, - meta_offset, pointer, '%04X' % tag_id)) + meta_offset, pointer, self.id_format_str % tag_id)) self.tree_id_to_index_ref[tag_id] = tag_index_ref except Exception: print(format_exc()) diff --git a/refinery/windows/settings_window.py b/refinery/windows/settings_window.py index c7623db..c9585a6 100644 --- a/refinery/windows/settings_window.py +++ b/refinery/windows/settings_window.py @@ -78,9 +78,10 @@ def __init__(self, *args, **kwargs): "generate_comp_verts", "generate_uncomp_verts", "force_lower_case_paths", "fix_tag_classes", "autoload_resources", "extract_yelo_cheape", + "base_10_ids", "base_10_offsets", "use_minimum_priorities", "use_heuristics", "rename_cached_tags", "show_all_fields", - "show_structure_meta", + "show_structure_meta", "disable_minimum_equal_priorities", "edit_all_fields", "allow_corrupt", "valid_tag_paths_are_accurate", "limit_tag_path_lengths", "scrape_tag_paths_from_scripts", "shallow_ui_widget_nesting", @@ -88,11 +89,15 @@ def __init__(self, *args, **kwargs): "do_printout", "print_heuristic_name_changes", "use_scenario_names_for_script_names", "skip_seen_tags_during_queue_processing", - "disable_safe_mode", "disable_tag_cleaning",): + "prioritize_model_names_over_message_strings", + "disable_safe_mode", "disable_tag_cleaning", + "treat_ce_map_as_yelo",): object.__setattr__(self, attr, settings.get(attr, tk.IntVar(self))) for attr in ("bitmap_extract_format", "globals_overwrite_mode", - "tags_dir", "data_dir", "tagslist_path"): + "tags_dir", "data_dir", "tagslist_path", + "root_dir_prefix" + ): object.__setattr__(self, attr, settings.get(attr, tk.StringVar(self))) @@ -119,6 +124,12 @@ def __init__(self, *args, **kwargs): self.tags_list_frame, text="Browse", command=self.tags_list_browse, width=6) + # root directory prefix + self.root_dir_frame = tk.LabelFrame( + self.heuristics_frame, text="Directory to prefix to all deprotected tag names") + self.root_dir_entry = tk.Entry( + self.root_dir_frame, textvariable=self.root_dir_prefix) + self.rename_scnr_dups_cbtn = tk.Checkbutton( self.tag_fixup_frame, text=( @@ -132,15 +143,26 @@ def __init__(self, *args, **kwargs): self.tag_fixup_frame, text="Generate uncompressed lightmap vertices", variable=self.generate_uncomp_verts) - self.dont_touch_frame = tk.LabelFrame( - self.tag_fixup_frame, - text="ONLY CHECK THESE IF YOU ARE NOT DEALING WITH PROTECTED MAPS") + self.dont_touch_frame = tk.LabelFrame(self.tag_fixup_frame) + self.dont_touch_label = tk.Label( + self.dont_touch_frame, justify="left", text=( + "\nThese settings change how Refinery reads tags from maps.\n" + "Use caution when changing these, as these settings can make Refinery\n" + "crash or freeze when reading from some heavily protected/corrupted maps.\n" + "They are safe to disable if you are certain the map isn't protected.\n" + )) self.disable_safe_mode_cbtn = tk.Checkbutton( self.dont_touch_frame, variable=self.disable_safe_mode, justify="left", - text="Disable safe-mode") + text="Disable bounds-checking while reading tags(i.e. safe-mode)") self.disable_tag_cleaning_cbtn = tk.Checkbutton( self.dont_touch_frame, variable=self.disable_tag_cleaning, justify="left", - text="Disable cleaning errors from tags when reading them.") + text=("Disable cleaning unused/invalid data from tags.\n" + "Be aware that tag cleaning can cause unused tags to not be\n" + "found when running deprotection or recursive extraction." + )) + self.treat_ce_map_as_yelo_cbtn = tk.Checkbutton( + self.dont_touch_frame, variable=self.treat_ce_map_as_yelo, justify="left", + text="Always use Open Sauce tag structures for CE maps(requires reload of map)") self.overwrite_cbtn = tk.Checkbutton( @@ -228,18 +250,24 @@ def __init__(self, *args, **kwargs): variable=self.fix_tag_index_offset, justify='left') + self.print_heuristic_progress_cbtn = tk.Checkbutton( + self.heuristics_frame, text=("Print heuristic tag path changes"), + variable=self.print_heuristic_name_changes, justify='left') self.valid_tag_paths_are_accurate_cbtn = tk.Checkbutton( self.heuristics_frame, text="Do not rename non-protected tag paths", variable=self.valid_tag_paths_are_accurate) self.shallow_ui_widget_nesting_cbtn = tk.Checkbutton( self.heuristics_frame, text="Use shallow ui_widget_definition nesting", variable=self.shallow_ui_widget_nesting) + self.prioritize_model_names_cbtn = tk.Checkbutton( + self.heuristics_frame, text="Prioritize weapon/vehicle model names over message strings", + variable=self.prioritize_model_names_over_message_strings) self.use_fast_heuristics_cbtn = tk.Checkbutton( self.heuristics_frame, text="Use fast heuristics", variable=self.use_minimum_priorities) - self.print_heuristic_progress_cbtn = tk.Checkbutton( - self.heuristics_frame, text=("Print heuristic tag path changes"), - variable=self.print_heuristic_name_changes, justify='left') + self.use_fastest_heuristics_cbtn = tk.Checkbutton( + self.heuristics_frame, text="Use fastest heuristics(experimental)", + variable=self.disable_minimum_equal_priorities, onvalue=0, offvalue=1) font_frame_widgets = {} @@ -313,7 +341,12 @@ def __init__(self, *args, **kwargs): self.allow_corrupt_cbtn = tk.Checkbutton( self.other_frame, variable=self.allow_corrupt, text="Allow previewing corrupt tags") - + self.base_10_ids_cbtn = tk.Checkbutton( + self.other_frame, variable=self.base_10_ids, + text="Show tag ids in base10 instead of base16") + self.base_10_offsets_cbtn = tk.Checkbutton( + self.other_frame, variable=self.base_10_offsets, + text="Show tag pointers in base10 instead of base16") # pack everything self.tabs.pack(fill="both", expand=True) @@ -341,7 +374,8 @@ def __init__(self, *args, **kwargs): w.pack(padx=4, anchor='w') self.dont_touch_frame.pack(padx=4, anchor='w', expand=True, fill="both") - for w in (self.disable_safe_mode_cbtn, self.disable_tag_cleaning_cbtn): + for w in (self.dont_touch_label, self.disable_safe_mode_cbtn, + self.disable_tag_cleaning_cbtn, self.treat_ce_map_as_yelo_cbtn): w.pack(padx=4, anchor='w') for w in (self.fix_tag_classes_cbtn, self.use_heuristics_cbtn, @@ -354,13 +388,17 @@ def __init__(self, *args, **kwargs): for w in (self.print_heuristic_progress_cbtn, self.valid_tag_paths_are_accurate_cbtn, self.shallow_ui_widget_nesting_cbtn, + self.prioritize_model_names_cbtn, self.use_fast_heuristics_cbtn, + self.use_fastest_heuristics_cbtn, + self.root_dir_frame ): w.pack(padx=4, anchor='w') for w in (self.autoload_resources_cbtn, self.extract_yelo_cheape_cbtn, self.show_all_fields_cbtn, self.show_structure_meta_cbtn, self.edit_all_fields_cbtn, self.allow_corrupt_cbtn, + self.base_10_ids_cbtn, self.base_10_offsets_cbtn, ): w.pack(padx=4, anchor='w') @@ -375,9 +413,11 @@ def __init__(self, *args, **kwargs): for w1, w2 in ((self.tags_dir_entry, self.tags_dir_browse_button), (self.data_dir_entry, self.data_dir_browse_button), - (self.tags_list_entry, self.browse_tags_list_button)): + (self.tags_list_entry, self.browse_tags_list_button), + (self.root_dir_entry, None)): w1.pack(padx=(4, 0), pady=2, side='left', expand=True, fill='x') - w2.pack(padx=(0, 4), pady=2, side='left') + if w2: + w2.pack(padx=(0, 4), pady=2, side='left') # make the window not show up on the start bar self.transient(self.master) From 5e5c4b7fe7abced97b5488eb190173ac348d0ee5 Mon Sep 17 00:00:00 2001 From: Steven Garcia Date: Tue, 20 Feb 2024 07:34:05 -0600 Subject: [PATCH 11/16] bump version --- refinery/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/refinery/__init__.py b/refinery/__init__.py index 3153a00..cdcd5ab 100644 --- a/refinery/__init__.py +++ b/refinery/__init__.py @@ -12,8 +12,8 @@ # ############## __author__ = "Sigmmma" # YYYY.MM.DD -__date__ = "2024.02.10" -__version__ = (2, 7, 4) +__date__ = "2024.02.20" +__version__ = (2, 8, 0) __website__ = "https://github.com/Sigmmma/refinery" __all__ = ( 'defs', 'heuristic_deprotection', 'repl', 'tag_index', 'widgets', 'windows', From 6ecc6378f6038af0bc972f519bc9ddd61a6ef41e Mon Sep 17 00:00:00 2001 From: Steven Garcia Date: Mon, 26 Feb 2024 01:31:53 -0600 Subject: [PATCH 12/16] Add more logging --- refinery/core.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/refinery/core.py b/refinery/core.py index ea190f2..cff2263 100644 --- a/refinery/core.py +++ b/refinery/core.py @@ -159,6 +159,9 @@ def expand_halo_map(halo_map, raw_data_expansion=0, vertex_data_expansion=0, class RefineryCore: + # enables more verbose error messages + debug = False + # active map settings active_engine_name = "" active_map_name = "" @@ -1276,11 +1279,12 @@ def process_queue(self, **kw): self.process_queue_item(item, **extract_kw) if kw.get("do_printout", self.do_printout): print() # print a new line to separate operations - except RefineryError as e: + except Exception as e: + if self.debug: + print(format_exc()) + print(" Could not process queue item: %s" % (e.args[0] if e.args else "Unspecified reason.")) - except Exception: - print(format_exc()) item = self.dequeue(0) @@ -1489,11 +1493,12 @@ def extract_tags(self, tag_ids, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX, **kw tag_path = "%s.%s" % (PureWindowsPath(tag_index_ref.path), tag_index_ref.class_1.enum_name) tagslist += "%s: %s\n" % (extract_mode, tag_path) - except RefineryError as e: + except Exception as e: + if self.debug: + print(format_exc()) + print(" Could not extract tag: %s" % (e.args[0] if e.args else "Unspecified reason.")) - except Exception: - print(format_exc()) tags_to_ignore.update(curr_tag_ids) curr_tag_ids = next_tag_ids From 77a1a692cd874392685040229fa641b5bc37479a Mon Sep 17 00:00:00 2001 From: Steven Garcia Date: Mon, 26 Feb 2024 01:32:21 -0600 Subject: [PATCH 13/16] bump version --- refinery/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/refinery/__init__.py b/refinery/__init__.py index cdcd5ab..6b03cfa 100644 --- a/refinery/__init__.py +++ b/refinery/__init__.py @@ -12,8 +12,8 @@ # ############## __author__ = "Sigmmma" # YYYY.MM.DD -__date__ = "2024.02.20" -__version__ = (2, 8, 0) +__date__ = "2024.02.26" +__version__ = (2, 8, 1) __website__ = "https://github.com/Sigmmma/refinery" __all__ = ( 'defs', 'heuristic_deprotection', 'repl', 'tag_index', 'widgets', 'windows', From 2e9bcedde79dbd6084ca71e487e4855f4788e5c0 Mon Sep 17 00:00:00 2001 From: Steven Garcia Date: Sat, 9 Mar 2024 23:22:30 -0600 Subject: [PATCH 14/16] Change how protected shader tags get named --- refinery/constants.py | 58 ++++++++++++++------------ refinery/core.py | 7 ++-- refinery/main.py | 5 +-- refinery/tag_index/tag_path_handler.py | 26 ++++++++++-- 4 files changed, 60 insertions(+), 36 deletions(-) diff --git a/refinery/constants.py b/refinery/constants.py index d335ae1..138a9fb 100644 --- a/refinery/constants.py +++ b/refinery/constants.py @@ -24,35 +24,41 @@ (INVALID, "NONE",) ) -H1_TAG_SUPERCLASSES = FrozenDict( - shader_environment=("shader", "NONE"), - shader_model=("shader", "NONE"), - shader_transparent_generic=("shader", "NONE"), - shader_transparent_chicago=("shader", "NONE"), - shader_transparent_chicago_extended=("shader", "NONE"), - shader_plasma=("shader", "NONE"), - shader_meter=("shader", "NONE"), - shader_water=("shader", "NONE"), - shader_glass=("shader", "NONE"), - - biped=("unit", "object"), - vehicle=("unit", "object"), - - weapon=("item", "object"), - equipment=("item", "object"), - garbage=("item", "object"), - - device_machine=("device", "object"), - device_control=("device", "object"), - device_light_fixture=("device", "object"), - - projectile=("object", "NONE"), - scenery=("object", "NONE"), - placeholder=("object", "NONE"), - sound_scenery=("object", "NONE"), +H1_SHADER_TAG_CLASSES = frozenset(( + "shader_environment", + "shader_model", + "shader_transparent_generic", + "shader_transparent_chicago", + "shader_transparent_chicago_extended", + "shader_plasma", + "shader_meter", + "shader_water", + "shader_glass", + )) +H1_UNIT_TAG_CLASSES = frozenset(( + "biped", "vehicle" + )) +H1_ITEM_TAG_CLASSES = frozenset(( + "weapon", "equipment", "garbage" + )) +H1_DEVICE_TAG_CLASSES = frozenset(( + "device_machine", "device_control", "device_light_fixture" + )) +H1_OBJECT_TAG_CLASSES = frozenset(( + "projectile", "scenery", "placeholder", "sound_scenery" + )) +H1_TAG_SUPERCLASSES = dict( effect_postprocess_generic=("effect_postprocess", "NONE"), shader_postprocess_generic=("shader_postprocess", "NONE"), + **{cls: ("shader", "NONE") for cls in H1_SHADER_TAG_CLASSES}, + **{cls: ("object", "NONE") for cls in H1_OBJECT_TAG_CLASSES}, ) + +H1_TAG_SUPERCLASSES.update({cls: ("unit", "object") for cls in H1_UNIT_TAG_CLASSES}) +H1_TAG_SUPERCLASSES.update({cls: ("unit", "object") for cls in H1_ITEM_TAG_CLASSES}) +H1_TAG_SUPERCLASSES.update({cls: ("unit", "object") for cls in H1_DEVICE_TAG_CLASSES}) + +H1_TAG_SUPERCLASSES = FrozenDict(H1_TAG_SUPERCLASSES) del FrozenDict diff --git a/refinery/core.py b/refinery/core.py index cff2263..ec0c948 100644 --- a/refinery/core.py +++ b/refinery/core.py @@ -57,7 +57,7 @@ from reclaimer.util import calc_halo_crc32 from refinery.constants import INF, ACTIVE_INDEX, MAP_TYPE_ANY,\ - MAP_TYPE_REGULAR, MAP_TYPE_RESOURCE + MAP_TYPE_REGULAR, MAP_TYPE_RESOURCE, H1_SHADER_TAG_CLASSES from refinery import crc_functions from refinery import editor_constants as e_c from refinery.exceptions import RefineryError, MapNotLoadedError,\ @@ -500,7 +500,7 @@ def save_map(self, save_path=None, map_name=ACTIVE_INDEX, expand_halo_map(halo_map, *expansions) # move the cheape.map pointer - if halo_map.engine == "halo1yelo": + if getattr(halo_map, "is_fully_yelo", False): cheape = map_header.yelo_header.cheape_definitions move_amount = 0 for end, exp in zip(section_ends, expansions): @@ -706,6 +706,7 @@ def deprotect(self, save_path=None, map_name=ACTIVE_INDEX, tag_path_handler = TagPathHandler(tag_index_array) tag_path_handler.root_dir_prefix = self.root_dir_prefix + tag_path_handler.set_perm_suffixed_tag_classes(H1_SHADER_TAG_CLASSES) tag_path_handler.set_path_by_priority( halo_map.tag_index.scenario_tag_id & 0xFFff, @@ -1225,7 +1226,7 @@ def _heuristics_deprotect(self, path_handler, map_name=ACTIVE_INDEX, def extract_cheape(self, filepath=Path(""), map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX): halo_map = self.maps_by_engine.get(engine, {}).get(map_name) - if not halo_map or halo_map.engine != "halo1yelo": + if not getattr(halo_map, "is_fully_yelo", False): return Path("") filepath = Path(filepath) diff --git a/refinery/main.py b/refinery/main.py index c3c0831..7fdb81d 100644 --- a/refinery/main.py +++ b/refinery/main.py @@ -1211,7 +1211,7 @@ def deprotect(self, save_path=None, map_name=ACTIVE_INDEX, filetypes = [("All", "*")] if halo_map.engine == "halo1vap": filetypes.insert(0, ("Halo mapfile(chimerified)", "*.vap")) - elif halo_map.engine == "halo1yelo": + elif getattr(halo_map, "is_fully_yelo", False): filetypes.insert(0, ("Halo mapfile(extra sauce)", "*.yelo")) else: filetypes.insert(0, ("Halo mapfile", "*.map")) @@ -1230,8 +1230,7 @@ def deprotect(self, save_path=None, map_name=ACTIVE_INDEX, self._running = True try: if not save_path.suffix: - save_path = save_path.with_suffix( - '.yelo' if 'yelo' in halo_map.engine else '.map') + save_path = save_path.with_suffix(halo_map.decomp_file_ext) start = time() diff --git a/refinery/tag_index/tag_path_handler.py b/refinery/tag_index/tag_path_handler.py index 06aa72b..5995eca 100644 --- a/refinery/tag_index/tag_path_handler.py +++ b/refinery/tag_index/tag_path_handler.py @@ -19,6 +19,8 @@ BALL_MATCH_SUB = re.compile('\b(?:ball|skull|oddball)\b') FLAG_MATCH_SUB = re.compile('\b(?:flag)\b') +DIGIT_CHARS = "0123456789" +DIGIT_US_CHARS = DIGIT_CHARS + "_" def str_to_identifier(string): ''' @@ -44,6 +46,7 @@ class TagPathHandler(): _index_map = () _priorities = () _priority_mins = () + _perm_suffixed_tag_classes = () _icon_strings = () _item_strings = () @@ -59,6 +62,7 @@ def __init__(self, tag_index_array, **kwargs): self._priority_mins = dict(kwargs.get('priority_mins', {})) self._path_map = dict() self._overwritables = dict() + self._perm_suffixed_tag_classes = set(kwargs.get('perm_suffixed_tag_classes', ())) i = 0 for ref in self._index_map: @@ -125,6 +129,9 @@ def set_icon_strings(self, strings_body): new_strings.append(string.strip()) self._icon_strings = new_strings + + def set_perm_suffixed_tag_classes(self, tag_classes): + self._perm_suffixed_tag_classes = set(tag_classes) def get_index_ref(self, index): if index is None: return @@ -215,23 +222,34 @@ def set_path(self, index, new_path_no_ext, do_printout=False, elif not new_path_no_ext or new_path_no_ext[-1] == "\\": new_path_no_ext += "protected_%s" % index - ext = "." + tag_ref.class_1.enum_name.lower() + ext = tag_ref.class_1.enum_name.lower() + can_be_digit_suffixed = ext not in self._perm_suffixed_tag_classes + ext = "." + ext new_path_no_ext = str(sanitize_win32_path(new_path_no_ext)).strip().lower() - if ensure_unique_name and self._path_map.get(new_path_no_ext + ext) not in (None, index): + unique_issue = self._path_map.get(new_path_no_ext + ext) not in (None, index) + suffix_issue = not can_be_digit_suffixed and new_path_no_ext[-1:] in set(DIGIT_CHARS) + + if suffix_issue or (ensure_unique_name and unique_issue): path_pieces = list(PureWindowsPath(new_path_no_ext).parts) try: # remove the digit suffix, but keep it if there's # more than digits on the end(its not one we made) basename_pieces = path_pieces[-1].rsplit("#", 1) - if not basename_pieces[-1].strip("0123456789"): + if not basename_pieces[-1].strip( + DIGIT_CHARS if can_be_digit_suffixed else + DIGIT_US_CHARS + ): path_pieces[-1] = basename_pieces[0] except Exception: pass new_path_no_ext = get_unique_name( - self._path_map, str(PureWindowsPath(*path_pieces)), ext, index + self._path_map, str(PureWindowsPath(*path_pieces)), + ("" if can_be_digit_suffixed else "_") + ext, index ) + if not can_be_digit_suffixed: + new_path_no_ext += "_" old_path = tag_ref.path.lower() + ext new_path = new_path_no_ext + ext From ca970da94633dabfc793ddcc44f9deba07e1ff71 Mon Sep 17 00:00:00 2001 From: Steven Garcia Date: Sat, 9 Mar 2024 23:23:02 -0600 Subject: [PATCH 15/16] bump version --- refinery/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/refinery/__init__.py b/refinery/__init__.py index 6b03cfa..c1ed2d1 100644 --- a/refinery/__init__.py +++ b/refinery/__init__.py @@ -12,8 +12,8 @@ # ############## __author__ = "Sigmmma" # YYYY.MM.DD -__date__ = "2024.02.26" -__version__ = (2, 8, 1) +__date__ = "2024.03.09" +__version__ = (2, 9, 0) __website__ = "https://github.com/Sigmmma/refinery" __all__ = ( 'defs', 'heuristic_deprotection', 'repl', 'tag_index', 'widgets', 'windows', From 6d64ad92d293e5427980e0053683d2f6b1a4a955 Mon Sep 17 00:00:00 2001 From: Steven Garcia Date: Sun, 10 Mar 2024 12:41:45 -0500 Subject: [PATCH 16/16] Fix tag crc --- refinery/core.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/refinery/core.py b/refinery/core.py index ec0c948..93b3b2f 100644 --- a/refinery/core.py +++ b/refinery/core.py @@ -1696,7 +1696,11 @@ def extract_tag(self, tag_id, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX, **kw): # calculate the tagdata crc and insert it in the header bytes # NOTE: don't hate me shelly, but this is the easiest, least # offensive way to add the crc32 to the extracted tags - crc = calc_halo_crc32(data_bytes) + # NOTE: passing 0 for offset as the serialize method isn't + # expected to reset the buffer's read/write position + # when it finishes. the crc would then be calculated + # starting at the wrong offset instead of the start. + crc = calc_halo_crc32(data_bytes, 0) header_bytes[40:44] = crc.to_bytes(4, byteorder='big', signed=False) f.truncate(0)