From 9f454db3bd6148c46cb06c64325c299a273898d2 Mon Sep 17 00:00:00 2001 From: Moses_of_Egypt Date: Sat, 18 Jan 2025 19:35:36 -0500 Subject: [PATCH 1/4] misc fixes and additions Issues closed: https://github.com/Sigmmma/refinery/issues/36 https://github.com/Sigmmma/refinery/issues/31 https://github.com/Sigmmma/refinery/issues/28 https://github.com/Sigmmma/refinery/issues/27 https://github.com/Sigmmma/refinery/issues/26 https://github.com/Sigmmma/refinery/issues/11 Fixes: exception thrown when extracting stubbs models Additions: setting to control using open-sauce definitions for regular CE maps setting to control showing tag ids/metadata offsets in base 16 or 10 sound playback in tag window(requires pyogg and simpleaudio) Changes: reduced stacktrace when tag extraction fails better descriptions of what disabling safe-mode settings will do Heuristic deprotection changes: significant speed increase(most maps deprotect in 15 seconds or less) renaming logic improved to select better names added ability to configure a root directory to put all renamed tags in added ability to prioritize model names over weapon/vehicle strings underscore appended to shader names to allow permutations to be used spaces in tag names replaced with underscores in most cases --- refinery/__init__.py | 4 +- refinery/arbytmap_ext.py | 160 ++++++++++ refinery/constants.py | 58 ++-- refinery/core.py | 248 +++++++++------ refinery/defs/config_def.py | 11 +- refinery/heuristic_deprotection/constants.py | 16 +- refinery/heuristic_deprotection/functions.py | 308 +++++++++++-------- refinery/heuristic_deprotection/util.py | 95 +++--- refinery/main.py | 75 +++-- refinery/tag_index/tag_path_handler.py | 90 +++++- refinery/util.py | 17 +- refinery/widgets/explorer_hierarchy_tree.py | 13 +- refinery/windows/actions_window.py | 16 +- refinery/windows/settings_window.py | 70 ++++- 14 files changed, 822 insertions(+), 359 deletions(-) create mode 100644 refinery/arbytmap_ext.py diff --git a/refinery/__init__.py b/refinery/__init__.py index c43e338..8618802 100644 --- a/refinery/__init__.py +++ b/refinery/__init__.py @@ -12,8 +12,8 @@ # ############## __author__ = "Sigmmma" # YYYY.MM.DD -__date__ = "2020.03.13" -__version__ = (2, 6, 0) +__date__ = "2025.01.18" +__version__ = (2, 9, 0) __website__ = "https://github.com/Sigmmma/refinery" __all__ = ( 'defs', 'heuristic_deprotection', 'repl', 'tag_index', 'widgets', 'windows', diff --git a/refinery/arbytmap_ext.py b/refinery/arbytmap_ext.py new file mode 100644 index 0000000..6947520 --- /dev/null +++ b/refinery/arbytmap_ext.py @@ -0,0 +1,160 @@ +''' +This module adds an new format to arbytmap, which allows it +to read new formats. Specifically it adds A16R16G16B16_F +and R9G9B9E5, which is are floating-point formats +''' + +import arbytmap +import array +import types + +FORMAT_R9G9B9E5_SHAREDEXP = "R9G9B9E5_SHAREDEXP" +FORMAT_A16R16G16B16F = "A16R16G16B16F" + +MAX_R9G9B9E5_UNPACK_VALUE = 64 +AVG_R9G9B9E5_UNPACK_SCALE = 10 + + +def _apply_exponent_scales_to_pixels(pixels, scales): + # scale the pixels to floats + float_pixels = map(int(511).__rtruediv__, pixels) + + # multiply the float pixels by their scales and return them + return list(map(float.__mul__, float_pixels, scales)) + + +def unpack_r9g9b9e5(arby, bitmap_index, width, height, depth=1): + # unpack the pixels so we can easily manipulate them + unpacked_pixels = arby._unpack_raw_4_channel( + arby.texture_block[bitmap_index], + _R9G9B9E5_OFFSETS, _R9G9B9E5_MASKS, + [_UPSCALE_5BIT, _UPSCALE_9BIT, _UPSCALE_9BIT, _UPSCALE_9BIT] + ) + + # slice the eponents out, and apply map operations to get an exponent + # scale for each exponent using the following algorithm: v = 2^(e-15) + exps = map(int(15).__rsub__, unpacked_pixels[0::4]) + exp_scales = list(map(int(2).__pow__, exps)) + + # apply the exponent to each channel + channel_values = [(0,)] + for i in (1, 2, 3): + channel_values.append(_apply_exponent_scales_to_pixels( + unpacked_pixels[i::4], exp_scales, + )) + + # NOTE: this part is a hack. we're scaling the color range values + # to take up as much of the [0, 65536] value range as possible + average_vals = list(sum(vals)/len(vals) for vals in channel_values) + max_val = min( + MAX_R9G9B9E5_UNPACK_VALUE, + AVG_R9G9B9E5_UNPACK_SCALE * max(average_vals) + ) + max_clamp = types.MethodType(min, max_val) + for i in (1, 2, 3): + # clamp the values that are just too bright to the max + channel_values[i] = map(max_clamp, channel_values[i]) + # scale the values to the range we defined + channel_values[i] = map(float(0xFFff/max_val).__mul__, channel_values[i]) + + # create a new array to hold the exponent applied pixels + exp_applied_pixels = array.array( + unpacked_pixels.typecode, b'\xFF' * (len(unpacked_pixels) * unpacked_pixels.itemsize) + ) + for i in (1, 2, 3): + # convert the floats to ints and slice them into + # the new array(leaving the alpha white) + exp_applied_pixels[i::4] = array.array("H", map(int, channel_values[i])) + + return exp_applied_pixels + + +def unpack_a16r16g16b16_f(arby, bitmap_index, width, height, depth=1): + offsets = arby.channel_offsets + masks = arby.channel_masks + upscalers = arby.channel_upscalers + chan_ct = len(offsets) + + blank_channels = [] + for i in range(chan_ct): + if len(set(upscalers[i])) == 1: + blank_channels.append(i) + upscalers[i] = _UPSCALE_16BIT_ERASE + + # convert the pixel channels to uint16's so we can easily manipulate them + packed_channels = arby._unpack_raw_4_channel( + arby.texture_block[bitmap_index], offsets, masks, upscalers, + ) + + channel_values = array.array("I", b'\x00' * (4 * len(packed_channels))) + # pad the half-floats to regular floats + for i, half_float in enumerate(packed_channels): + # sign, exponent, and mantissa + s = half_float >> 15 + e = (half_float >> 10) & 0x1F + m = half_float & 0x3FF + + if e == 0x1F: + e = 0xFF + elif e: + e += 0x70 + elif m: + e = 0x71 + while not m & 0x400: + m <<= 1 + e -= 1 + m &= ~0x400 + + channel_values[i] = (s << 31) | (e << 23) | (m << 13) + + channel_values = array.array("f", bytearray(channel_values)) + + # clamp the values to the [0.0, 1.0] bounds + channel_values = map(types.MethodType(max, 0.0), channel_values) + channel_values = map(types.MethodType(min, 1.0), channel_values) + # scale the values to the range we defined + channel_values = map(float(0x7FFF).__mul__, channel_values) + + # convert the floats to ints and slice them into + # the new array(leaving the alpha white) + unpacked_pixels = array.array("H", map(int, channel_values)) + + # replace any empty channels with their + for chan in blank_channels: + if chan == 0: + unpacked_pixels[::chan_ct] = array.array( + "H", b'\xFF\xFF' * (len(unpacked_pixels) // chan_ct) + ) + + return unpacked_pixels + + +def pack_r9g9b9e5(arby, unpacked, width, height, depth=1): + raise NotImplementedError("Not Implemented") + + +def pack_a16r16g16b16_f(arby, unpacked, width, height, depth=1): + raise NotImplementedError("Not Implemented") + + +arbytmap.format_defs.register_format( + FORMAT_R9G9B9E5_SHAREDEXP, 1, + depths=(9,9,9,5), offsets=(18,9,0,27), + unpacker=unpack_r9g9b9e5, packer=pack_r9g9b9e5 + ) + +arbytmap.format_defs.register_format( + FORMAT_A16R16G16B16F, 1, + depths=(16,16,16,16), offsets=(16,32,48,0), + unpacker=unpack_a16r16g16b16_f, packer=pack_a16r16g16b16_f + ) + +_R9G9B9E5_OFFSETS = array.array( + "B", arbytmap.format_defs.CHANNEL_OFFSETS[FORMAT_R9G9B9E5_SHAREDEXP] + ) +_R9G9B9E5_MASKS = array.array( + "H", arbytmap.format_defs.CHANNEL_MASKS[FORMAT_R9G9B9E5_SHAREDEXP] + ) +_UPSCALE_9BIT = array.array("H", list(range(2**9))) +_UPSCALE_5BIT = array.array("H", list(range(2**5))) +_UPSCALE_16BIT_ERASE = array.array("H", b'\x00\x00' * (2**16)) diff --git a/refinery/constants.py b/refinery/constants.py index d335ae1..138a9fb 100644 --- a/refinery/constants.py +++ b/refinery/constants.py @@ -24,35 +24,41 @@ (INVALID, "NONE",) ) -H1_TAG_SUPERCLASSES = FrozenDict( - shader_environment=("shader", "NONE"), - shader_model=("shader", "NONE"), - shader_transparent_generic=("shader", "NONE"), - shader_transparent_chicago=("shader", "NONE"), - shader_transparent_chicago_extended=("shader", "NONE"), - shader_plasma=("shader", "NONE"), - shader_meter=("shader", "NONE"), - shader_water=("shader", "NONE"), - shader_glass=("shader", "NONE"), - - biped=("unit", "object"), - vehicle=("unit", "object"), - - weapon=("item", "object"), - equipment=("item", "object"), - garbage=("item", "object"), - - device_machine=("device", "object"), - device_control=("device", "object"), - device_light_fixture=("device", "object"), - - projectile=("object", "NONE"), - scenery=("object", "NONE"), - placeholder=("object", "NONE"), - sound_scenery=("object", "NONE"), +H1_SHADER_TAG_CLASSES = frozenset(( + "shader_environment", + "shader_model", + "shader_transparent_generic", + "shader_transparent_chicago", + "shader_transparent_chicago_extended", + "shader_plasma", + "shader_meter", + "shader_water", + "shader_glass", + )) +H1_UNIT_TAG_CLASSES = frozenset(( + "biped", "vehicle" + )) +H1_ITEM_TAG_CLASSES = frozenset(( + "weapon", "equipment", "garbage" + )) +H1_DEVICE_TAG_CLASSES = frozenset(( + "device_machine", "device_control", "device_light_fixture" + )) +H1_OBJECT_TAG_CLASSES = frozenset(( + "projectile", "scenery", "placeholder", "sound_scenery" + )) +H1_TAG_SUPERCLASSES = dict( effect_postprocess_generic=("effect_postprocess", "NONE"), shader_postprocess_generic=("shader_postprocess", "NONE"), + **{cls: ("shader", "NONE") for cls in H1_SHADER_TAG_CLASSES}, + **{cls: ("object", "NONE") for cls in H1_OBJECT_TAG_CLASSES}, ) + +H1_TAG_SUPERCLASSES.update({cls: ("unit", "object") for cls in H1_UNIT_TAG_CLASSES}) +H1_TAG_SUPERCLASSES.update({cls: ("unit", "object") for cls in H1_ITEM_TAG_CLASSES}) +H1_TAG_SUPERCLASSES.update({cls: ("unit", "object") for cls in H1_DEVICE_TAG_CLASSES}) + +H1_TAG_SUPERCLASSES = FrozenDict(H1_TAG_SUPERCLASSES) del FrozenDict diff --git a/refinery/core.py b/refinery/core.py index 1c47809..93b3b2f 100644 --- a/refinery/core.py +++ b/refinery/core.py @@ -23,13 +23,17 @@ from supyr_struct.util import path_normalize -from reclaimer.constants import GEN_1_HALO_ENGINES, GEN_2_ENGINES +from reclaimer.constants import GEN_1_HALO_GBX_ENGINES, GEN_1_HALO_XBOX_ENGINES,\ + GEN_1_STUBBS_ENGINES, GEN_1_SHADOWRUN_ENGINES, GEN_1_HALO_CUSTOM_ENGINES,\ + GEN_1_HALO_ENGINES, GEN_1_ENGINES, GEN_2_ENGINES, GEN_3_ENGINES from reclaimer.halo_script.hsc import get_hsc_data_block, HSC_IS_GLOBAL,\ get_h1_scenario_script_object_type_strings from reclaimer.hek import hardcoded_ce_tag_paths from reclaimer.meta.wrappers.halo1_map import Halo1Map from reclaimer.meta.wrappers.halo1_yelo import Halo1YeloMap +from reclaimer.meta.wrappers.halo1_xbox_map import Halo1XboxMap from reclaimer.meta.wrappers.halo1_anni_map import Halo1AnniMap +from reclaimer.meta.wrappers.halo1_mcc_map import Halo1MccMap from reclaimer.meta.wrappers.halo1_rsrc_map import Halo1RsrcMap from reclaimer.meta.wrappers.halo2_map import Halo2Map from reclaimer.meta.wrappers.halo3_map import Halo3Map @@ -41,17 +45,19 @@ from reclaimer.meta.wrappers.halo4_beta_map import Halo4BetaMap from reclaimer.meta.wrappers.halo5_map import Halo5Map from reclaimer.meta.wrappers.stubbs_map import StubbsMap +from reclaimer.meta.wrappers.stubbs_map_64bit import StubbsMap64Bit from reclaimer.meta.wrappers.shadowrun_map import ShadowrunMap -from reclaimer.meta.halo_map import get_map_header, get_map_version,\ +from reclaimer.meta.halo_map import get_map_header, get_engine_name,\ get_tag_index, get_map_magic from reclaimer.meta.class_repair import class_repair_functions,\ get_tagc_refs from reclaimer.meta.rawdata_ref_editing import rawdata_ref_move_functions from reclaimer.meta.halo1_map_fast_functions import class_bytes_by_fcc +from reclaimer.util import calc_halo_crc32 from refinery.constants import INF, ACTIVE_INDEX, MAP_TYPE_ANY,\ - MAP_TYPE_REGULAR, MAP_TYPE_RESOURCE + MAP_TYPE_REGULAR, MAP_TYPE_RESOURCE, H1_SHADER_TAG_CLASSES from refinery import crc_functions from refinery import editor_constants as e_c from refinery.exceptions import RefineryError, MapNotLoadedError,\ @@ -69,35 +75,37 @@ # Map the different map wrappers that decide how to read the map # and what tag defintions to use -# Note the loops after this that handle engines that use the same map format halo_map_wrappers_by_engine = { - "stubbs": StubbsMap, - "stubbspc": StubbsMap, - "shadowrun_proto": ShadowrunMap, - "halo1anni": Halo1AnniMap, - "halo2": Halo2Map, - "halo3beta": Halo3BetaMap, - "halo3": Halo3Map, - "halo3odst": Halo3OdstMap, - "haloreachbeta": HaloReachBetaMap, - "haloreach": HaloReachMap, - "halo4beta": Halo4BetaMap, - "halo4": Halo4Map, - "halo5": Halo5Map, + **{engine: Halo1Map for engine in GEN_1_HALO_GBX_ENGINES}, + **{engine: Halo1XboxMap for engine in GEN_1_HALO_XBOX_ENGINES}, + **{engine: StubbsMap for engine in GEN_1_STUBBS_ENGINES}, + **{engine: ShadowrunMap for engine in GEN_1_SHADOWRUN_ENGINES}, + **{engine: Halo2Map for engine in GEN_2_ENGINES}, + "halo3": Halo3Map, + # below are unsupported, and are simply here to allow + # loading the map and showing the tag index and header data + "halo3beta": Halo3BetaMap, + "halo3odst": Halo3OdstMap, + "haloreachbeta": HaloReachBetaMap, + "haloreach": HaloReachMap, + "halo4beta": Halo4BetaMap, + "halo4": Halo4Map, + "halo5": Halo5Map, } -for name in GEN_1_HALO_ENGINES: - halo_map_wrappers_by_engine[name] = Halo1Map -for name in GEN_2_ENGINES: - halo_map_wrappers_by_engine[name] = Halo2Map -# Use a different wrapper for this so we can use a different set of tag defs -halo_map_wrappers_by_engine["halo1yelo"] = Halo1YeloMap +# override the above with a few engine-specific map wrappers +halo_map_wrappers_by_engine.update({ + "stubbspc64bit": StubbsMap64Bit, + "halo1anni": Halo1AnniMap, + "halo1yelo": Halo1YeloMap, + "halo1mcc": Halo1MccMap, + }) def get_halo_map_section_ends(halo_map): head = halo_map.map_header index = halo_map.tag_index raw_data_end = index.model_data_offset - vertex_data_end = index.vertex_data_size + raw_data_end + vertex_data_end = index.index_parts_offset + raw_data_end index_data_end = index.model_data_size + raw_data_end meta_data_end = head.tag_data_size + head.tag_index_header_offset return raw_data_end, vertex_data_end, index_data_end, meta_data_end @@ -129,9 +137,9 @@ def expand_halo_map(halo_map, raw_data_expansion=0, vertex_data_expansion=0, meta_ptr_diff = diff - meta_data_expansion # update the map_header and tag_index_header's offsets and sizes - tag_index.model_data_offset += raw_data_expansion - tag_index.vertex_data_size += vertex_data_expansion - tag_index.model_data_size += vertex_data_expansion + triangle_data_expansion + tag_index.model_data_offset += raw_data_expansion + tag_index.index_parts_offset += vertex_data_expansion + tag_index.model_data_size += vertex_data_expansion + triangle_data_expansion halo_map.map_magic -= meta_ptr_diff map_header.tag_index_header_offset += meta_ptr_diff map_header.decomp_len = map_end @@ -151,6 +159,9 @@ def expand_halo_map(halo_map, raw_data_expansion=0, vertex_data_expansion=0, class RefineryCore: + # enables more verbose error messages + debug = False + # active map settings active_engine_name = "" active_map_name = "" @@ -183,12 +194,15 @@ class RefineryCore: skip_seen_tags_during_queue_processing = True disable_safe_mode = False disable_tag_cleaning = False + treat_ce_map_as_yelo = False # deprotection settings fix_tag_classes = True fix_tag_index_offset = False use_minimum_priorities = True + disable_minimum_equal_priorities = False use_heuristics = True + root_dir_prefix = "" valid_tag_paths_are_accurate = True scrape_tag_paths_from_scripts = True limit_tag_path_lengths = True @@ -254,6 +268,8 @@ def map_loaded(self): return self.active_map is not None def active_maps(self): return self.maps_by_engine.get(ACTIVE_INDEX, {}) @property def active_map(self): return self.active_maps.get(ACTIVE_INDEX) + @property + def safe_mode(self): return not self.disable_safe_mode def enqueue(self, operation="extract_tags", **kwargs): if isinstance(operation, RefineryQueueItem): @@ -350,7 +366,8 @@ def unload_maps(self, map_type=MAP_TYPE_REGULAR, self.set_active_map() def unload_map(self, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX, **kw): - halo_map = self.maps_by_engine.get(engine, {}).get(map_name) + maps = self.maps_by_engine.get(engine, {}) + halo_map = maps.get(map_name) if halo_map is None: return @@ -358,6 +375,12 @@ def unload_map(self, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX, **kw): self.active_map_name = "" halo_map.unload_map() + # pop the map out in case the map uses its own separate maps dict + # due to conflicting resources being referenced. This can occur + # due to multiple yelo maps being loaded that each use different + # resource maps, or mcc maps that can do the same thing. + maps.pop(map_name, None) + maps.pop(halo_map.map_name, None) def save_map(self, save_path=None, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX, **kw): @@ -380,8 +403,7 @@ def save_map(self, save_path=None, map_name=ACTIVE_INDEX, raise KeyError("No map loaded and none provided.") elif halo_map.is_resource: raise TypeError("Cannot save resource maps.") - elif halo_map.engine not in ("halo1ce", "halo1yelo", - "halo1pc", "halo1vap"): + elif halo_map.engine not in GEN_1_HALO_GBX_ENGINES: raise TypeError("Cannot save this kind of map.") elif is_path_empty(save_path): save_path = halo_map.filepath @@ -395,7 +417,7 @@ def save_map(self, save_path=None, map_name=ACTIVE_INDEX, save_dir.mkdir(exist_ok=True, parents=True) try: - map_file = halo_map.map_data + map_file, out_file = halo_map.map_data, None if save_path == halo_map.filepath: out_file = map_file = halo_map.get_writable_map_data() else: @@ -478,7 +500,7 @@ def save_map(self, save_path=None, map_name=ACTIVE_INDEX, expand_halo_map(halo_map, *expansions) # move the cheape.map pointer - if halo_map.engine == "halo1yelo": + if getattr(halo_map, "is_fully_yelo", False): cheape = map_header.yelo_header.cheape_definitions move_amount = 0 for end, exp in zip(section_ends, expansions): @@ -497,7 +519,8 @@ def save_map(self, save_path=None, map_name=ACTIVE_INDEX, # set the size of the map in the header to 0 to fix a bug where # halo will leak file handles for very large maps. Also removes # the map size limitation so halo can load stupid big maps. - if halo_map.engine in ("halo1ce", "halo1yelo", "halo1vap"): + if (halo_map.engine != "halo1mcc" and + halo_map.engine in GEN_1_HALO_CUSTOM_ENGINES): map_header.decomp_len = 0 # write the map header so the calculate_ce_checksum can read it @@ -519,7 +542,7 @@ def save_map(self, save_path=None, map_name=ACTIVE_INDEX, halo_map.filepath = halo_map.decomp_filepath = save_path except Exception: - if halo_map.map_data is not out_file: + if out_file and halo_map.map_data is not out_file: out_file.close() raise @@ -536,7 +559,7 @@ def load_map(self, map_path, replace_if_same_name=False, **kw): with get_rawdata_context(filepath=map_path, writable=False) as f: head_sig = unpack(" priority): + elif not kw.get("use_minimum_priorities") or priority is None or curr_min_prio < priority: + pass + elif curr_min_prio > priority or (kw.get("use_minimum_equal_priorities") and + kw.get("return_on_equal_min_priority")): # do nothing if priority is lower than the lowest priority - # of any tag referenced by this tag or its dependencies + # of any tag referenced by this tag or its dependencies, or + # is equal and return_on_equal_min_priority is True return curr_min_prio kw.update(depth=kw["depth"] - 1, min_priority=min_priority) @@ -70,7 +75,7 @@ def rename_scnr(tag_id, halo_map, tag_path_handler, kw.update(halo_map=halo_map, root_dir=root_dir, tag_path_handler=tag_path_handler) - meta = halo_map.get_meta(tag_id) + meta = halo_map.get_meta(tag_id, reextract=True) if meta is None: return elif not name: @@ -275,7 +280,7 @@ def rename_scnr(tag_id, halo_map, tag_path_handler, kw['priority'] = LOW_PRIORITY sub_id = get_tag_id(b.reference) tag_cls = b.reference.tag_class.enum_name - tag_name = "protected %s" % sub_id + tag_name = "protected_%s" % sub_id if tag_cls == "model_animations": ref_sub_dir = cinematic_anims_dir @@ -330,7 +335,7 @@ def rename_matg(tag_id, halo_map, tag_path_handler, kwargs_no_priority = dict(kw) kwargs_no_priority.pop('priority') - meta = halo_map.get_meta(tag_id) + meta = halo_map.get_meta(tag_id, reextract=True) if meta is None: return elif not name: @@ -566,7 +571,7 @@ def rename_hudg(tag_id, halo_map, tag_path_handler, kw.update(halo_map=halo_map, root_dir=root_dir, tag_path_handler=tag_path_handler) - meta = halo_map.get_meta(tag_id) + meta = halo_map.get_meta(tag_id, reextract=True) if meta is None: return @@ -645,7 +650,7 @@ def rename_sbsp(tag_id, halo_map, tag_path_handler, for b in meta.collision_materials.STEPTREE: min_prio.val = heuristic_deprotect( get_tag_id(b.shader), sub_dir=coll_shdr_dir, - name="protected %s" % get_tag_id(b.shader), + name="protected_%s" % get_tag_id(b.shader), override=True, **kw) # lightmap materials(non-collidable) @@ -654,18 +659,18 @@ def rename_sbsp(tag_id, halo_map, tag_path_handler, for mat in lightmap.materials.STEPTREE: min_prio.val = heuristic_deprotect( get_tag_id(mat.shader), sub_dir=non_coll_shdr_dir, - name="protected %s" % get_tag_id(mat.shader), **kw) + name="protected_%s" % get_tag_id(mat.shader), **kw) # lens flares for b in meta.lens_flares.STEPTREE: min_prio.val = heuristic_deprotect( get_tag_id(b.shader), sub_dir=sub_dir + "lens flares\\", - name="protected %s" % get_tag_id(b.shader), **kw) + name="protected_%s" % get_tag_id(b.shader), **kw) # fog palettes for b in meta.fog_palettes.STEPTREE: min_prio.val = heuristic_deprotect(get_tag_id(b.fog), sub_dir=sub_dir + weather_dir, - name=sanitize_name_piece(b.name, "protected %s" % + name=sanitize_name_piece(b.name, "protected_%s" % get_tag_id(b.fog)), **kw) # weather palettes @@ -673,11 +678,11 @@ def rename_sbsp(tag_id, halo_map, tag_path_handler, min_prio.val = heuristic_deprotect(get_tag_id(b.particle_system), sub_dir=sub_dir + weather_dir, name=sanitize_name_piece( - b.name, "protected weather %s" % + b.name, "protected_weather_%s" % get_tag_id(b.particle_system)), **kw) min_prio.val = heuristic_deprotect(get_tag_id(b.wind), sub_dir=sub_dir + weather_dir, name=sanitize_name_piece( - b.name, "protected wind %s" % + b.name, "protected_wind_%s" % get_tag_id(b.wind)), **kw) # background sounds @@ -685,7 +690,7 @@ def rename_sbsp(tag_id, halo_map, tag_path_handler, min_prio.val = heuristic_deprotect(get_tag_id(b.background_sound), sub_dir=sub_dir + sounds_dir, name=sanitize_name_piece( - b.name, "protected bg sound %s" % + b.name, "protected_bg_sound_%s" % get_tag_id(b.background_sound)), **kw) # sound environments @@ -693,7 +698,7 @@ def rename_sbsp(tag_id, halo_map, tag_path_handler, min_prio.val = heuristic_deprotect(get_tag_id(b.sound_environment), sub_dir=snd_sound_env_dir, name=sanitize_name_piece( - b.name, "protected sound env %s" % + b.name, "protected_sound_env_%s" % get_tag_id(b.sound_environment)), **kw) @@ -718,7 +723,9 @@ def rename_sky_(tag_id, halo_map, tag_path_handler, tag_path_handler=tag_path_handler) kw.setdefault("priority", DEFAULT_PRIORITY) - min_prio.val = heuristic_deprotect(get_tag_id(meta.model), name=name, **kw) + for b in (meta.model, meta.animation_graph): + min_prio.val = heuristic_deprotect(get_tag_id(b), name=name, **kw) + for b in meta.lights.STEPTREE: light_name = sanitize_name(b.global_function_name) if not light_name: @@ -745,32 +752,40 @@ def rename_obje(tag_id, halo_map, tag_path_handler, replace(" source", "").replace(" src", "") i += 1 - if not name: + string_name = model_name = "" + if not name or name.startswith("protected"): if obje_type in ("weap", "eqip"): - name = tag_path_handler.get_item_string( + string_name = tag_path_handler.get_item_string( meta.item_attrs.message_index) eqip_attrs = getattr(meta, "eqip_attrs", None) if eqip_attrs and eqip_attrs.powerup_type.data in (1, 4): - name = eqip_attrs.powerup_type.enum_name.replace("_", " ") + string_name = eqip_attrs.powerup_type.enum_name.replace("_", " ") elif obje_type == "vehi": - name = tag_path_handler.get_icon_string( + string_name = tag_path_handler.get_icon_string( meta.obje_attrs.hud_text_message_index) - if name: - # up the priority if we could detect a name for - # this in the strings for the weapon or vehicles - kw.setdefault('priority', HIGH_PRIORITY) - else: - kw.setdefault('priority', MEDIUM_HIGH_PRIORITY) - name = tag_path_handler.get_basename(get_tag_id(obje_attrs.model)) - if not name or name.startswith("protected"): - name = "protected %s" % tag_id - if obje_type == "ssce": - name = get_sound_scenery_name(meta, halo_map, name) - else: - name = get_model_name( - halo_map, get_tag_id(obje_attrs.model), name) + model_tag_id = get_tag_id(obje_attrs.model) + model_name = tag_path_handler.get_basename(model_tag_id) + if not model_name or model_name.startswith("protected"): + if obje_type == "ssce": + model_name = get_sound_scenery_name(meta, halo_map, name) + else: + model_name = get_model_name( + halo_map=halo_map, tag_id=model_tag_id, name=name + ) + + if kw.get("prioritize_model_names") and model_name: + name = model_name + kw.setdefault('priority', HIGH_PRIORITY) + elif string_name: + # up the priority if we could detect a name for + # this in the strings for the weapon or vehicles + name = string_name + kw.setdefault('priority', HIGH_PRIORITY) + elif not name: + name = model_name or ("protected_%s" % tag_id) + kw.setdefault('priority', MEDIUM_HIGH_PRIORITY) if obje_type == "bipd": obje_dir = characters_dir @@ -802,13 +817,25 @@ def rename_obje(tag_id, halo_map, tag_path_handler, if not sub_dir: sub_dir = obje_dir - if sub_dir.lower().endswith(obje_dir): + orig_sub_dir = sub_dir + is_obje_sub_dir = sub_dir.lower().endswith(obje_dir) + if is_obje_sub_dir: sub_dir += name + "\\" min_prio.val = tag_path_handler.set_path_by_priority( tag_id, root_dir + sub_dir + name, kw.get('priority'), kw.get("override"), kw.get("do_printout")) + assigned_name = tag_path_handler.get_basename(tag_id) + if name != assigned_name: + name = assigned_name + if is_obje_sub_dir: + sub_dir = orig_sub_dir + name + "\\" + + min_prio.val = tag_path_handler.set_path_by_priority( + tag_id, root_dir + sub_dir + name, kw.get('priority'), True, + kw.get("do_printout")) + sub_dir = tag_path_handler.get_sub_dir(tag_id, root_dir) name = tag_path_handler.get_basename(tag_id) @@ -930,62 +957,62 @@ def rename_shdr(tag_id, halo_map, tag_path_handler, senv_attrs = meta.senv_attrs min_prio.val = heuristic_deprotect( get_tag_id(senv_attrs.diffuse.base_map), - name="senv %s diffuse" % name, **kw) + name="%s_senv_diffuse" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(senv_attrs.diffuse.primary_detail_map), - name="senv %s pri detail" % name, **kw) + name="%s_senv_pri_detail" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(senv_attrs.diffuse.secondary_detail_map), - name="senv %s sec detail" % name, **kw) + name="%s_senv_sec_detail" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(senv_attrs.diffuse.micro_detail_map), - name="senv %s micro detail" % name, **kw) + name="%s_senv_micro_detail" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(senv_attrs.bump_properties.map), - name="senv %s bump" % name, **kw) + name="%s_senv_bump" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(senv_attrs.self_illumination.map), - name="senv %s self illum" % name, **kw) + name="%s_senv_self_illum" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(senv_attrs.reflection.cube_map), - name="senv %s reflection" % name, **kw) + name="%s_senv_reflection" % name, **kw) senv_ext = getattr(getattr(senv_attrs, "os_shader_environment_ext", ()), "STEPTREE", ()) for b in senv_ext: min_prio.val = heuristic_deprotect(get_tag_id(b.specular_color_map), - name="senv %s spec color" % name, **kw) + name="%s_senv_spec_color" % name, **kw) elif shdr_type == "soso": soso_attrs = meta.soso_attrs min_prio.val = heuristic_deprotect( get_tag_id(soso_attrs.maps.diffuse_map), - name="soso %s diffuse" % name, **kw) + name="%s_soso_diffuse" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(soso_attrs.maps.multipurpose_map), - name="soso %s multi" % name, **kw) + name="%s_soso_multi" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(soso_attrs.maps.detail_map), - name="soso %s detail" % name, **kw) + name="%s_soso_detail" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(soso_attrs.reflection.cube_map), - name="soso %s reflection" % name, **kw) + name="%s_soso_reflection" % name, **kw) soso_ext = getattr(getattr(soso_attrs, "os_shader_model_ext", ()), "STEPTREE", ()) for b in soso_ext: min_prio.val = heuristic_deprotect( get_tag_id(b.specular_color_map), - name="soso %s spec color" % name, **kw) + name="%s_soso_spec_color" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(b.base_normal_map), - name="soso %s normal" % name, **kw) + name="%s_soso_normal" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(b.detail_normal_1_map), - name="soso %s normal detail 1" % name, **kw) + name="%s_soso_normal_detail_1" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(b.detail_normal_2_map), - name="soso %s normal detail 2" % name, **kw) + name="%s_soso_normal_detail_2" % name, **kw) elif shdr_type in ("sotr", "schi", "scex"): if shdr_type == "scex": @@ -1001,17 +1028,17 @@ def rename_shdr(tag_id, halo_map, tag_path_handler, for maps in maps_list: for map in maps.STEPTREE: - min_prio.val = heuristic_deprotect(get_tag_id(map.bitmap), name="%s %s" % - (shdr_type, name), **kw) + min_prio.val = heuristic_deprotect(get_tag_id(map.bitmap), name="%s_%s" % + (name, shdr_type), **kw) kw.update(sub_dir=sub_dir) i = 0 for extra_layer in extra_layers.STEPTREE: ex_layer_name = name if len(extra_layers.STEPTREE) == 1: - ex_layer_name += " ex " + ex_layer_name += "_ex_" else: - ex_layer_name += " ex %s" % i + ex_layer_name += "_ex_%s" % i min_prio.val = heuristic_deprotect( get_tag_id(extra_layer), name=ex_layer_name, **kw) i += 1 @@ -1020,40 +1047,40 @@ def rename_shdr(tag_id, halo_map, tag_path_handler, water_shader = meta.swat_attrs.water_shader min_prio.val = heuristic_deprotect( get_tag_id(water_shader.base_map), - name="swat %s base" % name, **kw) + name="%s_swat_base" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(water_shader.reflection_map), - name="swat %s reflection" % name, **kw) + name="%s_swat_reflection" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(water_shader.ripple_maps), - name="swat %s ripples" % name, **kw) + name="%s_swat_ripples" % name, **kw) elif shdr_type == "sgla": sgla_attrs = meta.sgla_attrs min_prio.val = heuristic_deprotect( get_tag_id(sgla_attrs.background_tint_properties.map), - name="sgla %s background tint" % name, **kw) + name="%s_sgla_background_tint" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(sgla_attrs.reflection_properties.map), - name="sgla %s reflection" % name, **kw) + name="%s_sgla_reflection" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(sgla_attrs.reflection_properties.bump_map), - name="sgla %s bump" % name, **kw) + name="%s_sgla_bump" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(sgla_attrs.diffuse_properties.map), - name="sgla %s diffuse" % name, **kw) + name="%s_sgla_diffuse" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(sgla_attrs.diffuse_properties.detail_map), - name="sgla %s diffuse detail" % name, **kw) + name="%s_sgla_diffuse_detail" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(sgla_attrs.specular_properties.map), - name="sgla %s specular" % name, **kw) + name="%s_sgla_specular" % name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(sgla_attrs.specular_properties.detail_map), - name="sgla %s specular detail" % name, **kw) + name="%s_sgla_specular_detail" % name, **kw) elif shdr_type == "smet": smet_attrs = meta.smet_attrs @@ -1065,10 +1092,10 @@ def rename_shdr(tag_id, halo_map, tag_path_handler, meter_name = func_name if not meter_name: - meter_name = name if "meter" in name else name + " meter" + meter_name = name if "meter" in name else name + " meter" min_prio.val = heuristic_deprotect(get_tag_id(smet_attrs.meter_shader.map), - name="smet %s" % name, **kw) + name="%s_smet" % name, **kw) elif shdr_type == "spla": spla_attrs = meta.spla_attrs @@ -1080,10 +1107,10 @@ def rename_shdr(tag_id, halo_map, tag_path_handler, min_prio.val = heuristic_deprotect( get_tag_id(spla_attrs.primary_noise_map.noise_map), - name="spla %s noise" % plasma_name, **kw) + name="%s_spla_noise" % plasma_name, **kw) min_prio.val = heuristic_deprotect( get_tag_id(spla_attrs.primary_noise_map.noise_map), - name="spla %s noise sec" % plasma_name, **kw) + name="%s_spla_noise_sec" % plasma_name, **kw) def rename_item_attrs(meta, tag_id, halo_map, tag_path_handler, @@ -1093,7 +1120,8 @@ def rename_item_attrs(meta, tag_id, halo_map, tag_path_handler, return kw.update(halo_map=halo_map, root_dir=root_dir, - tag_path_handler=tag_path_handler) + tag_path_handler=tag_path_handler, + return_on_equal_min_priority=True) item_attrs = meta.item_attrs eqip_attrs = getattr(meta, "eqip_attrs", None) @@ -1233,7 +1261,8 @@ def rename_unit_attrs(meta, tag_id, halo_map, tag_path_handler, return kw.update(halo_map=halo_map, root_dir=root_dir, - tag_path_handler=tag_path_handler) + tag_path_handler=tag_path_handler, + return_on_equal_min_priority=True) unit_attrs = meta.unit_attrs bipd_attrs = getattr(meta, "bipd_attrs", None) @@ -1329,7 +1358,7 @@ def rename_unit_attrs(meta, tag_id, halo_map, tag_path_handler, min_prio.val = heuristic_deprotect(get_tag_id(b.melee_damage.damage_effect), sub_dir=boarding_dir, name=seat_name + " melee damage", **kw) - min_prio.val = heuristic_deprotect(get_tag_id(b.region_damage.damage_effect), + min_prio.val = heuristic_deprotect(get_tag_id(b.region_targeting.damage_effect), sub_dir=boarding_dir, name=seat_name + " region damage", **kw) @@ -1407,7 +1436,7 @@ def rename_actv(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) if not name: - name = "protected %s" % tag_id + name = "protected_%s" % tag_id if not sub_dir: sub_dir = characters_dir + name + "\\" @@ -1505,9 +1534,21 @@ def rename_mode(tag_id, halo_map, tag_path_handler, if meta is None: return + if not name or name.startswith("protected"): + name = tag_path_handler.get_basename(tag_id) + + model_name = get_model_name(meta=meta) + if (not name or name.startswith("protected")) and model_name: + name = model_name + kw.setdefault('priority', + MEDIUM_HIGH_PRIORITY if kw.get("prioritize_model_names") else + MEDIUM_PRIORITY + ) + min_prio.val = tag_path_handler.set_path_by_priority( tag_id, root_dir + sub_dir + name, kw.get('priority'), kw.get("override"), kw.get("do_printout")) + sub_dir = tag_path_handler.get_sub_dir(tag_id, root_dir) name = tag_path_handler.get_basename(tag_id) @@ -1515,7 +1556,6 @@ def rename_mode(tag_id, halo_map, tag_path_handler, sub_dir=sub_dir + shaders_dir, tag_path_handler=tag_path_handler) - shader_names = {} for i in range(len(meta.regions.STEPTREE)): region = meta.regions.STEPTREE[i] @@ -1526,7 +1566,9 @@ def rename_mode(tag_id, halo_map, tag_path_handler, for j in range(len(region.permutations.STEPTREE)): perm = region.permutations.STEPTREE[j] - shader_name = region_name + perm.name.replace(' ', '').strip("_") + shader_name = sanitize_model_or_sound_name( + region_name + "_" + perm.name + ) or model_name for lod in ("superhigh", "high", "medium", "low", "superlow"): geom_index = getattr(perm, lod + "_geometry_block") @@ -1538,17 +1580,15 @@ def rename_mode(tag_id, halo_map, tag_path_handler, for part_i in range(len(parts)): final_shader_name = shader_name if len(parts) > 1: - final_shader_name += " part%s" % part_i + final_shader_name += "_part%s" % part_i if lod != "superhigh": - final_shader_name += " " + lod + final_shader_name += "_" + lod shader_names.setdefault(parts[part_i].shader_index, final_shader_name) for i in range(len(meta.shaders.STEPTREE)): - shader_name = shader_names.get(i, "").replace("_", " ").\ - replace(".", "_").strip() min_prio.val = heuristic_deprotect(get_tag_id(meta.shaders.STEPTREE[i].shader), - name=shader_name.strip(), **kw) + name=shader_names.get(i, ""), **kw) def rename_coll(tag_id, halo_map, tag_path_handler, @@ -1733,7 +1773,8 @@ def rename_deca(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) kw.update(halo_map=halo_map, root_dir=root_dir, - tag_path_handler=tag_path_handler) + tag_path_handler=tag_path_handler, + return_on_equal_min_priority=True) meta = halo_map.get_meta(tag_id) if meta is None: @@ -1747,10 +1788,10 @@ def rename_deca(tag_id, halo_map, tag_path_handler, sub_dir = tag_path_handler.get_sub_dir(tag_id, root_dir) name = tag_path_handler.get_basename(tag_id) - min_prio.val = heuristic_deprotect(get_tag_id(meta.next_decal_in_chain), name=name + " next", - sub_dir=sub_dir, **kw) min_prio.val = heuristic_deprotect(get_tag_id(meta.shader.shader_map), name=name + " bitmaps", sub_dir=sub_dir + bitmaps_dir, **kw) + min_prio.val = heuristic_deprotect(get_tag_id(meta.next_decal_in_chain), name=name + " next", + sub_dir=sub_dir, **kw) def rename_ant_(tag_id, halo_map, tag_path_handler, @@ -1854,7 +1895,7 @@ def rename_DeLa(tag_id, halo_map, tag_path_handler, seen = kw.get("seen", set()) if not name: - name = "protected %s" % tag_id + name = "protected_%s" % tag_id kw["priority"] = (MEDIUM_HIGH_PRIORITY if kw.get("priority", 0) < MEDIUM_HIGH_PRIORITY @@ -1917,12 +1958,12 @@ def rename_lsnd(tag_id, halo_map, tag_path_handler, if not sub_dir: sub_dir = snd_music_dir - meta = halo_map.get_meta(tag_id) + meta = halo_map.get_meta(tag_id, recache=True) if meta is None: return - if not name: - name = get_sound_looping_name(meta, halo_map, "protected %s" % tag_id) + if not name or name.startswith("protected"): + name = get_sound_looping_name(meta, halo_map, "protected_%s" % tag_id) kw.update(halo_map=halo_map, root_dir=root_dir, tag_path_handler=tag_path_handler) @@ -1940,21 +1981,17 @@ def rename_lsnd(tag_id, halo_map, tag_path_handler, i = 0 tracks_dir = sub_dir + name + " tracks & details\\" for b in meta.tracks.STEPTREE: - min_prio.val = heuristic_deprotect( - get_tag_id(b.start), name="track %s start" % i, - sub_dir=tracks_dir, **kw) - min_prio.val = heuristic_deprotect( - get_tag_id(b.loop), name="track %s loop" % i, - sub_dir=tracks_dir, **kw) - min_prio.val = heuristic_deprotect( - get_tag_id(b.end), name="track %s end" % i, - sub_dir=tracks_dir, **kw) - min_prio.val = heuristic_deprotect( - get_tag_id(b.alternate_loop), name="track %s alt loop" % i, - sub_dir=tracks_dir, **kw) - min_prio.val = heuristic_deprotect( - get_tag_id(b.alternate_end), name="track %s alt end" % i, - sub_dir=tracks_dir, **kw) + for tag_ref, sub_name in (( + [b.start, "track %s start" % i], + [b.loop, "track %s loop" % i], + [b.end, "track %s end" % i], + [b.alternate_loop, "track %s alt loop" % i], + [b.alternate_end, "track %s alt end" % i], + )): + min_prio.val = heuristic_deprotect( + get_tag_id(tag_ref), name=sub_name, + sub_dir=tracks_dir, **kw) + i += 1 i = 0 @@ -2038,7 +2075,7 @@ def rename_soul(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) if not sub_dir: sub_dir = ui_shell_dir - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id meta = halo_map.get_meta(tag_id) if meta is None: @@ -2060,7 +2097,7 @@ def rename_soul(tag_id, halo_map, tag_path_handler, def rename_tagc(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id meta = halo_map.get_meta(tag_id) if meta is None: @@ -2118,7 +2155,7 @@ def rename_devc(tag_id, halo_map, tag_path_handler, def rename_ligh(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effect_lights_dir meta = halo_map.get_meta(tag_id) @@ -2147,7 +2184,7 @@ def rename_ligh(tag_id, halo_map, tag_path_handler, def rename_glw_(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effect_lights_dir meta = halo_map.get_meta(tag_id) @@ -2170,7 +2207,7 @@ def rename_glw_(tag_id, halo_map, tag_path_handler, def rename_lens(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effect_lens_flares_dir meta = halo_map.get_meta(tag_id) @@ -2194,7 +2231,7 @@ def rename_lens(tag_id, halo_map, tag_path_handler, def rename_mgs2(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effect_lens_flares_dir meta = halo_map.get_meta(tag_id) @@ -2217,7 +2254,7 @@ def rename_mgs2(tag_id, halo_map, tag_path_handler, def rename_elec(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effect_lens_flares_dir meta = halo_map.get_meta(tag_id) @@ -2240,7 +2277,7 @@ def rename_elec(tag_id, halo_map, tag_path_handler, def rename_part(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effect_particles_dir meta = halo_map.get_meta(tag_id) @@ -2279,7 +2316,7 @@ def rename_part(tag_id, halo_map, tag_path_handler, def rename_pctl(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effect_contrails_dir meta = halo_map.get_meta(tag_id) @@ -2321,7 +2358,7 @@ def rename_pctl(tag_id, halo_map, tag_path_handler, def rename_cont(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effect_contrails_dir meta = halo_map.get_meta(tag_id) @@ -2352,7 +2389,7 @@ def rename_cont(tag_id, halo_map, tag_path_handler, def rename_jpt_(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effects_dir + "damage\\" meta = halo_map.get_meta(tag_id) @@ -2375,7 +2412,7 @@ def rename_jpt_(tag_id, halo_map, tag_path_handler, def rename_effe(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = effects_dir meta = halo_map.get_meta(tag_id) if meta is None: @@ -2389,7 +2426,8 @@ def rename_effe(tag_id, halo_map, tag_path_handler, name = tag_path_handler.get_basename(tag_id) kw.update(halo_map=halo_map, sub_dir=sub_dir + name + " events\\", - root_dir=root_dir, tag_path_handler=tag_path_handler) + root_dir=root_dir, tag_path_handler=tag_path_handler, + return_on_equal_min_priority=True) i = 0 event_name = "" @@ -2445,7 +2483,7 @@ def rename_hud_background(block, name, background_name="", **kw): def rename_grhi(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = ui_hud_dir meta = halo_map.get_meta(tag_id) if meta is None: @@ -2482,7 +2520,7 @@ def rename_grhi(tag_id, halo_map, tag_path_handler, def rename_unhi(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = ui_hud_dir meta = halo_map.get_meta(tag_id) if meta is None: @@ -2534,7 +2572,7 @@ def rename_unhi(tag_id, halo_map, tag_path_handler, def rename_wphi(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = ui_hud_dir meta = halo_map.get_meta(tag_id) if meta is None: @@ -2611,7 +2649,7 @@ def rename_ngpr(tag_id, halo_map, tag_path_handler, return meta_name = sanitize_name(meta.name) - if not name: name = meta_name if meta_name else "protected %s" % tag_id + if not name: name = meta_name if meta_name else "protected_%s" % tag_id if not sub_dir: sub_dir = ui_shell_dir + "netgame_prefs\\" kw.update(halo_map=halo_map, root_dir=root_dir, @@ -2665,7 +2703,7 @@ def rename_yelo(tag_id, halo_map, tag_path_handler, tag_path_handler=tag_path_handler) kw.setdefault('priority', INF) - meta = halo_map.get_meta(tag_id) + meta = halo_map.get_meta(tag_id, reextract=True) if meta is None: return @@ -2712,7 +2750,7 @@ def rename_gelo(tag_id, halo_map, tag_path_handler, tag_path_handler=tag_path_handler) kw.setdefault('priority', INF) - meta = halo_map.get_meta(tag_id) + meta = halo_map.get_meta(tag_id, reextract=True) if meta is None: return elif not name: @@ -2757,7 +2795,7 @@ def rename_gelc(tag_id, halo_map, tag_path_handler, tag_path_handler=tag_path_handler) kw.setdefault('priority', INF) - meta = halo_map.get_meta(tag_id) + meta = halo_map.get_meta(tag_id, reextract=True) if meta is None: return elif not name: @@ -2799,7 +2837,7 @@ def rename_efpc(tag_id, halo_map, tag_path_handler, def rename_efpg(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id sub_dir = "postprocess\\" meta = halo_map.get_meta(tag_id) if meta is None: @@ -2822,7 +2860,7 @@ def rename_efpg(tag_id, halo_map, tag_path_handler, def rename_shpg(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = "postprocess\\" meta = halo_map.get_meta(tag_id) if meta is None: @@ -2865,7 +2903,7 @@ def rename_sppg(tag_id, halo_map, tag_path_handler, def rename_efpp(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id meta = halo_map.get_meta(tag_id) if meta is not None: min_prio.val = tag_path_handler.set_path_by_priority( @@ -2877,7 +2915,7 @@ def rename_efpp(tag_id, halo_map, tag_path_handler, def rename_sily(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = ui_dir meta = halo_map.get_meta(tag_id) if meta is None: @@ -2913,7 +2951,7 @@ def rename_sily(tag_id, halo_map, tag_path_handler, def rename_unic(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id if not sub_dir: sub_dir = ui_dir meta = halo_map.get_meta(tag_id) if meta is None: @@ -2980,7 +3018,7 @@ def rename_keyframe_action(b, name, **kw): def rename_atvi(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id meta = halo_map.get_meta(tag_id) if meta is None: return @@ -3016,7 +3054,7 @@ def rename_atvi(tag_id, halo_map, tag_path_handler, def rename_atvo(tag_id, halo_map, tag_path_handler, root_dir="", sub_dir="", name="", **kw): min_prio = kw.get("min_priority", MinPriority()) - if not name: name = "protected %s" % tag_id + if not name: name = "protected_%s" % tag_id meta = halo_map.get_meta(tag_id) if meta is None: return diff --git a/refinery/heuristic_deprotection/util.py b/refinery/heuristic_deprotection/util.py index 17f0053..2b7f9ad 100644 --- a/refinery/heuristic_deprotection/util.py +++ b/refinery/heuristic_deprotection/util.py @@ -7,10 +7,16 @@ # See LICENSE for more information. # +import re from refinery.util import sanitize_win32_path from refinery.heuristic_deprotection import constants as const +INVALID_NAME_CHAR_SUB = re.compile(r'[~\\/]') +REDUCE_ID_DIV_NAME_SUB = re.compile(r'[-_\s.<>{};:\'"|=+`!@#$%^&*()[\]]+') +REMOVE_PERM_AND_TRAIL_SUB = re.compile(r'^_|[\d_]+$') + + class MinPriority: _val = const.INF @property @@ -20,8 +26,9 @@ def val(self, new_val): self._val = min(new_val, self.val) def sanitize_name(name): - return str(sanitize_win32_path(name)).lower()\ - .replace("~", "").replace("\\", " ").strip() + return INVALID_NAME_CHAR_SUB.sub( + '', str(sanitize_win32_path(name)) + ).lower().strip() def sanitize_name_piece(name, default_name): @@ -47,40 +54,52 @@ def join_names(names, max_len=100): return name -def sanitize_tag_name(name, def_name): - name = sanitize_name(name).replace("_", " ").replace("-", " ").strip() - name = " ".join(s for s in name.split(" ") if s) - while name and name[-1] in "0123456789": - name = name[: -1].strip() - - if name not in const.INVALID_MODEL_NAMES: - return name - - return def_name - - -def get_model_name(halo_map, tag_id, model_name=""): - meta = halo_map.get_meta(tag_id) - if not(hasattr(meta, "regions") and meta.regions.STEPTREE): - return model_name - - names = [] - for region in meta.regions.STEPTREE: - for perm in region.permutations.STEPTREE: - names.append(sanitize_tag_name(perm.name, "")) +def sanitize_model_or_sound_name(name, def_name=''): + name = REMOVE_PERM_AND_TRAIL_SUB.sub('', + REDUCE_ID_DIV_NAME_SUB.sub('_', sanitize_name(name) + )) + return def_name if name in const.INVALID_MODEL_NAMES else name - name = "" if not names else names.pop() - if not names and name not in const.INVALID_MODEL_NAMES: - # just one single valid name was found amidst the permutations - return name - if len(meta.regions.STEPTREE) == 1: - # more than 1 perm name found. try to get a valid name from the regions - name = sanitize_tag_name(meta.regions.STEPTREE[0].name, "") - if name not in const.INVALID_MODEL_NAMES: - return name - - return model_name +def get_model_name(meta=None, halo_map=None, tag_id=None, name=""): + if meta is None: + meta = halo_map.get_meta(tag_id) + + regions = getattr(meta, "regions", None) + nodes = getattr(meta, "nodes", None) + node_name = perm_name = region_name = "" + if regions and regions.STEPTREE: + names = [] + for region in regions.STEPTREE: + for perm in region.permutations.STEPTREE: + perm_name = sanitize_model_or_sound_name(perm.name, perm_name) + if perm_name: break + + if len(regions.STEPTREE) == 1: + # couldn't find a valid name amidst the permutations, or there is + # more than 1 permutation. try to get a valid name from the regions + region_name = sanitize_model_or_sound_name(regions.STEPTREE[0].name, "") + + if nodes and nodes.STEPTREE: + for node in nodes.STEPTREE: + node_name = sanitize_model_or_sound_name( + node.name.split("frame", 1)[-1].split("bip01", 1)[-1]. + split("bone24", 1)[-1].strip(" \r\n\t-_") + ) + break + + if len(region_name) <= 2: region_name = "" + if len(perm_name) <= 2: perm_name = "" + if len(node_name) <= 2: node_name = "" + + #print("%s '%s' '%s' '%s' '%s'" % (tag_id, perm_name, region_name, node_name, name)) + + return ( + perm_name if perm_name not in const.INVALID_MODEL_NAMES else + region_name if region_name not in const.INVALID_MODEL_NAMES else + node_name if node_name not in const.INVALID_MODEL_NAMES else + name + ) def get_sound_sub_dir_and_name(snd_meta, sub_dir="", snd_name=""): @@ -113,7 +132,7 @@ def get_sound_sub_dir_and_name(snd_meta, sub_dir="", snd_name=""): for pr in snd_meta.pitch_ranges.STEPTREE: for perm in pr.permutations.STEPTREE: - perm_name = sanitize_tag_name(perm.name, "") + perm_name = sanitize_model_or_sound_name(perm.name) if perm_name: snd_name = perm_name break @@ -129,9 +148,11 @@ def get_sound_looping_name(meta, halo_map, def_name=""): # try and determine a name for this sound_looping from its sound tags for b in meta.tracks.STEPTREE: - for snd_id in (get_tag_id(b.start), get_tag_id(b.loop), get_tag_id(b.end)): + for tag_ref in ( + b.start, b.loop, b.end, b.alternate_loop, b.alternate_end + ): _, snd_name = get_sound_sub_dir_and_name( - halo_map.get_meta(snd_id, ignore_rawdata=True)) + halo_map.get_meta(get_tag_id(tag_ref), ignore_rawdata=True)) snd_name = snd_name.lower() if snd_name not in ("", "in", "start", "begin", "loops", "loop", "lp", "lps", "out", "stop", "end"): diff --git a/refinery/main.py b/refinery/main.py index e9c42b7..7fdb81d 100644 --- a/refinery/main.py +++ b/refinery/main.py @@ -13,7 +13,7 @@ import sys import webbrowser -from refinery.core import RefineryCore +from refinery.core import RefineryCore, GEN_1_HALO_GBX_ENGINES from pathlib import Path from time import time @@ -40,6 +40,11 @@ from refinery.windows.crc_window import RefineryChecksumEditorWindow from refinery.util import is_path_empty +try: + from refinery import arbytmap_ext +except Exception: + pass + from supyr_struct.defs import constants as supyr_constants from supyr_struct.field_types import FieldType @@ -144,7 +149,9 @@ def __init__(self, *args, **kwargs): self._active_engine_name = tk.StringVar(self) self._autoload_resources = tk.IntVar(self, 1) - self._do_printout = tk.IntVar(self, 1) + self._do_printout = tk.IntVar(self, 1) + self._base_10_ids = tk.IntVar(self, 1) + self._base_10_offsets = tk.IntVar(self, 1) self._force_lower_case_paths = tk.IntVar(self, 1) self._extract_yelo_cheape = tk.IntVar(self) @@ -160,6 +167,7 @@ def __init__(self, *args, **kwargs): self._skip_seen_tags_during_queue_processing = tk.IntVar(self, 1) self._disable_safe_mode = tk.IntVar(self, 0) self._disable_tag_cleaning = tk.IntVar(self, 0) + self._treat_ce_map_as_yelo = tk.IntVar(self, 0) self._globals_overwrite_mode = tk.IntVar(self, 0) self._bitmap_extract_format = tk.StringVar(self) @@ -167,7 +175,10 @@ def __init__(self, *args, **kwargs): self._fix_tag_classes = tk.IntVar(self, 1) self._fix_tag_index_offset = tk.IntVar(self) self._use_minimum_priorities = tk.IntVar(self, 1) + self._disable_minimum_equal_priorities = tk.IntVar(self, 1) + self._prioritize_model_names_over_message_strings = tk.IntVar(self, 1) self._use_heuristics = tk.IntVar(self, 1) + self._root_dir_prefix = tk.StringVar(self) self._valid_tag_paths_are_accurate = tk.IntVar(self, 1) self._scrape_tag_paths_from_scripts = tk.IntVar(self, 1) self._limit_tag_path_lengths = tk.IntVar(self, 1) @@ -188,6 +199,8 @@ def __init__(self, *args, **kwargs): do_printout=self._do_printout, autoload_resources=self._autoload_resources, + base_10_ids=self._base_10_ids, + base_10_offsets=self._base_10_offsets, force_lower_case_paths=self._force_lower_case_paths, extract_yelo_cheape=self._extract_yelo_cheape, @@ -203,6 +216,7 @@ def __init__(self, *args, **kwargs): skip_seen_tags_during_queue_processing=self._skip_seen_tags_during_queue_processing, disable_safe_mode=self._disable_safe_mode, disable_tag_cleaning=self._disable_tag_cleaning, + treat_ce_map_as_yelo=self._treat_ce_map_as_yelo, globals_overwrite_mode=self._globals_overwrite_mode, bitmap_extract_format=self._bitmap_extract_format, @@ -210,7 +224,10 @@ def __init__(self, *args, **kwargs): fix_tag_classes=self._fix_tag_classes, fix_tag_index_offset=self._fix_tag_index_offset, use_minimum_priorities=self._use_minimum_priorities, + disable_minimum_equal_priorities=self._disable_minimum_equal_priorities, + prioritize_model_names_over_message_strings=self._prioritize_model_names_over_message_strings, use_heuristics=self._use_heuristics, + root_dir_prefix=self._root_dir_prefix, valid_tag_paths_are_accurate=self._valid_tag_paths_are_accurate, scrape_tag_paths_from_scripts=self._scrape_tag_paths_from_scripts, limit_tag_path_lengths=self._limit_tag_path_lengths, @@ -437,6 +454,13 @@ def last_dir(self, new_val): new_val = Path(new_val) self._last_dir = new_val + @property + def root_dir_prefix(self): + return self._root_dir_prefix.get() + @root_dir_prefix.setter + def root_dir_prefix(self, new_val): + self._root_dir_prefix.set(new_val) + @property def running(self): return self._running @@ -532,14 +556,16 @@ def apply_config(self): paths = self.config_file.data.paths app_window = self.config_file.data.app_window fonts = self.config_file.data.fonts - - self.tagslist_path = paths.tagslist.path - self.tags_dir = paths.tags_dir.path - self.data_dir = paths.data_dir.path - self.last_dir = paths.last_dir.path + for name in ("root_dir_prefix", "tagslist_path", + "tags_dir", "data_dir", "last_dir"): + try: + setattr(self, name, getattr(paths, name).path) + except (AttributeError, IndexError): + pass self._display_mode = header.flags.display_mode.enum_name - for name in ("do_printout", "autoload_resources"): + for name in ("do_printout", "autoload_resources", + "base_10_ids", "base_10_offsets"): setattr(self, name, bool(getattr(header.flags, name))) for attr_name in header.preview_flags.NAME_MAP: @@ -598,13 +624,15 @@ def update_config(self, config_file=None): if len(paths.NAME_MAP) > len(paths): paths.extend(len(paths.NAME_MAP) - len(paths)) - paths.tagslist.path = "" if is_path_empty(self.tagslist_path) else str(self.tagslist_path) + paths.root_dir_prefix.path = "" if is_path_empty(self.root_dir_prefix) else str(self.root_dir_prefix) + paths.tagslist_path.path = "" if is_path_empty(self.tagslist_path) else str(self.tagslist_path) paths.tags_dir.path = "" if is_path_empty(self.tags_dir) else str(self.tags_dir) paths.data_dir.path = "" if is_path_empty(self.data_dir) else str(self.data_dir) paths.last_dir.path = "" if is_path_empty(self.last_dir) else str(self.last_dir) header.flags.display_mode.set_to(self._display_mode) - for attr_name in ("do_printout", "autoload_resources"): + for attr_name in ("do_printout", "autoload_resources", + "base_10_ids", "base_10_offsets"): setattr(header.flags, attr_name, getattr(self, attr_name)) for attr_name in header.preview_flags.NAME_MAP: @@ -997,11 +1025,13 @@ def unload_maps(self, map_type=False, engines_to_unload=(ACTIVE_INDEX, ), def load_map(self, map_path, make_active=True, ask_close_open=False, **kw): autoload_resources = kw.pop("autoload_resources", self.autoload_resources) + unlink_mismatched = kw.pop("unlink_mismatched_resources", True) new_map = prev_active_engine = prev_active_map = None try: new_map = RefineryCore.load_map( self, map_path, not ask_close_open, make_active=False, - autoload_resources=False, decompress_overwrite=True) + autoload_resources=False, decompress_overwrite=True, + unlink_mismatched_resources=unlink_mismatched) except MapAlreadyLoadedError: if not(ask_close_open and messagebox.askyesno( "A map with that name is already loaded!", @@ -1015,7 +1045,9 @@ def load_map(self, map_path, make_active=True, ask_close_open=False, **kw): prev_active_map = self.active_map_name new_map = RefineryCore.load_map( self, map_path, True, make_active=False, - autoload_resources=False) + autoload_resources=False, + unlink_mismatched_resources=unlink_mismatched + ) if (new_map.engine == prev_active_engine and new_map.map_name == prev_active_map): @@ -1171,8 +1203,7 @@ def deprotect(self, save_path=None, map_name=ACTIVE_INDEX, elif halo_map.is_resource: print("Cannot deprotect resource maps.") return - elif halo_map.engine not in ("halo1ce", "halo1yelo", - "halo1pc", "halo1vap"): + elif halo_map.engine not in GEN_1_HALO_GBX_ENGINES: print("Cannot deprotect this kind of map.") return @@ -1180,7 +1211,7 @@ def deprotect(self, save_path=None, map_name=ACTIVE_INDEX, filetypes = [("All", "*")] if halo_map.engine == "halo1vap": filetypes.insert(0, ("Halo mapfile(chimerified)", "*.vap")) - elif halo_map.engine == "halo1yelo": + elif getattr(halo_map, "is_fully_yelo", False): filetypes.insert(0, ("Halo mapfile(extra sauce)", "*.yelo")) else: filetypes.insert(0, ("Halo mapfile", "*.map")) @@ -1199,8 +1230,7 @@ def deprotect(self, save_path=None, map_name=ACTIVE_INDEX, self._running = True try: if not save_path.suffix: - save_path = save_path.with_suffix( - '.yelo' if 'yelo' in halo_map.engine else '.map') + save_path = save_path.with_suffix(halo_map.decomp_file_ext) start = time() @@ -1272,8 +1302,7 @@ def save_map_as(self, e=None): elif halo_map.is_resource: print("Cannot save resource maps.") return Path("") - elif halo_map.engine not in ("halo1ce", "halo1yelo", - "halo1pc", "halo1vap"): + elif halo_map.engine not in GEN_1_HALO_GBX_ENGINES: print("Cannot save this kind of map.") return Path("") @@ -1317,8 +1346,7 @@ def save_map(self, save_path=None, map_name=ACTIVE_INDEX, elif halo_map.is_resource: print("Cannot save resource maps.") return Path("") - elif halo_map.engine not in ("halo1ce", "halo1yelo", - "halo1pc", "halo1vap"): + elif halo_map.engine not in GEN_1_HALO_GBX_ENGINES: print("Cannot save this kind of map.") return Path("") elif save_path is None or is_path_empty(save_path): @@ -1365,7 +1393,10 @@ def save_map(self, save_path=None, map_name=ACTIVE_INDEX, if reload_window and not is_path_empty(save_path): print("Reloading map to apply changes...") - self.load_map(save_path, make_active=True, autoload_resources=False) + self.load_map( + save_path, make_active=True, autoload_resources=False, + unlink_mismatched_resources=False + ) return save_path diff --git a/refinery/tag_index/tag_path_handler.py b/refinery/tag_index/tag_path_handler.py index 792d12c..5995eca 100644 --- a/refinery/tag_index/tag_path_handler.py +++ b/refinery/tag_index/tag_path_handler.py @@ -8,20 +8,45 @@ # import os +import re from pathlib import PureWindowsPath from refinery.constants import INF from refinery.util import sanitize_win32_path, get_unique_name -from supyr_struct.util import str_to_identifier +from supyr_struct.util import str_to_identifier as orig_str_to_identifier from queue import LifoQueue, Empty as EmptyQueueException +BALL_MATCH_SUB = re.compile('\b(?:ball|skull|oddball)\b') +FLAG_MATCH_SUB = re.compile('\b(?:flag)\b') +DIGIT_CHARS = "0123456789" +DIGIT_US_CHARS = DIGIT_CHARS + "_" + +def str_to_identifier(string): + ''' + This version of str_to_identifier won't die if it fails + to sanitize the string to something useful, but will + instead return a single underscore. Useful when dealing + with potentially completely invalid identifier strings. + ''' + name = "" + try: + name = orig_str_to_identifier(string) + except (AssertionError, TypeError): + pass + + return name or "_" + class TagPathHandler(): + # _root_dir_prefix is a directory name to prefix to + # all tags renamed through priority(i.e. deprotection) + _root_dir_prefix = "" _path_map = () _index_map = () _priorities = () _priority_mins = () + _perm_suffixed_tag_classes = () _icon_strings = () _item_strings = () @@ -37,6 +62,7 @@ def __init__(self, tag_index_array, **kwargs): self._priority_mins = dict(kwargs.get('priority_mins', {})) self._path_map = dict() self._overwritables = dict() + self._perm_suffixed_tag_classes = set(kwargs.get('perm_suffixed_tag_classes', ())) i = 0 for ref in self._index_map: @@ -47,6 +73,14 @@ def __init__(self, tag_index_array, **kwargs): self._overwritables[i] = False if ref.indexed else True i += 1 + @property + def root_dir_prefix(self): + return self._root_dir_prefix or "" + @root_dir_prefix.setter + def root_dir_prefix(self, value): + value = str(PureWindowsPath(value)).lower().rstrip("\\") + self._root_dir_prefix = value + "\\" if value else "" + @property def def_priority(self): return self._def_priority @@ -63,14 +97,20 @@ def get_icon_string(self, index): def set_item_strings(self, strings_body): new_strings = [] + for b in strings_body.strings.STEPTREE: string = str_to_identifier(b.data.lower()).\ replace("_", " ").replace(" d ", ' ').\ replace("picked up an ", '').replace("picked up a ", '').\ - replace("picked up the ", '').replace("picked up ", '') + replace("picked up the ", '').replace("picked up ", '').\ + replace("powerup", '') if string == "need a string entry here": string = "" + elif FLAG_MATCH_SUB.match(string): + string = "flag" + elif BALL_MATCH_SUB.match(string): + string = "ball" elif " for " in string: string = string.split(" for ")[-1] + " ammo" elif string.startswith("for "): @@ -89,6 +129,9 @@ def set_icon_strings(self, strings_body): new_strings.append(string.strip()) self._icon_strings = new_strings + + def set_perm_suffixed_tag_classes(self, tag_classes): + self._perm_suffixed_tag_classes = set(tag_classes) def get_index_ref(self, index): if index is None: return @@ -164,32 +207,49 @@ def get_will_overwrite(self, index, priority, override=False): return False return True - def set_path(self, index, new_path_no_ext, do_printout=False, ensure_unique_name=True): + def set_path(self, index, new_path_no_ext, do_printout=False, + ensure_unique_name=True, ensure_root_prefixed=False): if index is None: return index &= 0xFFff + new_path_no_ext = (new_path_no_ext or "").lower() + if ensure_root_prefixed and not new_path_no_ext.startswith(self.root_dir_prefix): + new_path_no_ext = self.root_dir_prefix + new_path_no_ext + tag_ref = self.get_index_ref(index) if tag_ref is None: return elif not new_path_no_ext or new_path_no_ext[-1] == "\\": - new_path_no_ext += "protected %s" % index + new_path_no_ext += "protected_%s" % index + + ext = tag_ref.class_1.enum_name.lower() + can_be_digit_suffixed = ext not in self._perm_suffixed_tag_classes + ext = "." + ext + new_path_no_ext = str(sanitize_win32_path(new_path_no_ext)).strip().lower() - ext = "." + tag_ref.class_1.enum_name.lower() - new_path_no_ext = str(sanitize_win32_path(new_path_no_ext.lower())).strip() + unique_issue = self._path_map.get(new_path_no_ext + ext) not in (None, index) + suffix_issue = not can_be_digit_suffixed and new_path_no_ext[-1:] in set(DIGIT_CHARS) - if ensure_unique_name and self._path_map.get(new_path_no_ext + ext) not in (None, index): - path_pieces = PureWindowsPath(new_path_no_ext).parts - path_basename = path_pieces[-1] + if suffix_issue or (ensure_unique_name and unique_issue): + path_pieces = list(PureWindowsPath(new_path_no_ext).parts) try: - path_basename_pieces = path_basename.split("#") - if len(path_basename_pieces) > 1: - path_basename = "#".join(path_basename_pieces[: -1]) + # remove the digit suffix, but keep it if there's + # more than digits on the end(its not one we made) + basename_pieces = path_pieces[-1].rsplit("#", 1) + if not basename_pieces[-1].strip( + DIGIT_CHARS if can_be_digit_suffixed else + DIGIT_US_CHARS + ): + path_pieces[-1] = basename_pieces[0] except Exception: pass - path_pieces = tuple(path_pieces[: -1]) + (path_basename, ) new_path_no_ext = get_unique_name( - self._path_map, str(PureWindowsPath(*path_pieces)), ext, index) + self._path_map, str(PureWindowsPath(*path_pieces)), + ("" if can_be_digit_suffixed else "_") + ext, index + ) + if not can_be_digit_suffixed: + new_path_no_ext += "_" old_path = tag_ref.path.lower() + ext new_path = new_path_no_ext + ext @@ -224,7 +284,7 @@ def set_path_by_priority(self, index, new_path_no_ext, priority=None, if not self.get_will_overwrite(index, priority, override): return self.get_priority(index) - self.set_path(index, new_path_no_ext, do_printout) + self.set_path(index, new_path_no_ext, do_printout, ensure_root_prefixed=True) self.set_priority(index, priority) return priority diff --git a/refinery/util.py b/refinery/util.py index 0cbbd57..a334b21 100644 --- a/refinery/util.py +++ b/refinery/util.py @@ -117,10 +117,17 @@ def sanitize_win32_path(name): def get_unique_name(collection, name="", ext="", curr_value=object()): - final_name = name - i = 1 - while collection.get(final_name + ext) not in (None, curr_value): - final_name = "%s #%s" % (name, i) + name_parts = name.split("\\") + basename = name_parts[-1] + # if the folder name is the same as the file name, update it too + update_folder_name = len(name_parts) > 1 and name_parts[-2] == basename + + i = 0 + while collection.get("\\".join(name_parts) + ext) not in (None, curr_value): i += 1 + new_basename = "%s#%s" % (basename, i) + name_parts[-1] = new_basename + if update_folder_name: + name_parts[-2] = new_basename - return final_name + return "\\".join(name_parts) diff --git a/refinery/widgets/explorer_hierarchy_tree.py b/refinery/widgets/explorer_hierarchy_tree.py index 831dac0..a1770cc 100644 --- a/refinery/widgets/explorer_hierarchy_tree.py +++ b/refinery/widgets/explorer_hierarchy_tree.py @@ -58,6 +58,13 @@ def __init__(self, *args, **kwargs): self.tags_tree.bind('', self.activate_item) self.tags_tree.bind('', self.set_sort_mode) + @property + def id_format_str(self): + return '%d' if getattr(self.app_root, 'base_10_ids', False) else '%04X' + @property + def pointer_format_str(self): + return '%d' if getattr(self.app_root, 'base_10_offsets', False) else '%08X' + def set_sort_mode(self, event): if self.tags_tree.identify_region(event.x, event.y) != "heading": return @@ -455,12 +462,12 @@ def add_tag_index_ref(self, parent_dir_parts, tag_path, tag_index_ref): if tag_index_ref.indexed and pointer_converter and not is_h1_rsrc_map: pointer = "not in map" elif pointer_converter is not None: - pointer = '%08X' % pointer_converter.v_ptr_to_f_ptr( + pointer = self.pointer_format_str % pointer_converter.v_ptr_to_f_ptr( tag_index_ref.meta_offset) else: pointer = 0 - meta_offset = '%08X' % tag_index_ref.meta_offset + meta_offset = self.pointer_format_str % tag_index_ref.meta_offset try: cls1 = cls2 = cls3 = "" @@ -478,7 +485,7 @@ def add_tag_index_ref(self, parent_dir_parts, tag_path, tag_index_ref): parent_iid, 'end', iid=str(tag_id), tags=("item", ), text=tag_path, values=(cls1, cls2, cls3, - meta_offset, pointer, '%04X' % tag_id)) + meta_offset, pointer, self.id_format_str % tag_id)) self.tree_id_to_index_ref[tag_id] = tag_index_ref except Exception: print(format_exc()) diff --git a/refinery/windows/actions_window.py b/refinery/windows/actions_window.py index 70b73d9..4e9b271 100644 --- a/refinery/windows/actions_window.py +++ b/refinery/windows/actions_window.py @@ -284,15 +284,23 @@ def tags_list_browse(self): init_dir = os.path.dirname(self.tagslist_path.get()) except Exception: init_dir = None - dirpath = asksaveasfilename( + filepath = asksaveasfilename( initialdir=init_dir, parent=self, title="Select where to save the tag list log", - filetypes=(("text log", "*.txt"), ("All", "*"))) + filetypes=( + ("text file", "*.txt"), + ("text log", "*.log"), + ("All", "*") + )) - if not dirpath: + if not filepath: return - self.tagslist_path.set(str(Path(dirpath))) + filepath = Path(filepath) + if not filepath.suffix: + filepath = filepath.with_suffix(".txt") + + self.tagslist_path.set(str(filepath)) def extract_to_browse(self): dirpath = askdirectory( diff --git a/refinery/windows/settings_window.py b/refinery/windows/settings_window.py index c7623db..c9585a6 100644 --- a/refinery/windows/settings_window.py +++ b/refinery/windows/settings_window.py @@ -78,9 +78,10 @@ def __init__(self, *args, **kwargs): "generate_comp_verts", "generate_uncomp_verts", "force_lower_case_paths", "fix_tag_classes", "autoload_resources", "extract_yelo_cheape", + "base_10_ids", "base_10_offsets", "use_minimum_priorities", "use_heuristics", "rename_cached_tags", "show_all_fields", - "show_structure_meta", + "show_structure_meta", "disable_minimum_equal_priorities", "edit_all_fields", "allow_corrupt", "valid_tag_paths_are_accurate", "limit_tag_path_lengths", "scrape_tag_paths_from_scripts", "shallow_ui_widget_nesting", @@ -88,11 +89,15 @@ def __init__(self, *args, **kwargs): "do_printout", "print_heuristic_name_changes", "use_scenario_names_for_script_names", "skip_seen_tags_during_queue_processing", - "disable_safe_mode", "disable_tag_cleaning",): + "prioritize_model_names_over_message_strings", + "disable_safe_mode", "disable_tag_cleaning", + "treat_ce_map_as_yelo",): object.__setattr__(self, attr, settings.get(attr, tk.IntVar(self))) for attr in ("bitmap_extract_format", "globals_overwrite_mode", - "tags_dir", "data_dir", "tagslist_path"): + "tags_dir", "data_dir", "tagslist_path", + "root_dir_prefix" + ): object.__setattr__(self, attr, settings.get(attr, tk.StringVar(self))) @@ -119,6 +124,12 @@ def __init__(self, *args, **kwargs): self.tags_list_frame, text="Browse", command=self.tags_list_browse, width=6) + # root directory prefix + self.root_dir_frame = tk.LabelFrame( + self.heuristics_frame, text="Directory to prefix to all deprotected tag names") + self.root_dir_entry = tk.Entry( + self.root_dir_frame, textvariable=self.root_dir_prefix) + self.rename_scnr_dups_cbtn = tk.Checkbutton( self.tag_fixup_frame, text=( @@ -132,15 +143,26 @@ def __init__(self, *args, **kwargs): self.tag_fixup_frame, text="Generate uncompressed lightmap vertices", variable=self.generate_uncomp_verts) - self.dont_touch_frame = tk.LabelFrame( - self.tag_fixup_frame, - text="ONLY CHECK THESE IF YOU ARE NOT DEALING WITH PROTECTED MAPS") + self.dont_touch_frame = tk.LabelFrame(self.tag_fixup_frame) + self.dont_touch_label = tk.Label( + self.dont_touch_frame, justify="left", text=( + "\nThese settings change how Refinery reads tags from maps.\n" + "Use caution when changing these, as these settings can make Refinery\n" + "crash or freeze when reading from some heavily protected/corrupted maps.\n" + "They are safe to disable if you are certain the map isn't protected.\n" + )) self.disable_safe_mode_cbtn = tk.Checkbutton( self.dont_touch_frame, variable=self.disable_safe_mode, justify="left", - text="Disable safe-mode") + text="Disable bounds-checking while reading tags(i.e. safe-mode)") self.disable_tag_cleaning_cbtn = tk.Checkbutton( self.dont_touch_frame, variable=self.disable_tag_cleaning, justify="left", - text="Disable cleaning errors from tags when reading them.") + text=("Disable cleaning unused/invalid data from tags.\n" + "Be aware that tag cleaning can cause unused tags to not be\n" + "found when running deprotection or recursive extraction." + )) + self.treat_ce_map_as_yelo_cbtn = tk.Checkbutton( + self.dont_touch_frame, variable=self.treat_ce_map_as_yelo, justify="left", + text="Always use Open Sauce tag structures for CE maps(requires reload of map)") self.overwrite_cbtn = tk.Checkbutton( @@ -228,18 +250,24 @@ def __init__(self, *args, **kwargs): variable=self.fix_tag_index_offset, justify='left') + self.print_heuristic_progress_cbtn = tk.Checkbutton( + self.heuristics_frame, text=("Print heuristic tag path changes"), + variable=self.print_heuristic_name_changes, justify='left') self.valid_tag_paths_are_accurate_cbtn = tk.Checkbutton( self.heuristics_frame, text="Do not rename non-protected tag paths", variable=self.valid_tag_paths_are_accurate) self.shallow_ui_widget_nesting_cbtn = tk.Checkbutton( self.heuristics_frame, text="Use shallow ui_widget_definition nesting", variable=self.shallow_ui_widget_nesting) + self.prioritize_model_names_cbtn = tk.Checkbutton( + self.heuristics_frame, text="Prioritize weapon/vehicle model names over message strings", + variable=self.prioritize_model_names_over_message_strings) self.use_fast_heuristics_cbtn = tk.Checkbutton( self.heuristics_frame, text="Use fast heuristics", variable=self.use_minimum_priorities) - self.print_heuristic_progress_cbtn = tk.Checkbutton( - self.heuristics_frame, text=("Print heuristic tag path changes"), - variable=self.print_heuristic_name_changes, justify='left') + self.use_fastest_heuristics_cbtn = tk.Checkbutton( + self.heuristics_frame, text="Use fastest heuristics(experimental)", + variable=self.disable_minimum_equal_priorities, onvalue=0, offvalue=1) font_frame_widgets = {} @@ -313,7 +341,12 @@ def __init__(self, *args, **kwargs): self.allow_corrupt_cbtn = tk.Checkbutton( self.other_frame, variable=self.allow_corrupt, text="Allow previewing corrupt tags") - + self.base_10_ids_cbtn = tk.Checkbutton( + self.other_frame, variable=self.base_10_ids, + text="Show tag ids in base10 instead of base16") + self.base_10_offsets_cbtn = tk.Checkbutton( + self.other_frame, variable=self.base_10_offsets, + text="Show tag pointers in base10 instead of base16") # pack everything self.tabs.pack(fill="both", expand=True) @@ -341,7 +374,8 @@ def __init__(self, *args, **kwargs): w.pack(padx=4, anchor='w') self.dont_touch_frame.pack(padx=4, anchor='w', expand=True, fill="both") - for w in (self.disable_safe_mode_cbtn, self.disable_tag_cleaning_cbtn): + for w in (self.dont_touch_label, self.disable_safe_mode_cbtn, + self.disable_tag_cleaning_cbtn, self.treat_ce_map_as_yelo_cbtn): w.pack(padx=4, anchor='w') for w in (self.fix_tag_classes_cbtn, self.use_heuristics_cbtn, @@ -354,13 +388,17 @@ def __init__(self, *args, **kwargs): for w in (self.print_heuristic_progress_cbtn, self.valid_tag_paths_are_accurate_cbtn, self.shallow_ui_widget_nesting_cbtn, + self.prioritize_model_names_cbtn, self.use_fast_heuristics_cbtn, + self.use_fastest_heuristics_cbtn, + self.root_dir_frame ): w.pack(padx=4, anchor='w') for w in (self.autoload_resources_cbtn, self.extract_yelo_cheape_cbtn, self.show_all_fields_cbtn, self.show_structure_meta_cbtn, self.edit_all_fields_cbtn, self.allow_corrupt_cbtn, + self.base_10_ids_cbtn, self.base_10_offsets_cbtn, ): w.pack(padx=4, anchor='w') @@ -375,9 +413,11 @@ def __init__(self, *args, **kwargs): for w1, w2 in ((self.tags_dir_entry, self.tags_dir_browse_button), (self.data_dir_entry, self.data_dir_browse_button), - (self.tags_list_entry, self.browse_tags_list_button)): + (self.tags_list_entry, self.browse_tags_list_button), + (self.root_dir_entry, None)): w1.pack(padx=(4, 0), pady=2, side='left', expand=True, fill='x') - w2.pack(padx=(0, 4), pady=2, side='left') + if w2: + w2.pack(padx=(0, 4), pady=2, side='left') # make the window not show up on the start bar self.transient(self.master) From 0ab4d541cad470dddd6a17351fcedaae3b48b933 Mon Sep 17 00:00:00 2001 From: Moses_of_Egypt Date: Sat, 18 Jan 2025 19:42:02 -0500 Subject: [PATCH 2/4] update changelog --- CHANGELOG.MD | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 32e3293..7e2b645 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -4,6 +4,24 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.9.0] +### Changed + - exception thrown when extracting stubbs models + - reduced stacktrace output when tag extraction fails(unless RefineryCore.debug == True) + - Heuristic deprotection changes: + - better descriptions of what disabling safe-mode settings will do + - significant speed increase(most maps deprotect in 15 seconds or less) + - renaming logic improved to select better names + - added ability to configure a root directory to put all renamed tags in + - added ability to prioritize model names over names taken from weapon/vehicle strings + - underscore appended to shader names to allow permutations to be used + - spaces in tag names replaced with underscores in most cases + +### Added + - config setting to control using open-sauce definitions for regular CE maps + - config setting to control showing tag ids and metadata offsets in base 16 or base 10 + - sound playback in tag window(requires pyogg and simpleaudio) + ## [2.6.0] ### Changed - Only maps with opensauce headers will have their tags treated as OS_V4 on extraction now. From 6de2f03fc871ec2871ec995beeaf3796f3e3793b Mon Sep 17 00:00:00 2001 From: Moses_of_Egypt Date: Sun, 19 Jan 2025 02:10:20 -0500 Subject: [PATCH 3/4] Update supported versions --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index b55b511..75e3223 100644 --- a/setup.py +++ b/setup.py @@ -68,6 +68,9 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3 :: Only", ], zip_safe=False, From d2bf9650c2b33acf6d13b6563e68594cd195650d Mon Sep 17 00:00:00 2001 From: Moses_of_Egypt Date: Thu, 30 Jan 2025 22:54:30 -0500 Subject: [PATCH 4/4] pylint fixes and cleanup --- refinery/__main__.py | 2 +- refinery/core.py | 1 + refinery/crc_functions.py | 86 ++++++++++---------- refinery/heuristic_deprotection/functions.py | 22 ++--- refinery/main.py | 14 ++-- refinery/repl_run.py | 44 +++++----- refinery/tag_index/tag_path_handler.py | 8 +- refinery/util.py | 6 +- refinery/widgets/explorer_hierarchy_tree.py | 2 +- refinery/windows/actions_window.py | 2 +- 10 files changed, 95 insertions(+), 92 deletions(-) diff --git a/refinery/__main__.py b/refinery/__main__.py index 333aa18..ee798c4 100644 --- a/refinery/__main__.py +++ b/refinery/__main__.py @@ -40,7 +40,7 @@ def main(): except Exception: pass print(exception, file=sys.stderr) - return 1; + return 1 if __name__ == "__main__": main() diff --git a/refinery/core.py b/refinery/core.py index 93b3b2f..bc16f09 100644 --- a/refinery/core.py +++ b/refinery/core.py @@ -201,6 +201,7 @@ class RefineryCore: fix_tag_index_offset = False use_minimum_priorities = True disable_minimum_equal_priorities = False + prioritize_model_names_over_message_strings = False use_heuristics = True root_dir_prefix = "" valid_tag_paths_are_accurate = True diff --git a/refinery/crc_functions.py b/refinery/crc_functions.py index 118e738..a2e0a31 100644 --- a/refinery/crc_functions.py +++ b/refinery/crc_functions.py @@ -7,7 +7,9 @@ # See LICENSE for more information. # -import refinery, zlib, gc +import refinery +import zlib +import gc from random import seed as P, getrandbits as Q from traceback import format_exc @@ -17,54 +19,54 @@ # this was intentionally obfuscated at one point, and Moses doesn't # have the unobfuscated version, so it's staying as-is def J(l=[0,0,0]): - if l[2]<0:return U([l[0],l[1],l[2]]) - while l[1]: - if l[1]&1:l[2]=V([l[2],l[0],0]) - l[1],l[0]=l[1]>>1,V([l[0],l[0],0]) - return l[2]|_crc + if l[2]<0:return U([l[0],l[1],l[2]]) + while l[1]: + if l[1]&1:l[2]=V([l[2],l[0],0]) + l[1],l[0]=l[1]>>1,V([l[0],l[0],0]) + return l[2]|_crc def E(l=[0,0,0]): - l.append(globals());l[1]=('[%s]'%'_+_=s]twlxuqa-fdeudr_-_'[::-2])[1:-1] - l[0]=getattr(l[-1][str(bytes([197]),'9733j0fpuc'[::-2])],l[1])[0];l[0][0]==l[0][-1] - while l[0][0]==l[0][-1]:l[0][:]=[l[-1][("p"''""'Vu'""'s'"Qe")[4]](63)for y in')(@__'] - return (l[0][0]^sum(l[0])^l[0][-1]|_crc)&((1<<32)-1) + l.append(globals());l[1]=('[%s]'%'_+_=s]twlxuqa-fdeudr_-_'[::-2])[1:-1] + l[0]=getattr(l[-1][str(bytes([197]),'9733j0fpuc'[::-2])],l[1])[0];l[0][0]==l[0][-1] + while l[0][0]==l[0][-1]:l[0][:]=[l[-1][("p"''""'Vu'""'s'"Qe")[4]](63)for y in')(@__'] + return (l[0][0]^sum(l[0])^l[0][-1]|_crc)&((1<<32)-1) def B(l=[0,0,0]): - l[2],l[0],l[1],b=l[0],0x104c11db7,0,1 - while l[2]: - q,r=S([l[0],l[2],l[1],0]) - l[0],l[2],l[1],b=l[2],r,b,l[1]^V([q,b,0]) - return l[1]|_crc + l[2],l[0],l[1],b=l[0],0x104c11db7,0,1 + while l[2]: + q,r=S([l[0],l[2],l[1],0]) + l[0],l[2],l[1],b=l[2],r,b,l[1]^V([q,b,0]) + return l[1]|_crc def G(l=[0,0,0]): - l.append(globals());l[1]=('(%s)'%'_+_qt-cdiudr_-_'[::-2])[:-1] - l[1]=getattr(l[-1][str(bytes([114]),'61371ecup'[::-2])],l[0])[0];l[1][0]=l[1][-1] - while l[1][0]==l[1][-1]:l[1][:]=[l[-1][('pV'""'us''Qe')[4]](31)for w in'][^@&'] - return (l[1][0]^sum(l[1])^l[1][-1]|_crc)&((1<<32)-1) + l.append(globals());l[1]=('(%s)'%'_+_qt-cdiudr_-_'[::-2])[:-1] + l[1]=getattr(l[-1][str(bytes([114]),'61371ecup'[::-2])],l[0])[0];l[1][0]=l[1][-1] + while l[1][0]==l[1][-1]:l[1][:]=[l[-1][('pV'""'us''Qe')[4]](31)for w in'][^@&'] + return (l[1][0]^sum(l[1])^l[1][-1]|_crc)&((1<<32)-1) def V(l=[0,0,0]): - if l[1]< -16:return U([l[0],l[1],-16]) - if not l[1]:return _crc - while l[1]: - l[0],l[1],l[2]=l[0]<<1,l[1]>>1,l[2]^(l[0]*(l[1]&1)) - l[0]=l[0]^0x104c11db7 if l[0]&(1<<32)else l[0] - return l[2]|_crc + if l[1]< -16:return U([l[0],l[1],-16]) + if not l[1]:return _crc + while l[1]: + l[0],l[1],l[2]=l[0]<<1,l[1]>>1,l[2]^(l[0]*(l[1]&1)) + l[0]=l[0]^0x104c11db7 if l[0]&(1<<32)else l[0] + return l[2]|_crc def U(l=[0,0,0]): - l[:]=l[:]+type(l)(eval(str(b'\x93\xba\xf1\xbbK\xa2\x85\x85\x92k\x93\xba\xf1\xbbK\xa3\x85\x93\x93k\x93\xba\xf1\xbbK\xa6\x99\x89\xa3\x85k\x93\xba\xf1\xbbK\x99\x85\x81\x84k\x7fmm\x84\x85\x86\x81\xa4\x93\xa3\xa2mm\x7f', '037'), globals(), locals())) - l[3](0,2);r,k=getattr(E,l[7])[0],getattr(O,l[7])[0] - l[:2]=sum(((l[0]>>(31-i))&1)<>(31-i))&1)<>(31-i))&1)<>3]^=(d>>f)&(15+(0xf<<4)) - l[3](l[2]);l[5](b) - return l[0]|_crc + l[:]=l[:]+type(l)(eval(str(b'\x93\xba\xf1\xbbK\xa2\x85\x85\x92k\x93\xba\xf1\xbbK\xa3\x85\x93\x93k\x93\xba\xf1\xbbK\xa6\x99\x89\xa3\x85k\x93\xba\xf1\xbbK\x99\x85\x81\x84k\x7fmm\x84\x85\x86\x81\xa4\x93\xa3\xa2mm\x7f', '037'), globals(), locals())) + l[3](0,2);r,k=getattr(E,l[7])[0],getattr(O,l[7])[0] + l[:2]=sum(((l[0]>>(31-i))&1)<>(31-i))&1)<>(31-i))&1)<>3]^=(d>>f)&(15+(0xf<<4)) + l[3](l[2]);l[5](b) + return l[0]|_crc def S(l=[0,0,0]): - if not l[0]:return _crc>>8,_crc<<8 - l[2]=l[1].bit_length() - for i in range(l[0].bit_length()-l[2],-1,-1): - if(l[0]>>(i+l[2]-1))&1:l[3],l[0]=l[3]|(1<>8),l[0]|(_crc<<8) + if not l[0]:return _crc>>8,_crc<<8 + l[2]=l[1].bit_length() + for i in range(l[0].bit_length()-l[2],-1,-1): + if(l[0]>>(i+l[2]-1))&1:l[3],l[0]=l[3]|(1<>8),l[0]|(_crc<<8) def O(l=[0,0,0]): - l.append(globals());l[0]=('{%s}'%'_+_=s*twlxuqa-fdeudr_-_'[::-2])[1:-1] - l[1]=getattr(l[-1][str(bytes([69]),'71374epuc'[::-2])],l[0])[0];l[1][0]=l[1][-1] - while l[1][0]==l[1][-1]:l[1][:]=[l[-1][('pV'""'us''Qe')[4]](31)for w in'#$}{!'] - return (l[1][0]^sum(l[1])^l[1][-1]|_crc)&((1<<32)-1) + l.append(globals());l[0]=('{%s}'%'_+_=s*twlxuqa-fdeudr_-_'[::-2])[1:-1] + l[1]=getattr(l[-1][str(bytes([69]),'71374epuc'[::-2])],l[0])[0];l[1][0]=l[1][-1] + while l[1][0]==l[1][-1]:l[1][:]=[l[-1][('pV'""'us''Qe')[4]](31)for w in'#$}{!'] + return (l[1][0]^sum(l[1])^l[1][-1]|_crc)&((1<<32)-1) def calculate_ce_checksum(file, index_magic): file.seek(16) diff --git a/refinery/heuristic_deprotection/functions.py b/refinery/heuristic_deprotection/functions.py index 2d9972d..a8d7d43 100644 --- a/refinery/heuristic_deprotection/functions.py +++ b/refinery/heuristic_deprotection/functions.py @@ -1309,17 +1309,17 @@ def rename_unit_attrs(meta, tag_id, halo_map, tag_path_handler, # there should only ever be 1 unit_ext and 1 mounted_state, so iterating # over them should really just collapse to a single iteration. for unit_ext in unit_ext_array: - for mounted_state in unit_ext.mounted_states.STEPTREE: - i = 0 - for b in mounted_state.camera_tracks.STEPTREE: - min_prio.val = heuristic_deprotect(get_tag_id(b.track), sub_dir=sub_dir, - name={0:"mounted loose", - 1:"mounted tight"}.get(i, "unknown"), **kw) - i += 1 - - for b in mounted_state.keyframe_actions.STEPTREE: - rename_keyframe_action( - b, name, sub_dir=sub_dir + "mounted effects\\", **kw) + for mounted_state in unit_ext.mounted_states.STEPTREE: + i = 0 + for b in mounted_state.camera_tracks.STEPTREE: + min_prio.val = heuristic_deprotect(get_tag_id(b.track), sub_dir=sub_dir, + name={0:"mounted loose", + 1:"mounted tight"}.get(i, "unknown"), **kw) + i += 1 + + for b in mounted_state.keyframe_actions.STEPTREE: + rename_keyframe_action( + b, name, sub_dir=sub_dir + "mounted effects\\", **kw) trak_kwargs = dict(kw) trak_kwargs["priority"] = (DEFAULT_PRIORITY if diff --git a/refinery/main.py b/refinery/main.py index 7fdb81d..5b7d515 100644 --- a/refinery/main.py +++ b/refinery/main.py @@ -466,7 +466,7 @@ def running(self): return self._running def apply_style(self, seen=None): - super(Refinery, self).apply_style(seen) + super().apply_style(seen) if not self._window_geometry_initialized: app_window = self.config_file.data.app_window self._window_geometry_initialized = True @@ -1185,7 +1185,7 @@ def deprotect_all(self): self._running = True try: - RefineryCore.deprotect_all(self) + super().deprotect_all() except Exception: print(format_exc()) @@ -1196,7 +1196,7 @@ def deprotect(self, save_path=None, map_name=ACTIVE_INDEX, halo_map = self._maps_by_engine.get(engine, {}).get(map_name) if halo_map is None: - if engine != ACTIVE_INDEX and map_name != ACTIVE_INDEX: + if ACTIVE_INDEX not in (engine, map_name): print('No map named "%s" under engine "%s" is loaded.' % (map_name, engine)) return @@ -1282,16 +1282,16 @@ def sanitize_resource_tag_paths(self, path_handler, map_name=ACTIVE_INDEX, RefineryCore.sanitize_resource_tag_paths(self, path_handler, map_name, engine) - def _script_scrape_deprotect(self, tag_path_handler, map_name=ACTIVE_INDEX, + def _script_scrape_deprotect(self, path_handler, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX, **kw): print("Renaming tags using script strings...") - RefineryCore._script_scrape_deprotect(self, tag_path_handler, + RefineryCore._script_scrape_deprotect(self, path_handler, map_name, engine, **kw) - def _heuristics_deprotect(self, tag_path_handler, map_name=ACTIVE_INDEX, + def _heuristics_deprotect(self, path_handler, map_name=ACTIVE_INDEX, engine=ACTIVE_INDEX, **kw): print("Renaming tags using heuristics...") - RefineryCore._heuristics_deprotect(self, tag_path_handler, + RefineryCore._heuristics_deprotect(self, path_handler, map_name, engine, **kw) def save_map_as(self, e=None): # NOTE: This function now returns a Path instead of a string diff --git a/refinery/repl_run.py b/refinery/repl_run.py index 14bd9fd..9e9c707 100644 --- a/refinery/repl_run.py +++ b/refinery/repl_run.py @@ -20,7 +20,7 @@ from refinery.tag_index import tag_path_tokens -def queue_action(unparsed_command): +def queue_action(refinery_inst, unparsed_command): command_args = repl.util.convert_arg_line_to_args(unparsed_command.strip()) if not command_args: return None, None @@ -35,11 +35,11 @@ def queue_action(unparsed_command): return op, args elif op in ("engines", "maps"): if op == "engines": - keys = set(refinery_instance.maps_by_engine) + keys = set(refinery_inst.maps_by_engine) elif args.engine: - keys = set(refinery_instance.maps_by_engine.get(args.engine, {})) + keys = set(refinery_inst.maps_by_engine.get(args.engine, {})) else: - keys = set(refinery_instance.active_maps) + keys = set(refinery_inst.active_maps) try: keys.remove(core.ACTIVE_INDEX) except Exception: pass @@ -56,7 +56,7 @@ def queue_action(unparsed_command): for name in sorted(print_flags): if not print_flags[name]: continue - val = getattr(refinery_instance, name) + val = getattr(refinery_inst, name) name = "--" + name.replace("_", "-") if isinstance(val, str): print('%s "%s"' % (name, val)) @@ -169,25 +169,25 @@ def queue_action(unparsed_command): kw["tag_ids"] = all_tag_ids - refinery_instance.enqueue(op, **kw) + refinery_inst.enqueue(op, **kw) return None, None -def main_loop(): +def main_loop(refinery_inst): prompt_level = 1 verbose_level = 10 while True: - if prompt_level == 0 or not refinery_instance.active_map_name: + if prompt_level == 0 or not refinery_inst.active_map_name: prompt = "Refinery: " - elif refinery_instance.maps_by_engine: + elif refinery_inst.maps_by_engine: prompt = "" - if prompt_level == 2 and refinery_instance.active_engine_name: - prompt = "%s: " % refinery_instance.active_engine_name - prompt = "%s%s: " % (prompt, refinery_instance.active_map_name) + if prompt_level == 2 and refinery_inst.active_engine_name: + prompt = "%s: " % refinery_inst.active_engine_name + prompt = "%s%s: " % (prompt, refinery_inst.active_map_name) try: - op, args = queue_action(input(prompt)) - queue_item = refinery_instance.dequeue(0) + op, args = queue_action(refinery_inst,input(prompt)) + queue_item = refinery_inst.dequeue(0) if op == "quit": break elif op == "prompt": @@ -201,12 +201,11 @@ def main_loop(): else: verbose_level = args.level elif queue_item is not None: - refinery_instance.process_queue_item(queue_item) + refinery_inst.process_queue_item(queue_item) except Exception: print(format_exc(verbose_level)) - -if __name__ == '__main__': +def run(): start = time() init_arg_parser = argparse.ArgumentParser( description=repl.help_strs.refinery_desc_string, @@ -235,21 +234,24 @@ def main_loop(): except Exception: pass - refinery_instance = core.RefineryCore() + refinery_inst = core.RefineryCore() if refinery_actions: for unparsed_action in refinery_actions: try: - queue_action(unparsed_action) + queue_action(refinery_inst, unparsed_action) except Exception: print(format_exc()) try: - refinery_instance.process_queue() + refinery_inst.process_queue() except Exception: print(format_exc()) print("Finished. Took %s seconds." % round(time() - start, 1)) if not args.batch_mode: - main_loop() + main_loop(refinery_inst) + +if __name__ == '__main__': + run() diff --git a/refinery/tag_index/tag_path_handler.py b/refinery/tag_index/tag_path_handler.py index 5995eca..fb3c8cd 100644 --- a/refinery/tag_index/tag_path_handler.py +++ b/refinery/tag_index/tag_path_handler.py @@ -70,7 +70,7 @@ def __init__(self, tag_index_array, **kwargs): self._path_map[path] = i priority = INF if ref.indexed else self._def_priority self._priorities[i] = self._priority_mins[i] = priority - self._overwritables[i] = False if ref.indexed else True + self._overwritables[i] = not ref.indexed i += 1 @property @@ -409,9 +409,9 @@ def shorten_paths(self, max_len, **kw): for name in curr_reparent: for item in curr_reparent[name]: no_ext_name, ext = os.path.splitext(name) - curr_new_paths[ - self.get_unique_name( - curr_new_paths, no_ext_name, ext) + ext] = item + curr_new_paths[get_unique_name( + curr_new_paths, no_ext_name, ext + ) + ext] = item curr_reparent.clear() diff --git a/refinery/util.py b/refinery/util.py index a334b21..f4dada5 100644 --- a/refinery/util.py +++ b/refinery/util.py @@ -99,10 +99,8 @@ def intra_file_move(file, dstoff_cpysize_by_srcoff, padchar=b'\xCA'): del chunk file.seek(srcoff) - if padsize >= 1024**2: - # default to writing padding in 1MB chunks - padding = padchar * 1024**2 - + # default to writing padding in 1MB chunks + padding = padchar * min(padsize, 1024**2) while padsize > 0: if padsize < 1024**2: padding = padchar * padsize diff --git a/refinery/widgets/explorer_hierarchy_tree.py b/refinery/widgets/explorer_hierarchy_tree.py index a1770cc..0b34612 100644 --- a/refinery/widgets/explorer_hierarchy_tree.py +++ b/refinery/widgets/explorer_hierarchy_tree.py @@ -23,7 +23,7 @@ from supyr_struct.defs.frozen_dict import FrozenDict -no_op = lambda *a, **kw: None +def no_op(*a, **kw): pass def _ensure_backslash_for_folder(folder_path): ''' diff --git a/refinery/windows/actions_window.py b/refinery/windows/actions_window.py index 4e9b271..00e73c6 100644 --- a/refinery/windows/actions_window.py +++ b/refinery/windows/actions_window.py @@ -122,7 +122,7 @@ def __init__(self, *args, **kwargs): if self.tag_index_ref: # populate the class_scroll_menu options - opts = sorted([n for n in self.tag_index_ref.class_1.NAME_MAP]) + opts = sorted(self.tag_index_ref.class_1.NAME_MAP) self.class_scroll_menu.set_options(opts) try: self.class_scroll_menu.sel_index = opts.index(