Various fixes and additions

CHANGELOG.MD

@@ -4,6 +4,24 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.9.0]
+### Changed
+ - exception thrown when extracting stubbs models
+ - reduced stacktrace output when tag extraction fails(unless RefineryCore.debug == True)
+ - Heuristic deprotection changes:
+  - better descriptions of what disabling safe-mode settings will do
+  - significant speed increase(most maps deprotect in 15 seconds or less)
+  - renaming logic improved to select better names
+  - added ability to configure a root directory to put all renamed tags in
+  - added ability to prioritize model names over names taken from weapon/vehicle strings
+  - underscore appended to shader names to allow permutations to be used
+  - spaces in tag names replaced with underscores in most cases
+
+### Added
+ - config setting to control using open-sauce definitions for regular CE maps
+ - config setting to control showing tag ids and metadata offsets in base 16 or base 10
+ - sound playback in tag window(requires pyogg and simpleaudio)
+
 ## [2.6.0]
 ### Changed
  - Only maps with opensauce headers will have their tags treated as OS_V4 on extraction now.

refinery/__init__.py

@@ -12,8 +12,8 @@
 # ##############
 __author__ = "Sigmmma"
 #           YYYY.MM.DD
-__date__ = "2020.03.13"
-__version__ = (2, 6, 0)
+__date__ = "2025.01.18"
+__version__ = (2, 9, 0)
 __website__ = "https://github.com/Sigmmma/refinery"
 __all__ = (
     'defs', 'heuristic_deprotection', 'repl', 'tag_index', 'widgets', 'windows',

refinery/__main__.py

@@ -40,7 +40,7 @@ def main():
             except Exception:
                 pass
         print(exception, file=sys.stderr)
-        return 1;
+        return 1
 
 if __name__ == "__main__":
     main()

refinery/arbytmap_ext.py

@@ -0,0 +1,160 @@
+'''
+This module adds an new format to arbytmap, which allows it
+to read new formats. Specifically it adds A16R16G16B16_F 
+and R9G9B9E5, which is are floating-point formats
+'''
+
+import arbytmap
+import array
+import types
+
+FORMAT_R9G9B9E5_SHAREDEXP = "R9G9B9E5_SHAREDEXP"
+FORMAT_A16R16G16B16F      = "A16R16G16B16F"
+
+MAX_R9G9B9E5_UNPACK_VALUE = 64
+AVG_R9G9B9E5_UNPACK_SCALE = 10
+
+
+def _apply_exponent_scales_to_pixels(pixels, scales):
+    # scale the pixels to floats
+    float_pixels = map(int(511).__rtruediv__, pixels)
+
+    # multiply the float pixels by their scales and return them
+    return list(map(float.__mul__, float_pixels, scales))
+
+
+def unpack_r9g9b9e5(arby, bitmap_index, width, height, depth=1):
+    # unpack the pixels so we can easily manipulate them
+    unpacked_pixels = arby._unpack_raw_4_channel(
+        arby.texture_block[bitmap_index],
+        _R9G9B9E5_OFFSETS, _R9G9B9E5_MASKS,
+        [_UPSCALE_5BIT, _UPSCALE_9BIT, _UPSCALE_9BIT, _UPSCALE_9BIT]
+        )
+
+    # slice the eponents out, and apply map operations to get an exponent
+    # scale for each exponent using the following algorithm:  v = 2^(e-15)
+    exps = map(int(15).__rsub__, unpacked_pixels[0::4])
+    exp_scales = list(map(int(2).__pow__, exps))
+
+    # apply the exponent to each channel
+    channel_values = [(0,)]
+    for i in (1, 2, 3):
+        channel_values.append(_apply_exponent_scales_to_pixels(
+            unpacked_pixels[i::4], exp_scales, 
+            ))
+
+    # NOTE: this part is a hack. we're scaling the color range values
+    #       to take up as much of the [0, 65536] value range as possible
+    average_vals = list(sum(vals)/len(vals) for vals in channel_values)
+    max_val = min(
+        MAX_R9G9B9E5_UNPACK_VALUE,
+        AVG_R9G9B9E5_UNPACK_SCALE * max(average_vals)
+        )
+    max_clamp = types.MethodType(min, max_val)
+    for i in (1, 2, 3):
+        # clamp the values that are just too bright to the max
+        channel_values[i] = map(max_clamp, channel_values[i])
+        # scale the values to the range we defined
+        channel_values[i] = map(float(0xFFff/max_val).__mul__, channel_values[i])
+
+    # create a new array to hold the exponent applied pixels
+    exp_applied_pixels = array.array(
+        unpacked_pixels.typecode, b'\xFF' * (len(unpacked_pixels) * unpacked_pixels.itemsize)
+        )
+    for i in (1, 2, 3):
+        # convert the floats to ints and slice them into
+        # the new array(leaving the alpha white)
+        exp_applied_pixels[i::4] = array.array("H", map(int, channel_values[i]))
+
+    return exp_applied_pixels
+
+
+def unpack_a16r16g16b16_f(arby, bitmap_index, width, height, depth=1):
+    offsets   = arby.channel_offsets
+    masks     = arby.channel_masks
+    upscalers = arby.channel_upscalers
+    chan_ct = len(offsets)
+
+    blank_channels = []
+    for i in range(chan_ct):
+        if len(set(upscalers[i])) == 1:
+            blank_channels.append(i)
+            upscalers[i] = _UPSCALE_16BIT_ERASE
+
+    # convert the pixel channels to uint16's so we can easily manipulate them
+    packed_channels = arby._unpack_raw_4_channel(
+        arby.texture_block[bitmap_index], offsets, masks, upscalers,
+        )
+
+    channel_values = array.array("I", b'\x00' * (4 * len(packed_channels)))
+    # pad the half-floats to regular floats
+    for i, half_float in enumerate(packed_channels):
+        # sign, exponent, and mantissa
+        s = half_float >> 15
+        e = (half_float >> 10) & 0x1F
+        m = half_float & 0x3FF
+
+        if e == 0x1F:
+            e = 0xFF
+        elif e:
+            e += 0x70
+        elif m:
+            e = 0x71
+            while not m & 0x400:
+                m <<= 1
+                e -= 1
+            m &= ~0x400
+
+        channel_values[i] = (s << 31) | (e << 23) | (m << 13)
+
+    channel_values = array.array("f", bytearray(channel_values))
+
+    # clamp the values to the [0.0, 1.0] bounds
+    channel_values = map(types.MethodType(max, 0.0), channel_values)
+    channel_values = map(types.MethodType(min, 1.0), channel_values)
+    # scale the values to the range we defined
+    channel_values = map(float(0x7FFF).__mul__, channel_values)
+
+    # convert the floats to ints and slice them into
+    # the new array(leaving the alpha white)
+    unpacked_pixels = array.array("H", map(int, channel_values))
+
+    # replace any empty channels with their 
+    for chan in blank_channels:
+        if chan == 0:
+            unpacked_pixels[::chan_ct] = array.array(
+                "H", b'\xFF\xFF' * (len(unpacked_pixels) // chan_ct)
+                )
+
+    return unpacked_pixels
+
+
+def pack_r9g9b9e5(arby, unpacked, width, height, depth=1):
+    raise NotImplementedError("Not Implemented")
+
+
+def pack_a16r16g16b16_f(arby, unpacked, width, height, depth=1):
+    raise NotImplementedError("Not Implemented")
+
+
+arbytmap.format_defs.register_format(
+    FORMAT_R9G9B9E5_SHAREDEXP, 1,
+    depths=(9,9,9,5), offsets=(18,9,0,27),
+    unpacker=unpack_r9g9b9e5, packer=pack_r9g9b9e5
+    )
+
+arbytmap.format_defs.register_format(
+    FORMAT_A16R16G16B16F, 1,
+    depths=(16,16,16,16), offsets=(16,32,48,0),
+    unpacker=unpack_a16r16g16b16_f, packer=pack_a16r16g16b16_f
+    )
+
+_R9G9B9E5_OFFSETS = array.array(
+    "B", arbytmap.format_defs.CHANNEL_OFFSETS[FORMAT_R9G9B9E5_SHAREDEXP]
+    )
+_R9G9B9E5_MASKS = array.array(
+    "H", arbytmap.format_defs.CHANNEL_MASKS[FORMAT_R9G9B9E5_SHAREDEXP]
+    )
+_UPSCALE_9BIT = array.array("H", list(range(2**9)))
+_UPSCALE_5BIT = array.array("H", list(range(2**5)))
+_UPSCALE_16BIT_ERASE = array.array("H", b'\x00\x00' * (2**16))

refinery/constants.py

@@ -24,35 +24,41 @@
     (INVALID, "NONE",)
     )
 
-H1_TAG_SUPERCLASSES = FrozenDict(
-    shader_environment=("shader", "NONE"),
-    shader_model=("shader", "NONE"),
-    shader_transparent_generic=("shader", "NONE"),
-    shader_transparent_chicago=("shader", "NONE"),
-    shader_transparent_chicago_extended=("shader", "NONE"),
-    shader_plasma=("shader", "NONE"),
-    shader_meter=("shader", "NONE"),
-    shader_water=("shader", "NONE"),
-    shader_glass=("shader", "NONE"),
-
-    biped=("unit", "object"),
-    vehicle=("unit", "object"),
-
-    weapon=("item", "object"),
-    equipment=("item", "object"),
-    garbage=("item", "object"),
-
-    device_machine=("device", "object"),
-    device_control=("device", "object"),
-    device_light_fixture=("device", "object"),
-
-    projectile=("object", "NONE"),
-    scenery=("object", "NONE"),
-    placeholder=("object", "NONE"),
-    sound_scenery=("object", "NONE"),
+H1_SHADER_TAG_CLASSES = frozenset((
+    "shader_environment",
+    "shader_model",
+    "shader_transparent_generic",
+    "shader_transparent_chicago",
+    "shader_transparent_chicago_extended",
+    "shader_plasma",
+    "shader_meter",
+    "shader_water",
+    "shader_glass",
+    ))
+H1_UNIT_TAG_CLASSES = frozenset((
+    "biped", "vehicle"
+    ))
+H1_ITEM_TAG_CLASSES = frozenset((
+    "weapon", "equipment", "garbage"
+    ))
+H1_DEVICE_TAG_CLASSES = frozenset((
+    "device_machine", "device_control", "device_light_fixture"
+    ))
+H1_OBJECT_TAG_CLASSES = frozenset((
+    "projectile", "scenery", "placeholder", "sound_scenery"
+    ))
 
+H1_TAG_SUPERCLASSES = dict(
     effect_postprocess_generic=("effect_postprocess", "NONE"),
     shader_postprocess_generic=("shader_postprocess", "NONE"),
+    **{cls: ("shader",  "NONE")     for cls in H1_SHADER_TAG_CLASSES},
+    **{cls: ("object",  "NONE")     for cls in H1_OBJECT_TAG_CLASSES},
     )
+
+H1_TAG_SUPERCLASSES.update({cls: ("unit",    "object")   for cls in H1_UNIT_TAG_CLASSES})
+H1_TAG_SUPERCLASSES.update({cls: ("unit",    "object")   for cls in H1_ITEM_TAG_CLASSES})
+H1_TAG_SUPERCLASSES.update({cls: ("unit",    "object")   for cls in H1_DEVICE_TAG_CLASSES})
+
+H1_TAG_SUPERCLASSES = FrozenDict(H1_TAG_SUPERCLASSES)
 
 del FrozenDict