add comments, fix for blindDeserialize (#504)

Author: barnjamin

core/base/src/utils/amount.ts

@@ -170,10 +170,11 @@ export function display(amount: Amount, precision?: number): string {
 export function whole(amount: Amount): number {
   return Number(display(amount));
 }
+
 /**
- *
- * @param amount
- * @param decimals
+ * fmt formats a bigint amount to a string with the given number of decimals
+ * @param amount bigint amount
+ * @param decimals number of decimals
  */
 export function fmt(amount: bigint, decimals: number): string {
   return display(fromBaseUnits(amount, decimals));

core/base/src/utils/encoding.ts

@@ -10,8 +10,11 @@ const isHexRegex = /^(?:0x)?[0-9a-fA-F]+$/;
 
 /** Base16/Hex encoding and decoding utilities */
 export const hex = {
+  /** check if a string is valid hex */
   valid: (input: string) => isHexRegex.test(input),
+  /** decode a hex string to Uint8Array */
   decode: (input: string) => base16.decode(stripPrefix("0x", input).toUpperCase()),
+  /** encode a string or Uint8Array to hex */
   encode: (input: string | Uint8Array, prefix: boolean = false) => {
     input = typeof input === "string" ? bytes.encode(input) : input;
     return (prefix ? "0x" : "") + base16.encode(input).toLowerCase();
@@ -24,33 +27,44 @@ const isB64Regex = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}
 
 /** Base64 encoding and decoding utilities */
 export const b64 = {
+  /** check if a string is valid base64 */
   valid: (input: string) => isB64Regex.test(input),
+  /** decode a base64 string to Uint8Array */
   decode: base64.decode,
+  /** encode a string or Uint8Array to base64 */
   encode: (input: string | Uint8Array) =>
     base64.encode(typeof input === "string" ? bytes.encode(input) : input),
 };
 
 /** Base58 encoding and decoding utilities */
 export const b58 = {
+  /** decode a base58 string to Uint8Array */
   decode: base58.decode,
+  /** encode a string or Uint8Array to base58 */
   encode: (input: string | Uint8Array) =>
     base58.encode(typeof input === "string" ? bytes.encode(input) : input),
 };
 
 /** BigInt encoding and decoding utilities */
 export const bignum = {
+  /** decode a hex string or bytes to a bigint */
   decode: (input: string | Uint8Array) => {
     if (typeof input !== "string") input = hex.encode(input, true);
     if (input === "" || input === "0x") return 0n;
     return BigInt(input);
   },
+  /** encode a bigint as a hex string */
   encode: (input: bigint, prefix: boolean = false) => bignum.toString(input, prefix),
+  /** convert a bigint to a hexstring */
   toString: (input: bigint, prefix: boolean = false) => {
     let str = input.toString(16);
     str = str.length % 2 === 1 ? (str = "0" + str) : str;
     if (prefix) return "0x" + str;
     return str;
   },
+  /** convert a bigint or number to bytes, 
+   *   optionally specify length, left padded with 0s to length
+   */
   toBytes: (input: bigint | number, length?: number) => {
     const b = hex.decode(bignum.toString(typeof input === "number" ? BigInt(input) : input));
     if (!length) return b;
@@ -60,14 +74,19 @@ export const bignum = {
 
 /** Uint8Array encoding and decoding utilities */
 export const bytes = {
+  /** encode a string to Uint8Array */
   encode: (value: string): Uint8Array => new TextEncoder().encode(value),
+  /** decode a Uint8Array to string */
   decode: (value: Uint8Array): string => new TextDecoder().decode(value),
+  /** compare two Uint8Arrays for equality */
   equals: (lhs: Uint8Array, rhs: Uint8Array): boolean =>
     lhs.length === rhs.length && lhs.every((v, i) => v === rhs[i]),
+  /** pad a Uint8Array to a given length, optionally specifying padding direction */
   zpad: (arr: Uint8Array, length: number, padStart: boolean = true): Uint8Array =>
     padStart
       ? bytes.concat(new Uint8Array(length - arr.length), arr)
       : bytes.concat(arr, new Uint8Array(length - arr.length)),
+  /** concatenate multiple Uint8Arrays into a single Uint8Array */
   concat: (...args: Uint8Array[]): Uint8Array => {
     const length = args.reduce((acc, curr) => acc + curr.length, 0);
     const result = new Uint8Array(length);

core/base/src/utils/layout/discriminate.ts

@@ -298,6 +298,8 @@ function buildAscendingBounds(sortedBounds: readonly (readonly [Bounds, LayoutIn
 function generateLayoutDiscriminator(
   layouts: readonly Layout[]
 ): [boolean, (encoded: BytesType) => readonly LayoutIndex[]] {
+  //for debug output:
+  // const candStr = (candidate: Bitset) => candidate.toString(2).padStart(layouts.length, '0');
 
   if (layouts.length === 0)
     throw new Error("Cannot discriminate empty set of layouts");
@@ -357,6 +359,11 @@ function generateLayoutDiscriminator(
       for (let j = 0; j < serialized.length; ++j)
         fixedKnownBytes[offset + j]!.push([serialized[j]!, i]);
 
+  //debug output:
+  // console.log("fixedKnownBytes:",
+  //   fixedKnownBytes.map((v, i) => v.length > 0 ? [i, v] : undefined).filter(v => v !== undefined)
+  // );
+
   let bestBytes = [];
   for (const [bytePos, fixedKnownByte] of fixedKnownBytes.entries()) {
     //the number of layouts with a given size is an upper bound on the discriminatory power of
@@ -376,7 +383,7 @@ function generateLayoutDiscriminator(
       distinctValues.set(byteVal, distinctValues.get(byteVal)! | 1n << BigInt(candidate));
     }
 
-    let power = count(lwba);
+    let power = layouts.length - Math.max(count(anyValueLayouts), count(outOfBoundsLayouts));
     for (const layoutsWithValue of distinctValues.values()) {
       //if we find the byte value associated with this set of layouts, we can eliminate
       //  all other layouts that don't have this value at this position and all layouts
@@ -385,6 +392,17 @@ function generateLayoutDiscriminator(
       power = Math.min(power, curPower);
     }
 
+    //debug output:
+    // console.log(
+    //   "bytePos:", bytePos,
+    //   "\npower:", power,
+    //   "\nfixedKnownByte:", fixedKnownByte,
+    //   "\nlwba:", candStr(lwba),
+    //   "\nanyValueLayouts:", candStr(anyValueLayouts),
+    //   "\noutOfBoundsLayouts:", candStr(outOfBoundsLayouts),
+    //   "\ndistinctValues:", new Map([...distinctValues].map(([k, v]) => [k, candStr(v)]))
+    // );
+
     if (power === 0)
       continue;
 
@@ -521,14 +539,31 @@ function generateLayoutDiscriminator(
     throw new Error("Implementation error in layout discrimination algorithm");
   };
 
+  //debug output:
+  // console.log("strategies:", JSON.stringify(
+  //     new Map([...strategies].map(([cands, strat]) => [
+  //       candStr(cands),
+  //       typeof strat === "string"
+  //         ? strat
+  //         : [
+  //           strat[0], //bytePos
+  //           candStr(strat[1]), //outOfBoundsLayouts
+  //           new Map([...strat[2]].map(([value, cands]) => [value, candStr(cands)]))
+  //         ]
+  //     ]
+  //   ))
+  // ));
+
   return [distinguishable, (encoded: BytesType) => {
     let candidates = allLayouts;
 
-    for (
-      let strategy = strategies.get(candidates)!;
-      strategy !== "indistinguishable";
-      strategy = strategies.get(candidates) ?? findSmallestSuperSetStrategy(candidates)
-    ) {
+    let strategy = strategies.get(candidates)!;
+    while (strategy !== "indistinguishable") {
+      //debug output:
+      // console.log(
+      //   "applying strategy", strategy,
+      //   "\nfor remaining candidates:", candStr(candidates)
+      // );
       if (strategy === "size")
         candidates &= layoutsWithSize(encoded.length);
       else {
@@ -546,9 +581,13 @@ function generateLayoutDiscriminator(
       }
 
       if (count(candidates) <= 1)
-        return bitsetToArray(candidates);
+        break;
+
+      strategy = strategies.get(candidates) ?? findSmallestSuperSetStrategy(candidates)
     }
 
+    //debug output:
+    // console.log("final candidates", candStr(candidates));
     return bitsetToArray(candidates);
   }];
 }

core/definitions/__tests__/blindDeserialize.ts

@@ -0,0 +1,34 @@
+import { encoding } from "@wormhole-foundation/sdk-base";
+import {
+  blindDeserializePayload,
+  deserialize,
+  deserializePayload,
+  exhaustiveDeserialize,
+} from "../src/index.js";
+import "../src/protocols/index.js";
+
+const cases = [
+  "AQAAAAQNAEI6Ol1ax5h7zmg2cv9pxdznRm1grHS9RL1JLj/I8HJ4Akb1nwXc+zMcOFdAiMASnEwUYpxyBwgJF5QHMS0hXwQBAvTwT11h4/z9dsCEW1pDNaDon0B9aQ6o9/S3zjUnCcuEEgxSwp5hXRjUJ4lds48rpDSfAlMzF/RB3x5+p1NxBDcBA5CMs0VxzRxgAxB+zy9Hn263MEEn5c98Lky8604/RI56C/O/mtoZrvOHtM3ln0yEapeqBcNMvl5L0CpuJ4xtWFsABCVTI/T6ou8+EHP4LC6PBCt/yjEr/QEVJsMx21eFT0y2Z86cBQCw6LmA1ER179Z9WO69FyGPmtHxxHovLi3+sZUABrnxQ0eXTkt7OjiHx/yj3rE6xqzwHqdGQzUBN5SgWCFhTxxfA9Kmt3hlJ6bRnfpR9QMeLxc5tlZCfQGODIQrra8BBwa71PjV61JJaipwAjA/pUsG9fI1qeX5CQknohKbpGUMUb1MZixB8YUMGOsQbBidNh67BwHe0kX7ofh2Y1hYxp0ACEsz46+CAuELDC3Q8jRarQLW20cAWWsRmjjXxSyqOrEDUdSHdJHe1dvzRL1LgD7gsOX7cGuuY/6USFFhPl7cCUAACXm7gAjTn194YTNUdWnZtgNyP+V02tr9a5kcM1xb6D7AcCMW6NbUnjby66L1RycPkyXGoITkGjXvsVxJFxcotzcBCrpiwyBJNx+XM1GEGAmbYYd5Fw6y69L0q9RTk5oNtOmIE2ssvW+/ZMbaLPB3fWf37fYzulNL0YU/u7+JpE+eSuIADSYc1vb3lV+P1rFBgVMlWnnpgpE3UJPq5ZbRHsiP4aLCQUAGkQMl/rtyAyh64fZGA9eRhRRWV76KiwY9CMaWMEYADmhp08yb3cBPvzpDlc1MQ52UKIgHU94cd1dmP27xJbrSJZiwSxL2HiJZdtHZ1EnEp/LBnIrzrTT5w/qwLa7T/0UBD9BzsGb9ekj0TrRLdYdN4PgAzBPV/M8XfFzW2Ex4hhg/BAnd4pH32FN8dGwueGciKr6/z5/ORV8UTCgUDcgTziEBEu7MNJ5xy2J5OWK9ZsKX4UnC29zaTCGIwfx/9bKctOAhIDLn+IIDEwPWQszd2mx4z1IeT0AhHs9Jpuf80uJGuu4BZi0+vA2QAAAADgAAAAAAAAAAAAAAAHlt/2108+JwYLcSVf5Re/sjyT7tAAAAAAACudkBAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAX14QAAAAAAAAAAAAAAAAAwCqqObIj/o0KDlxPJ+rZCDx1bMIAAg9A62uZJ8TprduI7DEsB7ndr+gKgbGMF8Hnc05bod8YAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
+  "AQAAAAQNAOk1OzNZ9DfkOsrjn2jpzz01k8MMqxEeS+NtyMPySJw6OhFAvz76Buk+O/hec/ifJ3m1y0joY1DUeru6sJ+PMF0BAunCoXhMBfluALeB3duNBsgwG12ZfuxOH33FPgWq98qnTh9xFUd6z3DPXb7CJA4oFYm9nYEZtfVTkYs6WEjYCc4BA+ZpLzYmZ46XLxRtHdggukDaK+gHiVyMxoL1BOb0OmCGIwV07bMMk1pG5B0IQwRDf68WDhe3PYQDjuvze9Gkw0gABANQu3VOyPj9tWTbC5jWglfzwgQ98JOau0liKfwF/GYiRV3r+7JPiTXhmmk6J+j7b8WGNidGqrRsomcTD2V1/TsABtmKBYXbmuSds3OWE9ZxubU8cw2h9gv6SHxeTpBWIwHPaiHggfWEznJcUkWByjWZ+nOA+kAAFjhoAlyjsBZkgRMBB7FezvEAnCYCneStFT3AolKVifA8mIBfiKxQpiX9mJkTGatmhMpi8Tp+YXgXYMIk3wCxALg/ZwMu3x76KkIEiIwBCCrfe4hZexgLJMZcReLKvvVfJmcThyEEk9aF/sM5pqaRBuAzG0jKvyN9oHYdq7p9qSzOYuJMbKneJ1ERAOdiWrEACVfoGUdlV4p3etUt32Lr8zUf1NtEuE47UtIUBbMRTZxgNqdP3wZIkrOFoOj5k6+2XTVNIRqTxiEMJohjvVtVms4ADSkByD3p1QM1A8GeGslv+wfsTDhq6cI0MRiGouS277uiXMTZ9MHPZ6VVyIb4gppeZm+A7xzJqp6584oPV+LtqsYAD9yLQKItaBTEWXpPzlExi38ztxCe2Soio+udkOIcbq7mdMx/UWvSpQf04/jhNInBpUEgI4GbSlxrqke9ue61p6cAECaPI5R1x7avvXlWP3hx0V08Jz40kqtf6x7M8ZSb+2gRGQTYdRaY5ZtVzOyq0Nz7+F2XYVZ0tGskcTIABikvWIwBEZJGZDxtl0iB1oyM9H3W0ErJ8Sjdjf7pKbrorQ8JnyqmBWws0zWI2Y4wXdadA5kg8kH6vpvsk8JaJ9vvdsbaEGkBEtVhl/Dg4KMluxuPch+EOwEJ8eY+FFm0jxZeHW65zwX6G3QvqlBsGsFbWeXsWrOXdIhhfSI3uUO3WrU3hb08RCIBZi0/JAABaM8AFczO6yk0j3G90i/+9DoqGcH1teF8XMpUEVKRIBgmcq3lAAAAAAAB50YAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADv68CkAAAAAAAAAAAAAAAAisdqUcyVDZgi1ouD/hrZezLNWA0ABAAAAAAAAAAAAAAAAP5/vLIafffsHZUHsJ1mYOEqD6EWAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
+];
+
+describe("Blind Deserialize", function () {
+  it("Should deserialize a blind message", function () {
+    for (const c of cases) {
+      const decoded = encoding.b64.decode(c);
+      const vaa = deserialize("Uint8Array", decoded);
+      const actual = deserializePayload("TokenBridge:Transfer", vaa.payload);
+      expect(actual).toBeDefined();
+
+      console.time("exhaustive");
+      const result = exhaustiveDeserialize(vaa.payload);
+      expect(result).toHaveLength(1);
+      console.timeEnd("exhaustive");
+
+      console.time("blind");
+      const blind = blindDeserializePayload(vaa.payload);
+      expect(blind).toHaveLength(1);
+      console.timeEnd("blind");
+    }
+  });
+});

core/definitions/src/vaa/create.ts

@@ -25,6 +25,13 @@ type DynamicProperties<PL extends PayloadLiteral> = LayoutToType<
   DynamicItemsOfLayout<[...typeof baseLayout, PayloadLiteralToPayloadItemLayout<PL>]>
 >;
 
+/**
+ * Create a VAA from a payload literal and a set of dynamic properties.
+ * @param payloadLiteral The payload literal to create a VAA for.
+ * @param vaaData The dynamic properties to include in the VAA.
+ * @returns A VAA with the given payload literal and dynamic properties.
+ * @throws If the dynamic properties do not match the payload literal.
+ */
 export function createVAA<PL extends PayloadLiteral>(
   payloadLiteral: PL,
   vaaData: DynamicProperties<PL>,

core/definitions/src/vaa/functions.ts

@@ -38,6 +38,12 @@ export function payloadLiteralToPayloadItemLayout<PL extends PayloadLiteral>(pay
   } as PayloadLiteralToPayloadItemLayout<PL>;
 }
 
+/**
+ * serialize a VAA to a Uint8Array
+ * @param vaa the VAA to serialize
+ * @returns a Uint8Array representation of the VAA
+ * @throws if the VAA is not valid
+ */
 export function serialize<PL extends PayloadLiteral>(vaa: VAA<PL>): Uint8Array {
   const layout = [
     ...baseLayout,
@@ -46,6 +52,13 @@ export function serialize<PL extends PayloadLiteral>(vaa: VAA<PL>): Uint8Array {
   return serializeLayout(layout, vaa as unknown as LayoutToType<typeof layout>);
 }
 
+/**
+ * serialize a VAA payload to a Uint8Array
+ *
+ * @param payloadLiteral The payload literal to use for serialization
+ * @param payload  The dynamic properties to include in the payload
+ * @returns a Uint8Array representation of the VAA Payload
+ */
 export function serializePayload<PL extends PayloadLiteral>(
   payloadLiteral: PL,
   payload: Payload<PL>,
@@ -136,6 +149,15 @@ export function payloadDiscriminator<
 
 type ExtractLiteral<T> = T extends PayloadDiscriminator<infer LL> ? LL : T;
 
+/**
+ * deserialize a VAA from a Uint8Array
+ *
+ * @param payloadDet The payload literal or discriminator to use for deserialization
+ * @param data the data to deserialize
+ * @returns a VAA object with the given payload literal or discriminator
+ * @throws if the data is not a valid VAA
+ */
+
 export function deserialize<T extends PayloadLiteral | PayloadDiscriminator>(
   payloadDet: T,
   data: Byteish,
@@ -188,6 +210,15 @@ type DeserializePayloadReturn<T> = T extends infer PL extends PayloadLiteral
   ? DeserializedPair<LL>
   : never;
 
+/**
+ * deserialize a payload from a Uint8Array
+ *
+ * @param payloadDet the payload literal or discriminator to use for deserialization
+ * @param data the data to deserialize
+ * @param offset the offset to start deserializing from
+ * @returns the deserialized payload
+ * @throws if the data is not a valid payload
+ */
 export function deserializePayload<T extends PayloadLiteral | PayloadDiscriminator>(
   payloadDet: T,
   data: Byteish,
@@ -211,6 +242,42 @@ export function deserializePayload<T extends PayloadLiteral | PayloadDiscriminat
   })() as DeserializePayloadReturn<T>;
 }
 
+/**
+ * Attempt to deserialize a payload from a Uint8Array using all registered layouts
+ *
+ * @param data the data to deserialize
+ * @returns an array of all possible deserialized payloads
+ * @throws if the data is not a valid payload
+ */
+export const exhaustiveDeserialize = (() => {
+  const rebuildDiscrimininator = () => {
+    const layoutLiterals = Array.from(payloadFactory.keys());
+    const layouts = layoutLiterals.map((l) => payloadFactory.get(l)!);
+    return [layoutLiterals, layoutDiscriminator(layouts, true)] as const;
+  };
+
+  let layoutLiterals = [] as LayoutLiteral[];
+
+  return (data: Byteish): readonly DeserializedPair[] => {
+    if (payloadFactory.size !== layoutLiterals.length) [layoutLiterals] = rebuildDiscrimininator();
+
+    const candidates = layoutLiterals;
+    return candidates.reduce((acc, literal) => {
+      try {
+        acc.push([literal, deserializePayload(literal!, data)] as DeserializedPair);
+      } catch {}
+      return acc;
+    }, [] as DeserializedPair[]);
+  };
+})();
+
+/**
+ * Blindly deserialize a payload from a Uint8Array
+ *
+ * @param data the data to deserialize
+ * @returns an array of all possible deserialized payloads
+ * @throws if the data is not a valid payload
+ */
 export const blindDeserializePayload = (() => {
   const rebuildDiscrimininator = () => {
     const layoutLiterals = Array.from(payloadFactory.keys());
@@ -226,7 +293,6 @@ export const blindDeserializePayload = (() => {
       [layoutLiterals, discriminator] = rebuildDiscrimininator();
 
     if (typeof data === "string") data = encoding.hex.decode(data);
-
     const candidates = discriminator(data).map((c) => layoutLiterals[c]);
     return candidates.reduce((acc, literal) => {
       try {