Make serialisation endianness agnostic (#5)

Author: Christoph M. Wintersteiger

CMakeLists.txt

@@ -76,9 +76,9 @@ if(TESTS)
 
     if(CMAKE_BUILD_TYPE STREQUAL "Debug" AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
       target_compile_options(
-        ${NAME} PRIVATE -fsanitize=address -fno-omit-frame-pointer
+        ${NAME} PRIVATE -fsanitize=undefined,address -fno-omit-frame-pointer
       )
-      target_link_options(${NAME} PRIVATE -fsanitize=address)
+      target_link_options(${NAME} PRIVATE -fsanitize=undefined,address)
     endif()
 
     add_test(${NAME} ${NAME})

merklecpp.h

@@ -16,22 +16,6 @@
 #include <stack>
 #include <vector>
 
-#ifdef WIN32
-#  include <stdlib.h>
-#  if BYTE_ORDER == LITTLE_ENDIAN
-#    define htobe32(X) _byteswap_ulong(X)
-#    define be32toh(X) _byteswap_ulong(X)
-#    define htobe64(X) _byteswap_uint64(X)
-#    define be64toh(X) _byteswap_uint64(X)
-#  else
-#    define htobe32(X) (X)
-#    define be32toh(X) (X)
-#    define htobe64(X) (X)
-#    define be64toh(X) (X)
-#  endif
-#  undef max
-#endif
-
 #ifdef HAVE_OPENSSL
 #  include <openssl/sha.h>
 #endif
@@ -63,22 +47,43 @@
 
 namespace merkle
 {
-  inline void serialise_size_t(size_t size, std::vector<uint8_t>& bytes)
+  static inline uint32_t convert_endianness(uint32_t n)
+  {
+    const uint32_t sz = sizeof(uint32_t);
+#if defined(htobe32)
+    // If htobe32 happens to be a macro, use it.
+    return htobe32(n);
+#elif defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN)
+    // Just as fast.
+    uint32_t r = 0;
+    for (size_t i = 0; i < sz; i++)
+      r |= ((n >> (8 * ((sz - 1) - i))) & 0xFF) << (8 * i);
+    return *reinterpret_cast<uint32_t*>(&r);
+#else
+    // A little slower, but works for both endiannesses.
+    uint8_t r[8];
+    for (size_t i = 0; i < sz; i++)
+      r[i] = (n >> (8 * ((sz - 1) - i))) & 0xFF;
+    return *reinterpret_cast<uint32_t*>(&r);
+#endif
+  }
+
+  static inline void serialise_uint64_t(uint64_t n, std::vector<uint8_t>& bytes)
   {
-    size_t size_be = htobe64(size);
-    uint8_t* size_bytes = (uint8_t*)&size_be;
-    for (size_t i = 0; i < sizeof(size_t); i++)
-      bytes.push_back(size_bytes[i]);
+    size_t sz = sizeof(uint64_t);
+    bytes.reserve(bytes.size() + sz);
+    for (uint64_t i = 0; i < sz; i++)
+      bytes.push_back((n >> (8 * (sz - i - 1))) & 0xFF);
   }
 
-  inline size_t deserialise_size_t(
+  static inline uint64_t deserialise_uint64_t(
     const std::vector<uint8_t>& bytes, size_t& index)
   {
-    size_t result = 0;
-    uint8_t* result_bytes = (uint8_t*)&result;
-    for (size_t i = 0; i < sizeof(size_t); i++)
-      result_bytes[i] = bytes[index++];
-    return be64toh(result);
+    uint64_t r = 0;
+    uint64_t sz = sizeof(uint64_t);
+    for (uint64_t i = 0; i < sz; i++)
+      r |= static_cast<uint64_t>(bytes[index++]) << (8 * (sz - i - 1));
+    return r;
   }
 
   /// @brief Template for fixed-size hashes
@@ -163,7 +168,7 @@ namespace merkle
       std::string r(num_chars, '_');
       for (size_t i = 0; i < num_bytes; i++)
         snprintf(
-          r.data() + 2 * i,
+          const_cast<char*>(r.data() + 2 * i),
           num_chars + 1 - 2 * i,
           lower_case ? "%02x" : "%02X",
           bytes[i]);
@@ -354,9 +359,9 @@ namespace merkle
     {
       MERKLECPP_TRACE(MERKLECPP_TOUT << "> PathT::serialise " << std::endl);
       _leaf.serialise(bytes);
-      serialise_size_t(_leaf_index, bytes);
-      serialise_size_t(_max_index, bytes);
-      serialise_size_t(elements.size(), bytes);
+      serialise_uint64_t(_leaf_index, bytes);
+      serialise_uint64_t(_max_index, bytes);
+      serialise_uint64_t(elements.size(), bytes);
       for (auto& e : elements)
       {
         e.hash.serialise(bytes);
@@ -372,9 +377,9 @@ namespace merkle
       MERKLECPP_TRACE(MERKLECPP_TOUT << "> PathT::deserialise " << std::endl);
       elements.clear();
       _leaf.deserialise(bytes, position);
-      _leaf_index = deserialise_size_t(bytes, position);
-      _max_index = deserialise_size_t(bytes, position);
-      size_t num_elements = deserialise_size_t(bytes, position);
+      _leaf_index = deserialise_uint64_t(bytes, position);
+      _max_index = deserialise_uint64_t(bytes, position);
+      size_t num_elements = deserialise_uint64_t(bytes, position);
       for (size_t i = 0; i < num_elements; i++)
       {
         HashT<HASH_SIZE> hash(bytes, position);
@@ -452,7 +457,8 @@ namespace merkle
     /// @brief Convert a path to a string
     /// @param num_bytes The maximum number of bytes to convert
     /// @param lower_case Enables lower-case hex characters
-    std::string to_string(size_t num_bytes = HASH_SIZE, bool lower_case = true) const
+    std::string to_string(
+      size_t num_bytes = HASH_SIZE, bool lower_case = true) const
     {
       std::stringstream stream;
       stream << _leaf.to_string(num_bytes);
@@ -1201,8 +1207,9 @@ namespace merkle
     {
       MERKLECPP_TRACE(MERKLECPP_TOUT << "> serialise " << std::endl;);
 
-      serialise_size_t(leaf_nodes.size() + uninserted_leaf_nodes.size(), bytes);
-      serialise_size_t(num_flushed, bytes);
+      serialise_uint64_t(
+        leaf_nodes.size() + uninserted_leaf_nodes.size(), bytes);
+      serialise_uint64_t(num_flushed, bytes);
       for (auto& n : leaf_nodes)
         n->hash.serialise(bytes);
       for (auto& n : uninserted_leaf_nodes)
@@ -1243,8 +1250,8 @@ namespace merkle
         (to < min_index() || max_index() < to) || from > to)
         throw std::runtime_error("invalid leaf indices");
 
-      serialise_size_t(to - from + 1, bytes);
-      serialise_size_t(from, bytes);
+      serialise_uint64_t(to - from + 1, bytes);
+      serialise_uint64_t(from, bytes);
       for (size_t i = from; i <= to; i++)
         leaf(i).serialise(bytes);
 
@@ -1294,8 +1301,8 @@ namespace merkle
       walk_stack.clear();
       _root = nullptr;
 
-      size_t num_leaf_nodes = deserialise_size_t(bytes, position);
-      num_flushed = deserialise_size_t(bytes, position);
+      size_t num_leaf_nodes = deserialise_uint64_t(bytes, position);
+      num_flushed = deserialise_uint64_t(bytes, position);
 
       leaf_nodes.reserve(num_leaf_nodes);
       for (size_t i = 0; i < num_leaf_nodes; i++)
@@ -1832,7 +1839,7 @@ namespace merkle
     uint32_t cws[64] = {0};
 
     for (int i=0; i < 16; i++)
-      cws[i] = be32toh(((int32_t *)block)[i]);
+      cws[i] = convert_endianness(((int32_t *)block)[i]);
 
     for (int i = 16; i < 64; i++) {
       uint32_t t16 = cws[i - 16];
@@ -1864,7 +1871,7 @@ namespace merkle
     }
 
     for (int i=0; i < 8; i++)
-      ((uint32_t*)out.bytes)[i] = htobe32(s[i] + h[i]);
+      ((uint32_t*)out.bytes)[i] = convert_endianness(s[i] + h[i]);
   }
   // clang-format on
 
@@ -1887,7 +1894,7 @@ namespace merkle
     SHA256_Transform(&ctx, &block[0]);
 
     for (int i = 0; i < 8; i++)
-      ((uint32_t*)out.bytes)[i] = htobe32(((uint32_t*)ctx.h)[i]);
+      ((uint32_t*)out.bytes)[i] = convert_endianness(((uint32_t*)ctx.h)[i]);
   }
 
   /// @brief OpenSSL SHA256

test/CMakeLists.txt

@@ -27,6 +27,7 @@ add_merklecpp_test(past_root past_root.cpp)
 add_merklecpp_test(past_paths past_paths.cpp)
 add_merklecpp_test(serialisation serialisation.cpp)
 add_merklecpp_test(partial_serialisation partial_serialisation.cpp)
+add_merklecpp_test(serialise_to_file serialise_to_file.cpp)
 
 if(TARGET evercrypt.host)
   add_merklecpp_test(compare_evercrypt compare_evercrypt.cpp)

test/past_paths.cpp

@@ -28,8 +28,6 @@ std::shared_ptr<merkle::Path> past_root_spec(
   return result;
 }
 
-extern char** environ;
-
 int main(int argc, char** argv)
 {
   auto test_start_time = std::chrono::high_resolution_clock::now();

test/serialise_to_file.cpp

@@ -0,0 +1,80 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+#include <chrono>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+
+#ifdef HAVE_EVERCRYPT
+#  include <MerkleTree.h>
+#endif
+
+#include "util.h"
+
+#include <merklecpp.h>
+
+#define PRINT_HASH_SIZE 3
+
+int main()
+{
+  try
+  {
+    const size_t num_leaves = 11;
+    {
+      auto hashes = make_hashes(num_leaves);
+      merkle::Tree tree1;
+      for (auto h : hashes)
+        tree1.insert(h);
+      auto root1 = tree1.root();
+      std::cout << "ROOT1=" << root1.to_string() << std::endl;
+
+      {
+        // Write a new file if it doesn't exist.
+        std::ifstream fi("tree.bytes", std::ifstream::binary);
+        if (!fi.good())
+        {
+          std::vector<uint8_t> bytes;
+          tree1.serialise(bytes);
+          std::ofstream f("tree.bytes", std::ofstream::binary);
+          for (char b : bytes)
+            f.write(&b, 1);
+          f.close();
+          fi.close();
+        }
+      }
+
+      // Read file if it exists
+      std::ifstream f("tree.bytes", std::ifstream::binary);
+      if (f.good())
+      {
+        merkle::Tree tree2;
+        std::vector<uint8_t> bytes;
+        char t;
+        while (!f.eof())
+        {
+          f.read(&t, 1);
+          bytes.push_back(t);
+        }
+        tree2.deserialise(bytes);
+        f.close();
+        auto root2 = tree2.root();
+        std::cout << "ROOT2=" << root2.to_string() << std::endl;
+        if (root1 != root2)
+          throw std::runtime_error("root hash mismatch");
+      }
+    }
+  }
+  catch (std::exception& ex)
+  {
+    std::cout << "Error: " << ex.what() << std::endl;
+    return 1;
+  }
+  catch (...)
+  {
+    std::cout << "Error" << std::endl;
+    return 1;
+  }
+
+  return 0;
+}