Merge remote-tracking branch 'origin/main' into db/sc-60643/rbin

.github/DIRECTORY.md

@@ -30,4 +30,4 @@ the version-specific tests. This fan-out setup reduces the runtime of each job f
 
 ## Documentation
 
-Documentation is built by `workflows/build-docs.yml` using `<root>/scripts/ci/build_docs.sh`.
+Documentation is built by `workflows/build-docs.yml` using `<root>/scripts/ci/build_docs.sh`.

.github/workflows/build-docs.yml

@@ -2,7 +2,7 @@ name: build-docs
 on:
   push:
     branches:
-      - dev
+      - main
       - 'release-*'
     paths-ignore:
       - '.github/workflows/quarto-render.yml'

.github/workflows/build-windows.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-2019, windows-2022]
+        os: [windows-2022]
         #considering https://stackoverflow.com/questions/65035256/how-to-access-matrix-variables-in-github-actions
         environ: [azure, s3, gcs, serialization]
         include:

.github/workflows/check-formatting.yml

@@ -36,8 +36,5 @@ jobs:
           printenv
         shell: bash
 
-      - name: Check formatting (linux only)'
-        run: |
-          source $GITHUB_WORKSPACE/scripts/ci/check_formatting_linux.sh
-        shell: bash
-        if: ${{ runner.os == 'Linux' }}
+      - name: Check formatting
+        run: $GITHUB_WORKSPACE/scripts/ci/check_formatting_linux.sh

.github/workflows/ci-linux_mac.yml

@@ -236,7 +236,13 @@ jobs:
         run: ./scripts/ci/posix/dump-core-stacks.sh
 
       - name: List files in build directory on failure
-        run: du --exclude _deps -h $GITHUB_WORKSPACE/build
+        run: |
+          if [ $(uname) = "Darwin" ] ; then
+            # MacOS du lacks --exclude
+            du -h $GITHUB_WORKSPACE/build
+          else
+            du --exclude _deps -h $GITHUB_WORKSPACE/build
+          fi
         if: ${{ failure() }}
 
       - name: 'Upload failure artifacts (Linux)' # https://github.com/actions/upload-artifact#where-does-the-upload-go

.github/workflows/full-ci.yml

@@ -3,7 +3,7 @@ on:
   workflow_dispatch:
   push:
     branches:
-      - dev
+      - main
       - release-*
       - refs/tags/*
 

.github/workflows/nightly-test.yml

@@ -20,7 +20,6 @@ jobs:
           - os: macos-latest
             experimental: ON
           - os: windows-latest
-          - os: windows-2019
           - os: windows-latest
             config: "Debug"
             # Insufficient space on default D:/ for debug build
@@ -83,7 +82,7 @@ jobs:
         include:
           - os: ubuntu-latest
             triplet: x64-linux
-          - os: windows-2019
+          - os: windows-2022
             triplet: x64-windows
           - os: macos-13
             triplet: x64-osx

.github/workflows/quarto-render.yml

@@ -4,7 +4,7 @@ name: Render and deploy Quarto files
 on:
   push:
     # publish on merge only
-    branches: dev
+    branches: main
     # when changes match any path below
     paths:
       - '.github/workflows/quarto-render.yml'

.github/workflows/release.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   push:
     branches:
-      - dev
+      - main
       - 'release-*'
     tags:
       - '*'
@@ -45,7 +45,7 @@ jobs:
       matrix:
         include:
           - platform: windows-x86_64
-            os: windows-2019
+            os: windows-2022
             triplet: x64-windows-release
           - platform: linux-x86_64
             os: ubuntu-20.04

.github/workflows/test-cloud-e2e.yml

@@ -9,7 +9,7 @@ on:
         type: boolean
   push:
     branches:
-      - dev
+      - main
       - release-*
 
 env:

.gitlab-ci.yml

@@ -4,7 +4,7 @@ stages:
 trigger_pipeline:
   stage: test
   rules: 
-    - if: $CI_COMMIT_BRANCH =~ /^dev|^release-.*/ || $CI_COMMIT_TAG != "" # only/except rules are no longer actively developed. Please use `rules` instead.
+    - if: $CI_COMMIT_BRANCH =~ /^main|^release-.*/ || $CI_COMMIT_TAG != "" # only/except rules are no longer actively developed. Please use `rules` instead.
     - if: $CI_PIPELINE_SOURCE == "external_pull_request_event"
       changes: 
         - "!.github/workflows/quarto-render.yml"

CMakeLists.txt

@@ -201,7 +201,7 @@ else()
     endif()
   endif()
 
-  if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang" AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER_EQUAL "16")
+  if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER_EQUAL "16")
     add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-fexperimental-library>)
   endif()
 endif()

CONTRIBUTING.md

@@ -5,9 +5,9 @@ help you file issues, bug reports, or contribute code to the open source TileDB
 
 ## Contribution Checklist:
 
-* Reporting a bug?  Please read [how to file a bug report](https://github.com/TileDB-Inc/TileDB/blob/dev/CONTRIBUTING.md#reporting-a-bug) section to make sure sufficient information is included.
+* Reporting a bug?  Please read [how to file a bug report](https://github.com/TileDB-Inc/TileDB/blob/main/CONTRIBUTING.md#reporting-a-bug) section to make sure sufficient information is included.
 
-* Contributing code? You rock! Be sure to [review the contributor section](https://github.com/TileDB-Inc/TileDB/blob/dev/CONTRIBUTING.md#contributing-code) for helpful tips on the tools we use to build TileDB, format code, and issue pull requests (PR)'s.
+* Contributing code? You rock! Be sure to [review the contributor section](https://github.com/TileDB-Inc/TileDB/blob/main/CONTRIBUTING.md#contributing-code) for helpful tips on the tools we use to build TileDB, format code, and issue pull requests (PR)'s.
 
 ## Reporting a Bug
 
@@ -16,7 +16,7 @@ A useful bug report filed as a GitHub issue provides information about how to re
 1. Before opening a new [GitHub issue](https://github.com/TileDB-Inc/TileDB/issues) try searching the existing issues to see if someone else has already noticed the same problem.
 
 2. When filing a bug report, provide where possible:
-  - The version TileDB (`tiledb_version()`) or if a `dev` version, specific commit that triggers the error.
+  - The version TileDB (`tiledb_version()`) or if a `main` version, specific commit that triggers the error.
   - The full error message, including the backtrace (if possible).  Verbose error reporting is enabled by building TileDB with `../bootstrap --enable-verbose`.
   - A minimal working example, i.e. the smallest chunk of code that triggers the error. Ideally, this should be code that can be a small reduced C / C++ source file. If the code to reproduce is somewhat long, consider putting it in a [gist](https://gist.github.com).
 
@@ -25,7 +25,7 @@ A useful bug report filed as a GitHub issue provides information about how to re
 
 ## Contributing Code
 
-*By contributing code to TileDB, you are agreeing to release it under the [MIT License](https://github.com/TileDB-Inc/TileDB/tree/dev/LICENSE).*
+*By contributing code to TileDB, you are agreeing to release it under the [MIT License](https://github.com/TileDB-Inc/TileDB/tree/main/LICENSE).*
 
 ### Quickstart Workflow:
 
@@ -49,8 +49,7 @@ for dependencies and detailed build instructions.
 [Issue a PR from your updated TileDB fork](https://help.github.com/articles/creating-a-pull-request-from-a-fork/)
 
 Branch conventions:
-- `dev` is the development branch of TileDB, all PR's are merged into `dev`.
-- `master` tracks the latest stable / released version of TileDB.
+- `main` is the development branch of TileDB, all PR's are merged into `main`.
 - `release-x.y.z` are major / bugfix release branches of TileDB.
 
 Formatting conventions:
@@ -63,13 +62,13 @@ Formatting conventions:
 
 ### Pull Requests:
 
-- `dev` is the development branch, all PR’s should be rebased on top of the latest `dev` commit.
+- `main` is the development branch, all PR’s should be rebased on top of the latest `main` commit.
 
 - Commit changes to a local branch.  The convention is to use your initials to identify branches: (ex. “Fred Jones” , `fj/my_bugfix_branch`).  Branch names should be identifiable and reflect the feature or bug that they want to address / fix. This helps in deleting old branches later.
 
 - Make sure the test suite passes by running `make check`.
 
-- When ready to submit a PR, `git rebase` the branch on top of the latest `dev` commit.  Be sure to squash / cleanup the commit history so that the PR preferably one, or a couple commits at most.  Each atomic commit in a PR should be able to pass the test suite.
+- When ready to submit a PR, `git rebase` the branch on top of the latest `main` commit.  Be sure to squash / cleanup the commit history so that the PR preferably one, or a couple commits at most.  Each atomic commit in a PR should be able to pass the test suite.
 
 - Run the limiting / code format tooling (`make format`) before submitting a final PR.
   Make sure that your contribution generally follows the format and naming conventions used by surrounding code.

README.md

@@ -1,6 +1,6 @@
-<a href="https://tiledb.com"><img src="https://github.com/TileDB-Inc/TileDB/raw/dev/doc/source/_static/tiledb-logo_color_no_margin_@4x.png" alt="TileDB logo" width="400"></a>
+<a href="https://tiledb.com"><img src="https://github.com/TileDB-Inc/TileDB/raw/main/doc/source/_static/tiledb-logo_color_no_margin_@4x.png" alt="TileDB logo" width="400"></a>
 
-[![Full CI](https://github.com/TileDB-Inc/TileDB/actions/workflows/full-ci.yml/badge.svg?branch=dev)](https://github.com/TileDB-Inc/TileDB/actions/workflows/full-ci.yml) [![Azure Pipelines](https://dev.azure.com/TileDB-Inc/CI/_apis/build/status/TileDB-Inc.TileDB?branchName=dev)](https://dev.azure.com/TileDB-Inc/CI/_build/latest?definitionId=2&branchName=dev) [![](doc/anaconda.svg)![Anaconda download count badge](https://anaconda.org/conda-forge/TileDB/badges/downloads.svg)](https://anaconda.org/conda-forge/TileDB)
+[![Full CI](https://github.com/TileDB-Inc/TileDB/actions/workflows/full-ci.yml/badge.svg?branch=main)](https://github.com/TileDB-Inc/TileDB/actions/workflows/full-ci.yml) [![Azure Pipelines](https://dev.azure.com/TileDB-Inc/CI/_apis/build/status/TileDB-Inc.TileDB?branchName=main)](https://dev.azure.com/TileDB-Inc/CI/_build/latest?definitionId=2&branchName=main) [![](doc/anaconda.svg)![Anaconda download count badge](https://anaconda.org/conda-forge/TileDB/badges/downloads.svg)](https://anaconda.org/conda-forge/TileDB)
 
 # The Universal Storage Engine
 
@@ -41,10 +41,10 @@ $ docker pull tiledb/tiledb
 $ docker run -it tiledb/tiledb
 ```
 
-We include several [examples](https://github.com/TileDB-Inc/TileDB/tree/dev/examples). You can start with the following:
+We include several [examples](https://github.com/TileDB-Inc/TileDB/tree/main/examples). You can start with the following:
 
-* [Dense array example](https://github.com/TileDB-Inc/TileDB/blob/dev/examples/cpp_api/quickstart_dense.cc)
-* [Sparse array example](https://github.com/TileDB-Inc/TileDB/blob/dev/examples/cpp_api/quickstart_sparse.cc)
+* [Dense array example](https://github.com/TileDB-Inc/TileDB/blob/main/examples/cpp_api/quickstart_dense.cc)
+* [Sparse array example](https://github.com/TileDB-Inc/TileDB/blob/main/examples/cpp_api/quickstart_sparse.cc)
 
 ## Documentation
 
@@ -71,8 +71,8 @@ The TileDB data format is open-source and can be found [here](format_spec/FORMAT
 
 The TileDB team maintains a variety of APIs built on top of the C++ library:
 
-* C ([examples](https://github.com/TileDB-Inc/TileDB/tree/dev/examples/c_api), [API docs](https://tiledb-inc-tiledb.readthedocs-hosted.com/en/stable/c-api.html))
-* C++ ([examples](https://github.com/TileDB-Inc/TileDB/tree/dev/examples/cpp_api), [API docs](https://tiledb-inc-tiledb.readthedocs-hosted.com/en/stable/c++-api.html))
+* C ([examples](https://github.com/TileDB-Inc/TileDB/tree/main/examples/c_api), [API docs](https://tiledb-inc-tiledb.readthedocs-hosted.com/en/stable/c-api.html))
+* C++ ([examples](https://github.com/TileDB-Inc/TileDB/tree/main/examples/cpp_api), [API docs](https://tiledb-inc-tiledb.readthedocs-hosted.com/en/stable/c++-api.html))
 * [Python](https://github.com/TileDB-Inc/TileDB-Py)
 * [R](https://github.com/TileDB-Inc/TileDB-R)
 * [Java](https://github.com/TileDB-Inc/TileDB-Java)
@@ -91,6 +91,6 @@ TileDB is also integrated with several popular databases and data science tools:
 
 ## Get involved
 
-TileDB Embedded is an open-source project and welcomes all forms of contributions. Contributors to the project should read over the [contribution docs](https://github.com/TileDB-Inc/TileDB/blob/dev/CONTRIBUTING.md) for more information.
+TileDB Embedded is an open-source project and welcomes all forms of contributions. Contributors to the project should read over the [contribution docs](https://github.com/TileDB-Inc/TileDB/blob/main/CONTRIBUTING.md) for more information.
 
 We'd love to hear from you. Drop us a line at [hello@tiledb.com](mailto:hello@tiledb.com), visit our [forum](https://forum.tiledb.com/) or [contact form](https://tiledb.com/contact), or [follow us on Twitter](https://twitter.com/tiledb) to stay informed of updates and news.

bootstrap

@@ -119,7 +119,7 @@ while test $# != 0; do
     --dependency=*) dir=`arg "$1"`
                 dependency_dir="$dir";;
     --linkage=*) linkage=`arg "$1"`;;
-    --force-build-all-deps) echo "Argument '--force-build-all-deps' has no effect and will be removed in a future version. Vcpkg builds all dependencies by default, please consult the guide in doc/dev/BUILD.md or vcpkg's documentation to see how to provide your own dependencies.";;
+    --force-build-all-deps) echo "Argument '--force-build-all-deps' has no effect and will be removed in a future version. Vcpkg builds all dependencies by default, please consult the guide in doc/main/BUILD.md or vcpkg's documentation to see how to provide your own dependencies.";;
     --remove-deprecations) tiledb_remove_deprecations="ON";;
     --disable-werror) tiledb_werror="OFF";;
     --disable-tests) tiledb_tests="OFF";;

scripts/ci/check_formatting_linux.sh

@@ -32,7 +32,7 @@ set -e pipefail
 ls -la
 sudo ./scripts/install-clangformat.sh
 
-src=$GITHUB_WORKSPACE
+src=$(dirname $0)/../..
 cd $src
 
 $src/scripts/run-clang-format.sh $src clang-format-17 0

scripts/install-clangformat.sh

@@ -58,6 +58,10 @@ install_clang_format() {
 
 run() {
   install_clang_format
+
+  echo
+  echo "clang-format --version" $(clang-format --version)
+  echo
 }
 
 run

test/CMakeLists.txt

@@ -112,6 +112,7 @@ set(TILEDB_UNIT_TEST_SOURCES
   src/unit-ctx.cc
   src/unit-current-domain-rest.cc
   src/unit-dense-reader.cc
+  src/unit-dense-global-order-reader.cc
   src/unit-DenseTiler.cc
   src/unit-dimension.cc
   src/unit-duplicates.cc

test/src/test-cppapi-current-domain.cc

@@ -468,7 +468,7 @@ TEST_CASE_METHOD(
   CHECK_THROWS_WITH(
       query_read2.submit(),
       Catch::Matchers::ContainsSubstring(
-          "A range was set outside of the current domain."));
+          "was set outside of the current domain"));
 
   tiledb::Query query_read3(ctx_, array_read, TILEDB_READ);
   tiledb::Subarray subarray_read3(ctx_, array_read);
@@ -483,7 +483,7 @@ TEST_CASE_METHOD(
   CHECK_THROWS_WITH(
       query_read3.submit(),
       Catch::Matchers::ContainsSubstring(
-          "A range was set outside of the current domain."));
+          "was set outside of the current domain"));
 
   array_read.close();
 
@@ -588,7 +588,7 @@ TEST_CASE_METHOD(
       .set_data_buffer("a", a_with_cd2)
       .set_data_buffer("dim1", dim1_with_cd2);
   auto matcher = Catch::Matchers::ContainsSubstring(
-      "A range was set outside of the current domain.");
+      "was set outside of the current domain");
   REQUIRE_THROWS_WITH(query_for_cd2.submit(), matcher);
   array_with_cd2.close();
 

test/src/unit-cppapi-schema-evolution.cc

@@ -881,6 +881,13 @@ TEST_CASE(
       .set_offsets_buffer("a", offsets);
   query_r.submit();
   array_r.close();
+  CHECK(buffer.size() == 0);
+  CHECK_THAT(buffer, Catch::Matchers::Equals(std::string{""}));
+  CHECK(offsets.size() == 10);
+  CHECK_THAT(
+      offsets,
+      Catch::Matchers::Equals(
+          std::vector<uint64_t>{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}));
 
   // Read the original array
   std::vector<int> a_data(10);