Update spirv-tools (#1127)

Jake-Shadle · web-flow · commit e96418f9fa93 · 2024-02-07T07:46:23.000Z
* Update to spirv-tools 0.10.0

* Use pre-built binaries from spirv-tools-rs

* Oops

* Target != host for android

* Use non-ancient ubuntu

* Oh right

* Update expected output

* Address feedback

* Oops

* Cancel actions when new commits are pushed

* Update CHANGELOG

* Fixup
diff --git a/.github/install-spirv-tools/Cargo.toml b/.github/install-spirv-tools/Cargo.toml
@@ -0,0 +1,14 @@
+# We make this tool its own workspace as it doesn't share dependencies with the
+# rest of the workspace, and it shouldn't be built normally, only as a helper
+# for CI so it would just slow down local development for no reason
+[workspace]
+
+[package]
+name = "install-spirv-tools"
+edition = "2021"
+version = "0.1.0"
+publish = false
+
+[dependencies]
+tar = "0.4"
+zstd = "0.13"
diff --git a/.github/install-spirv-tools/src/main.rs b/.github/install-spirv-tools/src/main.rs
@@ -0,0 +1,109 @@
+use std::{env, fs, io::Write as _, process::Command};
+
+struct Group(bool);
+
+impl Group {
+    fn new(group: &str) -> Self {
+        let is_gh = env::var_os("CI").is_some();
+        if is_gh {
+            println!("::group::{group}");
+        } else {
+            println!("{group}");
+        }
+        Self(is_gh)
+    }
+}
+
+impl Drop for Group {
+    fn drop(&mut self) {
+        if self.0 {
+            println!("::endgroup::");
+        }
+    }
+}
+
+fn main() {
+    let (triple, release, td) = {
+        let mut args = env::args().skip(1);
+        let triple = args.next().expect("expected target triple");
+        let release = args.next().expect("expected release tag name");
+        let td = args.next().expect("expected output directory");
+
+        (triple, release, td)
+    };
+
+    let compressed = {
+        let _s = Group::new(&format!("downloading {triple} tarball"));
+        let mut cmd = Command::new("curl");
+        cmd.args(["-f", "-L"])
+        .arg(format!("https://github.com/EmbarkStudios/spirv-tools-rs/releases/download/{release}/{triple}.tar.zst"))
+        .stdout(std::process::Stdio::piped());
+
+        let output = cmd
+            .spawn()
+            .expect("curl is not installed")
+            .wait_with_output()
+            .expect("failed to wait for curl");
+
+        if !output.status.success() {
+            panic!("failed to download tarball via curl");
+        }
+
+        output.stdout
+    };
+
+    let decoded = {
+        let _s = Group::new(&format!("decompressing {triple} tarball"));
+        // All archives are <8MiB decompressed
+        let uncompressed = Vec::with_capacity(8 * 1024 * 1024);
+        let mut decoder =
+            zstd::stream::write::Decoder::new(uncompressed).expect("failed to create decoder");
+        decoder
+            .write_all(&compressed)
+            .expect("failed to decompress");
+        decoder.flush().expect("failed to flush decompress stream");
+
+        decoder.into_inner()
+    };
+
+    {
+        let _s = Group::new(&format!("untarring {triple} tarball"));
+        {
+            let mut tar = tar::Archive::new(std::io::Cursor::new(&decoded));
+
+            if tar
+                .entries()
+                .expect("failed to retrieve entries")
+                .filter(|ent| ent.is_ok())
+                .count()
+                == 0
+            {
+                panic!("no valid entries found in tarball");
+            }
+        }
+
+        let mut tar = tar::Archive::new(std::io::Cursor::new(decoded));
+        tar.unpack(&td).expect("failed to untar files");
+    }
+
+    if let Some(gh_path) = env::var_os("GITHUB_PATH") {
+        let _s = Group::new(&format!("adding '{td}' to $GITHUB_PATH ({gh_path:?})"));
+
+        // emulate >> for both empty and non-empty files
+        let has_contents = fs::metadata(&gh_path).map_or(false, |md| md.len() > 0);
+
+        let mut file = fs::OpenOptions::new()
+            .append(true)
+            .open(gh_path)
+            .expect("failed to open $GITHUB_PATH");
+
+        let td = if has_contents {
+            format!("\n{td}\n")
+        } else {
+            td
+        };
+
+        file.write_all(td.as_bytes())
+            .expect("failed to write to $GITHUB_PATH");
+    }
+}
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -6,6 +6,11 @@ on:
 
 name: CI
 
+# Cancel PR actions on new commits
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     name: Test
@@ -20,52 +25,22 @@ jobs:
             target: x86_64-apple-darwin
           - os: ubuntu-20.04-16core
             target: aarch64-linux-android
+            host: x86_64-unknown-linux-gnu
     runs-on: ${{ matrix.os }}
     env:
-      # Get platform-specific download links from https://github.com/KhronosGroup/SPIRV-Tools/blob/master/docs/downloads.md
-      # which will point to the `spirv-tools` Google Cloud Storage Bucket - if
-      # you need to manually look around, you can search for `spirv_tools_version`
-      # (which should be in the `YYYYMMDD` format and appear in paths) in these
-      # listings (NB: they're limited to 1000 results and may need adjustment):
-      # https://storage.googleapis.com/spirv-tools/?list-type=2&start-after=artifacts/prod/graphics_shader_compiler/spirv-tools/linux-clang-release/continuous/1800
-      # https://storage.googleapis.com/spirv-tools/?list-type=2&start-after=artifacts/prod/graphics_shader_compiler/spirv-tools/macos-clang-release/continuous/1800
-      # https://storage.googleapis.com/spirv-tools/?list-type=2&start-after=artifacts/prod/graphics_shader_compiler/spirv-tools/windows-msvc-2017-release/continuous/1800
-      spirv_tools_version: "20221024"
-      # NOTE(eddyb) do not forget to update both the above date and below links!
-      # FIXME(eddyb) automate this somewhat by taking advantage of the bucket APIs,
-      # and look for the first build with the date in `spirv_tools_version`.
-      spirv_tools_linux_url: "https://storage.googleapis.com/spirv-tools/artifacts/prod/graphics_shader_compiler/spirv-tools/linux-clang-release/continuous/1863/20221024-094528/install.tgz"
-      spirv_tools_macos_url: "https://storage.googleapis.com/spirv-tools/artifacts/prod/graphics_shader_compiler/spirv-tools/macos-clang-release/continuous/1875/20221024-094531/install.tgz"
-      spirv_tools_windows_url: "https://storage.googleapis.com/spirv-tools/artifacts/prod/graphics_shader_compiler/spirv-tools/windows-msvc-2017-release/continuous/1851/20221024-094908/install.zip"
       RUSTUP_UNPACK_RAM: "26214400"
       RUSTUP_IO_THREADS: "1"
     steps:
-      - uses: actions/checkout@v2
-      - if: ${{ runner.os == 'Linux' }}
-        name: Linux - Install native dependencies and spirv-tools
-        run: |
-          sudo apt install libwayland-cursor0 libxkbcommon-dev libwayland-dev
-          mkdir "${HOME}/spirv-tools"
-          curl -fL "$spirv_tools_linux_url" | tar -xz -C "${HOME}/spirv-tools"
-          echo "${HOME}/spirv-tools/install/bin" >> $GITHUB_PATH
-      - if: ${{ runner.os == 'macOS' }}
-        name: Mac - Install spirv-tools
-        # FIXME(eddyb) deduplicate with Linux (and maybe even Windows?).
-        run: |
-          mkdir "${HOME}/spirv-tools"
-          curl -fL "$spirv_tools_macos_url" | tar -xz -C "${HOME}/spirv-tools"
-          echo "${HOME}/spirv-tools/install/bin" >> $GITHUB_PATH
-      - if: ${{ runner.os == 'Windows' }}
-        name: Windows - Install spirv-tools
+      - uses: actions/checkout@v4
+      # Install the spirv-tools binaries from tarballs hosted on each release
+      # of spirv-tools. This downloads the tarball, decompresses it, unpacks
+      # the binaries to the specified path, and adds them to PATH
+      - name: Install spirv-tools binaries
         shell: bash
-        run: |
-          tmparch=$(mktemp)
-          mkdir "${HOME}/spirv-tools"
-          curl -fL -o "$tmparch" "$spirv_tools_windows_url"
-          unzip "$tmparch" -d "${HOME}/spirv-tools"
-      - if: ${{ runner.os == 'Windows' }}
-        # Runs separately to add spir-v tools to Powershell's Path.
-        run: echo "$HOME/spirv-tools/install/bin" >> $env:GITHUB_PATH
+        run: cargo run --manifest-path .github/install-spirv-tools/Cargo.toml -- ${{matrix.host || matrix.target}} 0.10.0 "${{github.workspace}}/bin"
+      - if: ${{ runner.os == 'Linux' }}
+        name: Linux - Install native dependencies
+        run: sudo apt install libwayland-cursor0 libxkbcommon-dev libwayland-dev
       # cargo version is a random command that forces the installation of rust-toolchain
       - name: install rust-toolchain
         run: cargo version
@@ -142,16 +117,13 @@ jobs:
       # Note that we are explicitly NOT checking out submodules, to validate
       # that we haven't accidentally enabled spirv-tools native compilation
       # and regressed CI times
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: "false"
       - name: Install native dependencies
         run: sudo apt install libwayland-cursor0 libxkbcommon-dev libwayland-dev
       - name: Install spirv-tools
-        run: |
-          mkdir "${HOME}/spirv-tools"
-          curl -fL https://storage.googleapis.com/spirv-tools/artifacts/prod/graphics_shader_compiler/spirv-tools/linux-clang-release/continuous/1409/20210313-175801/install.tgz | tar -xz -C "${HOME}/spirv-tools"
-          echo "${HOME}/spirv-tools/install/bin" >> $GITHUB_PATH
+        run: cargo run --manifest-path .github/install-spirv-tools/Cargo.toml -- x86_64-unknown-linux-gnu 0.10.0 "${{github.workspace}}/bin"
       - name: Install rustup components
         run: rustup component add rustfmt clippy
       # cargo version is a random command that forces the installation of rust-toolchain
@@ -173,7 +145,7 @@ jobs:
         run: .github/workflows/lint.sh
 
   cargo-deny:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: EmbarkStudios/cargo-deny-action@v1
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,4 @@ target/
 .vscode/
 .vim/
 tests/Cargo.lock
+.github/install-spirv-tools/Cargo.lock
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Changed 🛠
+- [PR#1127](https://github.com/EmbarkStudios/rust-gpu/pull/1127) updated `spirv-tools` to `0.10.0`, which follows `vulkan-sdk-1.3.275`.
 - [PR#1101](https://github.com/EmbarkStudios/rust-gpu/pull/1101) Add `ignore` and `no_run` to documentation to make `cargo test` pass.
 - [PR#1112](https://github.com/EmbarkStudios/rust-gpu/pull/1112) updated wgpu and winit in example runners
 - [PR#1100](https://github.com/EmbarkStudios/rust-gpu/pull/1100) updated toolchain to `nightly-2023-09-30`
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -31,10 +31,11 @@ license = "MIT OR Apache-2.0"
 repository = "https://github.com/EmbarkStudios/rust-gpu"
 
 [workspace.dependencies]
+spirv-builder = { path = "./crates/spirv-builder", version = "=0.9.0", default-features = false }
 spirv-std = { path = "./crates/spirv-std", version = "=0.9.0" }
 spirv-std-types = { path = "./crates/spirv-std/shared", version = "=0.9.0" }
 spirv-std-macros = { path = "./crates/spirv-std/macros", version = "=0.9.0" }
-spirv-builder = { path = "./crates/spirv-builder", version = "=0.9.0", default-features = false  }
+spirv-tools = { version = "0.10", default-features = false }
 rustc_codegen_spirv = { path = "./crates/rustc_codegen_spirv", version = "=0.9.0", default-features = false }
 rustc_codegen_spirv-types = { path = "./crates/rustc_codegen_spirv-types", version = "=0.9.0" }
 
diff --git a/crates/rustc_codegen_spirv/Cargo.toml b/crates/rustc_codegen_spirv/Cargo.toml
@@ -51,14 +51,14 @@ ar = "0.9.0"
 either = "1.8.0"
 indexmap = "1.6.0"
 rspirv = "0.11"
+rustc_codegen_spirv-types.workspace = true
 rustc-demangle = "0.1.21"
 sanitize-filename = "0.4"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 smallvec = { version = "1.6.1", features = ["union"] }
-spirv-tools = { version = "0.9", default-features = false }
-rustc_codegen_spirv-types.workspace = true
 spirt = "0.3.0"
+spirv-tools.workspace = true
 lazy_static = "1.4.0"
 itertools = "0.10.5"
 
diff --git a/tests/ui/lang/core/ref/member_ref_arg-broken.stderr b/tests/ui/lang/core/ref/member_ref_arg-broken.stderr
@@ -3,6 +3,7 @@ warning: `#[inline(never)]` function `member_ref_arg_broken::h` needs to be inli
 warning: `#[inline(never)]` function `member_ref_arg_broken::h_newtyped` needs to be inlined because it has illegal argument or return types
 
 error: error:0:0 - OpLoad Pointer <id> '$ID[%$ID]' is not a logical pointer.
+         %39 = OpLoad %uint %38
   |
   = note: spirv-val failed
   = note: module `$TEST_BUILD_DIR/lang/core/ref/member_ref_arg-broken.default`

Original file line number	Diff line number	Diff line change
@@ -3,6 +3,7 @@ warning: `#[inline(never)]` function `member_ref_arg_broken::h` needs to be inli
`3`	`3`	warning: `#[inline(never)]` function `member_ref_arg_broken::h_newtyped` needs to be inlined because it has illegal argument or return types
`4`	`4`
`5`	`5`	`error: error:0:0 - OpLoad Pointer <id> '$ID[%$ID]' is not a logical pointer.`
	`6`	`+ %39 = OpLoad %uint %38`
`6`	`7`	`\|`
`7`	`8`	`= note: spirv-val failed`
`8`	`9`	= note: module `$TEST_BUILD_DIR/lang/core/ref/member_ref_arg-broken.default`