Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ci/Add Windows pipeline #925

Merged
merged 29 commits into from
Dec 14, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
4a36a58
ci: Add Windows
syl20bnr Dec 11, 2023
10602b2
Use std::env::temp_dir instead of hardcoded path
Luni-4 Nov 9, 2023
02881ba
Try to update gix-tempfile
Luni-4 Nov 10, 2023
41a269f
Horrible hack
Luni-4 Nov 10, 2023
56d0a9f
Install warp and mesa dependencies on windows in test workflow
Luni-4 Nov 10, 2023
d12a118
Set WGPU_BACKEND to dx12 for windows in test workflow
Luni-4 Nov 10, 2023
e7b2e37
Install Vulkan on windows in test workflow
Luni-4 Nov 17, 2023
b48a269
Revert "Horrible hack"
syl20bnr Nov 22, 2023
c88947d
Replace remaning ubuntu-latest occurrences by ubuntu-22.04
syl20bnr Nov 24, 2023
4022eb8
Disable clippy for now
syl20bnr Nov 24, 2023
b5e25d3
Focus on Windows std tests by removing other matrix values
syl20bnr Nov 24, 2023
af92682
Set the version of Windows runner explicitly
syl20bnr Nov 24, 2023
cd7a812
Install DirectX on windows-2022 runners in test workflow
syl20bnr Nov 24, 2023
a253b9d
Focus on burn-wgpu failing tests on windows platform
syl20bnr Nov 24, 2023
670c8f3
Install warp and mesa on windows-2022 in test workflow
syl20bnr Nov 24, 2023
fb8731f
Try to setup Vulkan on windows-2022 in test workflow
syl20bnr Nov 24, 2023
25957b1
temp debug
syl20bnr Nov 24, 2023
3af123d
Update workgroup size default in wgpu kernel
syl20bnr Nov 26, 2023
b89ee7e
temp debug
syl20bnr Nov 26, 2023
a5a1938
Use AUTO_GRAPHICS_BACKEND env var for windows test and do some clean up
syl20bnr Dec 11, 2023
82c8e6e
Disable fail-fast for now
syl20bnr Dec 11, 2023
67ed6e8
Revert "Update workgroup size default in wgpu kernel"
syl20bnr Dec 11, 2023
4bf63e7
Enable WGPU tests on macos and disable them on windows
syl20bnr Dec 11, 2023
9841d02
Remove debug prints for CI debugging
syl20bnr Dec 11, 2023
65f91e4
Fix workspace path discovery for windows in run-checks xtask
syl20bnr Dec 12, 2023
ebef645
Cleanup Windows CI investigations
syl20bnr Dec 12, 2023
4cbb474
Remove fail-fast flag in test CI workflow
syl20bnr Dec 13, 2023
a3b2517
Remove wgpu-info debug step in test CI workflow
syl20bnr Dec 13, 2023
884d7e2
Force bash shell on Windows CI runners
syl20bnr Dec 13, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 53 additions & 10 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,29 +34,36 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [macos-13, ubuntu-22.04]
os: [macos-13, ubuntu-22.04, windows-2022]
rust: [stable, 1.71.0]
test: ['std', 'no-std', 'examples']
include:
- cache: stable
rust: stable
- cache: 1-71-0
rust: 1.71.0
- coverage-flags: COVERAGE=1
- os: ubuntu-22.04
coverage-flags: COVERAGE=1
rust: stable
test: std
os: ubuntu-22.04
- wgpu-flags: DISABLE_WGPU=1
- os: macos-13
rust: stable
test: std
os: macos-13
- os: windows-2022
wgpu-flags: "DISABLE_WGPU=1"
# not used yet, as wgpu tests are disabled on windows for now
# see issue: https://github.com/tracel-ai/burn/issues/1062
# auto-graphics-backend-flags: "AUTO_GRAPHICS_BACKEND=dx12";'
exclude:
# only need to check this once
- rust: 1.71.0
test: 'examples'
# only need to check this once
# Do not run no-std tests on macos
- os: macos-13
test: 'no-std'
# Do not run no-std tests on Windows
- os: windows-2022
test: 'no-std'

steps:

Expand Down Expand Up @@ -94,7 +101,7 @@ jobs:
sudo apt install -y libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers

- name: Run cargo clippy for stable version
if: matrix.rust == 'stable' && matrix.test == 'std' && runner.os == 'Linux'
if: runner.os == 'Linux' && matrix.rust == 'stable' && matrix.test == 'std'
uses: giraffate/clippy-action@v1
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
Expand All @@ -107,25 +114,61 @@ jobs:
reporter: github-pr-check

- name: Install grcov
if: matrix.rust == 'stable' && matrix.test == 'std' && runner.os == 'Linux'
if: runner.os == 'Linux' && matrix.rust == 'stable' && matrix.test == 'std'
env:
GRCOV_LINK: https://github.com/mozilla/grcov/releases/download
GRCOV_VERSION: v0.8.18
run: |
curl -L "$GRCOV_LINK/$GRCOV_VERSION/grcov-x86_64-unknown-linux-musl.tar.bz2" |
tar xj -C $HOME/.cargo/bin

- name: (windows) install warp
if: runner.os == 'Windows'
shell: bash
run: |
set -e

curl.exe -L https://www.nuget.org/api/v2/package/Microsoft.Direct3D.WARP/1.0.7.1 -o warp.zip
7z.exe e warp.zip -owarp build/native/amd64/d3d10warp.dll

mkdir -p target/debug/deps

cp -v warp/d3d10warp.dll target/debug/
cp -v warp/d3d10warp.dll target/debug/deps

- name: (windows) install mesa
if: runner.os == 'Windows'
shell: bash
run: |
set -e

curl.exe -L https://github.com/pal1000/mesa-dist-win/releases/download/23.2.1/mesa3d-23.2.1-release-msvc.7z -o mesa.7z
7z.exe e mesa.7z -omesa x64/{opengl32.dll,libgallium_wgl.dll,libglapi.dll,vulkan_lvp.dll,lvp_icd.x86_64.json}

mkdir -p target/debug/deps

cp -v mesa/* target/debug/
cp -v mesa/* target/debug/deps

echo "VK_DRIVER_FILES=$PWD/mesa/lvp_icd.x86_64.json" >> "$GITHUB_ENV"
echo "GALLIUM_DRIVER=llvmpipe" >> "$GITHUB_ENV"

- name: (windows) install directx
if: runner.os == 'Windows'
uses: napokue/setup-dxc@v1.1.0

- name: run checks & tests
shell: bash
run: ${{ matrix.coverage-flags }} ${{ matrix.wgpu-flags }} cargo xtask run-checks ${{ matrix.test }}

- name: Codecov upload
if: matrix.rust == 'stable' && matrix.test == 'std' && runner.os == 'Linux'
if: runner.os == 'Linux' && matrix.rust == 'stable' && matrix.test == 'std'
uses: codecov/codecov-action@v3
with:
files: lcov.info

check-typos:
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
steps:
- name: checkout
uses: actions/checkout@v4
Expand Down
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ fake = "2.9.1"
flate2 = "1.0.28"
float-cmp = "0.9.0"
getrandom = { version = "0.2.11", default-features = false }
gix-tempfile = { version = "8.0.0", features = ["signals"] }
gix-tempfile = { version = "11.0.0", features = ["signals"] }
hashbrown = "0.14.2"
indicatif = "0.17.7"
js-sys = "0.3.65"
Expand Down
5 changes: 4 additions & 1 deletion burn-core/src/optim/adagrad.rs
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,10 @@ mod tests {
let grads = GradientsParams::from_grads(grads, &linear);
let _linear = optimizer.step(LEARNING_RATE, linear, grads);
BinFileRecorder::<FullPrecisionSettings>::default()
.record(optimizer.to_record(), "/tmp/test_optim".into())
.record(
optimizer.to_record(),
std::env::temp_dir().as_path().join("test_optim"),
)
.unwrap();

let state_optim_before = optimizer.to_record();
Expand Down
5 changes: 4 additions & 1 deletion burn-core/src/optim/adam.rs
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,10 @@ mod tests {
let grads = GradientsParams::from_grads(grads, &linear);
let _linear = optimizer.step(LEARNING_RATE, linear, grads);
BinFileRecorder::<FullPrecisionSettings>::default()
.record(optimizer.to_record(), "/tmp/test_optim".into())
.record(
optimizer.to_record(),
std::env::temp_dir().as_path().join("test_optim"),
)
.unwrap();

let state_optim_before = optimizer.to_record();
Expand Down
11 changes: 8 additions & 3 deletions burn-core/src/record/file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,12 @@ mod tests {

use crate as burn;

static FILE_PATH: &str = "/tmp/burn_test_file_recorder";
#[inline(always)]
fn file_path() -> PathBuf {
std::env::temp_dir()
.as_path()
.join("burn_test_file_recorder")
}

#[test]
fn test_can_save_and_load_jsongz_format() {
Expand Down Expand Up @@ -344,10 +349,10 @@ mod tests {
fn test_can_save_and_load<Recorder: FileRecorder>(recorder: Recorder) {
let model_before = create_model();
recorder
.record(model_before.clone().into_record(), FILE_PATH.into())
.record(model_before.clone().into_record(), file_path())
.unwrap();

let model_after = create_model().load_record(recorder.load(FILE_PATH.into()).unwrap());
let model_after = create_model().load_record(recorder.load(file_path()).unwrap());

let byte_recorder = BinBytesRecorder::<FullPrecisionSettings>::default();
let model_bytes_before = byte_recorder
Expand Down
30 changes: 18 additions & 12 deletions burn-core/tests/derive_config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,21 @@ pub enum TestEnumConfig {
Named { first: f32, second: String },
}

#[cfg(feature = "std")]
#[inline(always)]
fn file_path(file_name: &str) -> std::path::PathBuf {
std::env::temp_dir().join(file_name)
}

#[cfg(feature = "std")]
#[test]
fn struct_config_should_impl_serde() {
let config = TestStructConfig::new(2, 3.0, "Allow".to_string(), TestEmptyStructConfig::new());
let file_path = "/tmp/test_struct_config.json";
let file_path = file_path("test_struct_config.json");

config.save(file_path).unwrap();
config.save(&file_path).unwrap();

let config_loaded = TestStructConfig::load(file_path).unwrap();
let config_loaded = TestStructConfig::load(&file_path).unwrap();
assert_eq!(config, config_loaded);
}

Expand All @@ -52,35 +58,35 @@ fn struct_config_should_impl_display() {
#[test]
fn enum_config_no_value_should_impl_serde() {
let config = TestEnumConfig::None;
let file_path = "/tmp/test_enum_no_value_config.json";
let file_path = file_path("test_enum_no_value_config.json");

config.save(file_path).unwrap();
config.save(&file_path).unwrap();

let config_loaded = TestEnumConfig::load(file_path).unwrap();
let config_loaded = TestEnumConfig::load(&file_path).unwrap();
assert_eq!(config, config_loaded);
}

#[cfg(feature = "std")]
#[test]
fn enum_config_one_value_should_impl_serde() {
let config = TestEnumConfig::Single(42.0);
let file_path = "/tmp/test_enum_one_value_config.json";
let file_path = file_path("test_enum_one_value_config.json");

config.save(file_path).unwrap();
config.save(&file_path).unwrap();

let config_loaded = TestEnumConfig::load(file_path).unwrap();
let config_loaded = TestEnumConfig::load(&file_path).unwrap();
assert_eq!(config, config_loaded);
}

#[cfg(feature = "std")]
#[test]
fn enum_config_multiple_values_should_impl_serde() {
let config = TestEnumConfig::Multiple(42.0, "Allow".to_string());
let file_path = "/tmp/test_enum_multiple_values_config.json";
let file_path = file_path("test_enum_multiple_values_config.json");

config.save(file_path).unwrap();
config.save(&file_path).unwrap();

let config_loaded = TestEnumConfig::load(file_path).unwrap();
let config_loaded = TestEnumConfig::load(&file_path).unwrap();
assert_eq!(config, config_loaded);
}

Expand Down
15 changes: 10 additions & 5 deletions burn-core/tests/record_resilience.rs
Original file line number Diff line number Diff line change
Expand Up @@ -175,11 +175,16 @@ mod tests {
.unwrap();
}

#[inline(always)]
fn file_path(filename: String) -> PathBuf {
std::env::temp_dir().join(filename)
}

fn deserialize_with_new_optional_field<R>(name: &str, recorder: R) -> Result<(), RecorderError>
where
R: FileRecorder,
{
let file_path: PathBuf = format!("/tmp/deserialize_with_new_optional_field-{name}").into();
let file_path: PathBuf = file_path(format!("deserialize_with_new_optional_field-{name}"));
let model = Model {
single_const: 32.0,
linear1: nn::LinearConfig::new(20, 20).init::<TestBackend>(),
Expand All @@ -205,7 +210,7 @@ mod tests {
R: FileRecorder,
{
let file_path: PathBuf =
format!("/tmp/deserialize_with_removed_optional_field-{name}").into();
file_path(format!("deserialize_with_removed_optional_field-{name}"));
let model = ModelNewOptionalField {
single_const: 32.0,
linear1: nn::LinearConfig::new(20, 20).init::<TestBackend>(),
Expand All @@ -228,7 +233,7 @@ mod tests {
where
R: FileRecorder,
{
let file_path: PathBuf = format!("/tmp/deserialize_with_new_constant_field-{name}").into();
let file_path: PathBuf = file_path(format!("deserialize_with_new_constant_field-{name}"));
let model = Model {
single_const: 32.0,
array_const: [2, 2],
Expand All @@ -254,7 +259,7 @@ mod tests {
R: FileRecorder,
{
let file_path: PathBuf =
format!("/tmp/deserialize_with_removed_constant_field-{name}").into();
file_path(format!("deserialize_with_removed_constant_field-{name}"));
let model = ModelNewConstantField {
single_const: 32.0,
array_const: [2, 2],
Expand All @@ -277,7 +282,7 @@ mod tests {
where
R: FileRecorder,
{
let file_path: PathBuf = format!("/tmp/deserialize_with_new_field_order-{name}").into();
let file_path: PathBuf = file_path(format!("deserialize_with_new_field_order-{name}"));
let model = Model {
array_const: [2, 2],
single_const: 32.0,
Expand Down
7 changes: 6 additions & 1 deletion xtask/src/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,12 @@ pub(crate) fn get_workspaces(w_type: WorkspaceMemberType) -> Vec<WorkspaceMember
let (workspace_name, workspace_path) =
(parts.first()?.to_owned(), parts.last()?.to_owned());

let workspace_path = workspace_path.replace("(path+file://", "").replace(')', "");
let prefix = if cfg!(target_os = "windows") {
"(path+file:///"
} else {
"(path+file://"
};
let workspace_path = workspace_path.replace(prefix, "").replace(')', "");

match w_type {
WorkspaceMemberType::Crate
Expand Down
Loading