From 6ed183120afbbb08a58503418331812fe0dd3518 Mon Sep 17 00:00:00 2001 From: Mathieu <60658558+enitrat@users.noreply.github.com> Date: Fri, 22 Mar 2024 15:45:45 +0100 Subject: [PATCH] feat: initial support dencun (#677) * feat(wip): initial support dencun * feat: add option to skip an entire directory * feat: ignore BEACON_ROOT address in post_state * feat: add fallback mechanism if secretKey is not found * adapt blockchain-tests-skip.yml * fix: fmt * feat: support multi-tx blocks * refactor: check case name end for fork name * comment on multi-tx blocks' * fix: add specific fork-filter case for Pyspec tests * move secret_key fallback in secret_key() * feat: base filter on formatted test identifier * address review --- Cargo.lock | 1 + Makefile | 2 +- blockchain-tests-skip.yml | 104 +++++++++--------- crates/build-utils/Cargo.toml | 1 + crates/build-utils/src/constants.rs | 33 +++++- crates/build-utils/src/content_reader.rs | 74 +++++++++++-- crates/build-utils/src/converter.rs | 62 ++++++++--- crates/build-utils/src/filter.rs | 36 +++++- .../ef-testing/src/evm_sequencer/constants.rs | 10 +- crates/ef-testing/src/models/case.rs | 52 +++++---- crates/ef-testing/src/models/result.rs | 8 ++ scripts/generate_skip_file.py | 60 +++++----- 12 files changed, 317 insertions(+), 126 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 051a1911..2ac5c176 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -951,6 +951,7 @@ dependencies = [ "lazy_static", "rayon", "regex", + "reth-primitives", "serde", "serde_json", "serde_yaml", diff --git a/Makefile b/Makefile index 8d8a0b76..098f337b 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ ifneq ("$(wildcard .env)","") endif # The release tag of https://github.com/ethereum/tests to use for EF tests -EF_TESTS_TAG := v12.4 +EF_TESTS_TAG := v13.2 EF_TESTS_URL := https://github.com/ethereum/tests/archive/refs/tags/$(EF_TESTS_TAG).tar.gz EF_TESTS_DIR := ./crates/ef-testing/ethereum-tests diff --git a/blockchain-tests-skip.yml b/blockchain-tests-skip.yml index 6832b52d..4cedddf6 100644 --- a/blockchain-tests-skip.yml +++ b/blockchain-tests-skip.yml @@ -2,62 +2,66 @@ # The first level corresponds to the directory, the second to the list of file names regex to ignore. # In the ef-tests repo, we skip all tests but the vmArithmeticTest. +directories: + - Pyspecs + - Cancun + filename: None: - None testname: vmArithmeticTest: - - addmod_d10g0v0_Shanghai - - addmod_d11g0v0_Shanghai - - addmod_d8g0v0_Shanghai - - addmod_d9g0v0_Shanghai - - divByZero_d63g0v0_Shanghai - - divByZero_d64g0v0_Shanghai - - divByZero_d65g0v0_Shanghai - - divByZero_d66g0v0_Shanghai - - divByZero_d67g0v0_Shanghai - - divByZero_d68g0v0_Shanghai - - divByZero_d69g0v0_Shanghai - - divByZero_d70g0v0_Shanghai - - divByZero_d71g0v0_Shanghai - - divByZero_d72g0v0_Shanghai - - divByZero_d73g0v0_Shanghai - - divByZero_d74g0v0_Shanghai - - divByZero_d75g0v0_Shanghai - - divByZero_d76g0v0_Shanghai - - divByZero_d77g0v0_Shanghai - - divByZero_d78g0v0_Shanghai - - divByZero_d79g0v0_Shanghai - - divByZero_d80g0v0_Shanghai - - divByZero_d81g0v0_Shanghai - - divByZero_d82g0v0_Shanghai - - divByZero_d83g0v0_Shanghai - - divByZero_d84g0v0_Shanghai - - divByZero_d85g0v0_Shanghai - - divByZero_d86g0v0_Shanghai - - divByZero_d87g0v0_Shanghai - - divByZero_d88g0v0_Shanghai - - divByZero_d89g0v0_Shanghai - - divByZero_d90g0v0_Shanghai - - divByZero_d91g0v0_Shanghai - - divByZero_d92g0v0_Shanghai - - divByZero_d93g0v0_Shanghai - - divByZero_d94g0v0_Shanghai - - divByZero_d95g0v0_Shanghai - - divByZero_d96g0v0_Shanghai - - divByZero_d97g0v0_Shanghai - - expPower256Of256_d0g0v0_Shanghai - - exp_d1g0v0_Shanghai - - exp_d3g0v0_Shanghai - - exp_d8g0v0_Shanghai - - exp_d9g0v0_Shanghai - - expPower256_d0g0v0_Shanghai - - mulmod_d12g0v0_Shanghai - - mulmod_d13g0v0_Shanghai - - mulmod_d14g0v0_Shanghai - - mulmod_d15g0v0_Shanghai - - twoOps_d0g0v0_Shanghai + - addmod_d10g0v0_Cancun + - addmod_d11g0v0_Cancun + - addmod_d8g0v0_Cancun + - addmod_d9g0v0_Cancun + - divByZero_d63g0v0_Cancun + - divByZero_d64g0v0_Cancun + - divByZero_d65g0v0_Cancun + - divByZero_d66g0v0_Cancun + - divByZero_d67g0v0_Cancun + - divByZero_d68g0v0_Cancun + - divByZero_d69g0v0_Cancun + - divByZero_d70g0v0_Cancun + - divByZero_d71g0v0_Cancun + - divByZero_d72g0v0_Cancun + - divByZero_d73g0v0_Cancun + - divByZero_d74g0v0_Cancun + - divByZero_d75g0v0_Cancun + - divByZero_d76g0v0_Cancun + - divByZero_d77g0v0_Cancun + - divByZero_d78g0v0_Cancun + - divByZero_d79g0v0_Cancun + - divByZero_d80g0v0_Cancun + - divByZero_d81g0v0_Cancun + - divByZero_d82g0v0_Cancun + - divByZero_d83g0v0_Cancun + - divByZero_d84g0v0_Cancun + - divByZero_d85g0v0_Cancun + - divByZero_d86g0v0_Cancun + - divByZero_d87g0v0_Cancun + - divByZero_d88g0v0_Cancun + - divByZero_d89g0v0_Cancun + - divByZero_d90g0v0_Cancun + - divByZero_d91g0v0_Cancun + - divByZero_d92g0v0_Cancun + - divByZero_d93g0v0_Cancun + - divByZero_d94g0v0_Cancun + - divByZero_d95g0v0_Cancun + - divByZero_d96g0v0_Cancun + - divByZero_d97g0v0_Cancun + - expPower256Of256_d0g0v0_Cancun + - exp_d1g0v0_Cancun + - exp_d3g0v0_Cancun + - exp_d8g0v0_Cancun + - exp_d9g0v0_Cancun + - expPower256_d0g0v0_Cancun + - mulmod_d12g0v0_Cancun + - mulmod_d13g0v0_Cancun + - mulmod_d14g0v0_Cancun + - mulmod_d15g0v0_Cancun + - twoOps_d0g0v0_Cancun regex: stArgsZeroOneBalance: diff --git a/crates/build-utils/Cargo.toml b/crates/build-utils/Cargo.toml index f6305f34..4db14262 100644 --- a/crates/build-utils/Cargo.toml +++ b/crates/build-utils/Cargo.toml @@ -19,3 +19,4 @@ serde = { workspace = true } serde_json = { workspace = true } serde_yaml = { workspace = true } walkdir = { workspace = true } +reth-primitives = { workspace = true } diff --git a/crates/build-utils/src/constants.rs b/crates/build-utils/src/constants.rs index 8eb961c1..0a664a98 100644 --- a/crates/build-utils/src/constants.rs +++ b/crates/build-utils/src/constants.rs @@ -1,2 +1,33 @@ +use lazy_static::lazy_static; +use reth_primitives::alloy_primitives::{address, Address}; +use std::collections::HashMap; + pub const ROOT: &str = "GeneralStateTests"; -pub const FORK: &str = "Shanghai"; +pub const FORK: &str = "Cancun"; + +lazy_static! { + // A registry of the most common addresses and their associated secret keys. + // Most secret keys can be read from filler files directly - however, for python-based + // tests, the secret keys are not present in the filler files. This registry + // is used to fill in the missing secret keys (only two used in pyspec tests). + pub static ref ADDRESSES_KEYS: HashMap = { + let mut registry = HashMap::new(); + registry.insert( + address!("a94f5374fce5edbc8e2a8697c15331677e6ebf0b"), + "0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8", + ); + registry.insert( + address!("8a0a19589531694250d570040a0c4b74576919b8"), + "0x9e7645d0cfd9c3a04eb7a9db59a4eb7d359f2e75c9164a9d6b9a7d54e1b6a36f", + ); + registry.insert( + address!("d02d72e067e77158444ef2020ff2d325f929b363"), + "41f6e321b31e72173f8ff2e292359e1862f24fba42fe6f97efaf641980eff298", + ); + registry.insert( + address!("97a7cb1de3cc7d556d0aa32433b035067709e1fc"), + "0x0b2986cc45bd8a8d028c3fcf6f7a11a52f1df61f3ea5d63f05ca109dd73a3fa0" + ); + registry + }; +} diff --git a/crates/build-utils/src/content_reader.rs b/crates/build-utils/src/content_reader.rs index b2dcf08e..d0740b64 100644 --- a/crates/build-utils/src/content_reader.rs +++ b/crates/build-utils/src/content_reader.rs @@ -1,8 +1,12 @@ use std::collections::BTreeMap; +use reth_primitives::{revm_primitives::FixedBytes, Address}; use serde_json::Value; -use crate::{path::PathWrapper, utils::blockchain_tests_to_general_state_tests_path}; +use crate::{ + constants::ADDRESSES_KEYS, path::PathWrapper, + utils::blockchain_tests_to_general_state_tests_path, +}; /// The `ContentReader` is used to read the content of the ef-test tests files. /// The tests files are located in the `BlockchainTests` folder and contain @@ -22,20 +26,45 @@ impl ContentReader { /// /// Test location: BlockchainTests/GeneralStateTests/stRandom/randomStatetest0.json /// Secret key location: GeneralStateTests/stRandom/randomStatetest0.json - pub fn secret_key(path: PathWrapper) -> Result, eyre::Error> { + pub fn secret_key( + path: PathWrapper, + case_without_secret: &Value, + ) -> Result { let path = blockchain_tests_to_general_state_tests_path(path); - let content = path.read_file_to_string()?; + let maybe_content_with_secret = path.read_file_to_string(); + let case = match maybe_content_with_secret { + Ok(content) => { + let cases: BTreeMap = serde_json::from_str(&content)?; + cases.into_values().next() + } + Err(_) => Some(case_without_secret.clone()), + }; - let cases: BTreeMap = serde_json::from_str(&content)?; - let case = cases.into_values().next(); - - Ok(case + let key = match case .as_ref() .and_then(|value| value.get("transaction")) .and_then(|value| value.get("secretKey")) - .cloned()) - } + { + Some(key) => key.to_string(), + None => { + let block = Self::block(case_without_secret)?; + let transaction = Self::transaction(case_without_secret, &block)?; + let sender = transaction + .get("sender") + .and_then(|value| value.as_str()) + .ok_or_else(|| eyre::eyre!("Key 'sender' not found"))?; + let sender_address: Address = sender.parse::>()?.into(); + ADDRESSES_KEYS + .get(&sender_address) + .map(|addr| format!("\"{}\"", addr)) + .unwrap_or_else(|| panic!("No secret key found for {sender_address}")) + } + }; + Ok(key) + } + // Ok( + // )) pub fn pre_state(test_case: &Value) -> Result { Ok(serde_json::from_value( test_case @@ -71,4 +100,31 @@ impl ContentReader { // Return a clone of the block Ok(first_block.clone()) } + + pub fn transaction(test_case: &Value, block: &Value) -> Result { + let maybe_transaction = block.get("transactions"); + + match maybe_transaction { + Some(transaction) => { + // Ensure it's an array + let transaction_array = transaction + .as_array() + .ok_or_else(|| eyre::eyre!("'transactions' is not an array"))?; + + // Get the first transaction - multi-txs tests are not supported by the runner + let first_tx = transaction_array + .first() + .ok_or_else(|| eyre::eyre!("'transactions' array is empty"))?; + + Ok(first_tx.clone()) + } + None => { + let transaction = test_case + .get("transaction") + .ok_or_else(|| eyre::eyre!("key 'transaction' not found"))?; + + Ok(transaction.clone()) + } + } + } } diff --git a/crates/build-utils/src/converter.rs b/crates/build-utils/src/converter.rs index 3690ed2d..480d29fd 100644 --- a/crates/build-utils/src/converter.rs +++ b/crates/build-utils/src/converter.rs @@ -15,26 +15,26 @@ use crate::{ /// /// Test location: BlockchainTests/GeneralStateTests/stRandom/ /// List of tests: [randomStatetest0.json, randomStatetest1.json, ...] -/// Inner tests: [`randomStatetest0_d0g0v0_Shanghai`, `randomStatetest0_d1g0v0_Shanghai`, -/// ..., `randomStatetest1_d0g0v0_Shanghai`, `randomStatetest1_d1g0v0_Shanghai`, ...] +/// Inner tests: [`randomStatetest0_d0g0v0_Cancun`, `randomStatetest0_d1g0v0_Cancun`, +/// ..., `randomStatetest1_d0g0v0_Cancun`, `randomStatetest1_d1g0v0_Cancun`, ...] /// Generated String: /// r#" /// mod randomStatetest0 { /// use super::*; /// #[test] -/// fn test_randomStatetest0_d0g0v0_Shanghai() { +/// fn test_randomStatetest0_d0g0v0_Cancun() { /// ... /// } /// #[test] -/// fn test_randomStatetest0_d1g0v0_Shanghai() { +/// fn test_randomStatetest0_d1g0v0_Cancun() { /// ... /// } /// #[test] -/// fn test_randomStatetest1_d0g0v0_Shanghai() { +/// fn test_randomStatetest1_d0g0v0_Cancun() { /// ... /// } /// #[test] -/// fn test_randomStatetest1_d1g0v0_Shanghai() { +/// fn test_randomStatetest1_d1g0v0_Cancun() { /// ... /// } /// ... @@ -93,12 +93,16 @@ impl<'a> EfTests<'a> { let file_contents = cases .par_iter() .map(|(case_name, content)| { - if !case_name.contains(FORK) { + if !(case_name.ends_with(FORK) || case_name.contains(&format!("fork_{}", FORK))) + { return Ok(String::new()); } - let secret_key = ContentReader::secret_key(file_path.clone())? - .ok_or_else(|| eyre::eyre!("Missing secret key"))?; let is_skipped = self.filter.is_skipped(file_path, Some(case_name.clone())); + let secret_key = if is_skipped { + String::default() // secret key is not needed if the test is skipped + } else { + ContentReader::secret_key(file_path.clone(), content)? + }; Self::format_to_test(case_name, parent_dir, &secret_key, content, is_skipped) }) .collect::, eyre::Error>>()?; @@ -140,7 +144,7 @@ impl<'a> EfTests<'a> { fn format_to_test( case_name: &str, parent_dir: &str, - secret_key: &Value, + secret_key: &String, content: &Value, is_skipped: bool, ) -> Result { @@ -166,7 +170,7 @@ impl<'a> EfTests<'a> { fn format_test_content( case_name: &str, parent_dir: &str, - secret_key: &Value, + secret_key: &String, content: &Value, is_skipped: bool, ) -> Result { @@ -198,9 +202,37 @@ impl<'a> EfTests<'a> { } /// Formats the given string into a valid rust identifier. - fn format_into_identifier(s: &str) -> String { - s.replace('-', "_minus_") - .replace('+', "_plus_") - .replace('^', "_xor_") + pub fn format_into_identifier(s: &str) -> String { + // Pyspec tests are in form test_src/GeneralStateTestsFillerFiller/Pyspecs/berlin/eip2930_access_list/test_acl.py::test_access_list[fork_Cancun_minus_blockchain_test]() + // We only keep the test name and its parameters. + if s.contains("Pyspecs") { + let test_name = s + .split('/') + .last() + .unwrap_or_default() + .split("::") + .last() + .unwrap_or_default(); + + let test_name = test_name + .to_string() + .replace("test_", "") + .replace('(', "_lpar_") + .replace(')', "_rpar") + .replace('[', "__") + .replace(']', "") + .replace('-', "_minus_") + .split(',') + .map(|part| part.trim()) + .collect::>() + .join("_"); + + // add the fork name after the test name + test_name + } else { + s.replace('-', "_minus_") + .replace('+', "_plus_") + .replace('^', "_xor_") + } } } diff --git a/crates/build-utils/src/filter.rs b/crates/build-utils/src/filter.rs index 4101ce39..136fac50 100644 --- a/crates/build-utils/src/filter.rs +++ b/crates/build-utils/src/filter.rs @@ -1,8 +1,12 @@ use regex::Regex; use serde::{Deserialize, Serialize}; -use std::{collections::BTreeMap, fs}; +use std::{ + collections::BTreeMap, + fs, + path::{Path, PathBuf}, +}; -use crate::path::PathWrapper; +use crate::{converter::EfTests, path::PathWrapper}; type Folder = String; type FilterMap = BTreeMap>; @@ -10,6 +14,8 @@ type FilterMap = BTreeMap>; /// Filter to be applied on the tests files #[derive(Deserialize, Default, Serialize)] pub struct Filter { + // List of directories that should be skipped. e.g.: Pyspecs. + directories: Vec, /// Mapping containing the directories and the files that should be skipped filename: FilterMap, /// Mapping containing the directories and the regex patterns that should be skipped @@ -27,9 +33,32 @@ impl Filter { /// Checks if the given path is inside the filter object pub fn is_skipped(&self, path: &PathWrapper, case_name: Option) -> bool { + let pathb: PathBuf = (*path).clone().into(); + let path_str = pathb.to_string_lossy(); + + let relative_path = path_str + .split_once("GeneralStateTests/") + .map(|(_, path)| path) + .unwrap_or(&path_str); + + let relative_path = Path::new(relative_path) + .components() + .next() + .and_then(|c| c.as_os_str().to_str()) + .map(ToString::to_string) + .unwrap_or_default(); + let dir_name = path.parent().file_stem_to_string(); let file_name = path.file_stem_to_string(); + if self + .directories + .iter() + .any(|dir| relative_path.contains(dir)) + { + return true; + } + let mut should_skip = self .filename .get(&dir_name) @@ -49,10 +78,11 @@ impl Filter { .unwrap_or_default(); if let Some(case_name) = case_name { + let test_identifier = &EfTests::format_into_identifier(&case_name); should_skip |= self .test_name .get(&dir_name) - .map(|tests| tests.iter().any(|test| test == &case_name)) + .map(|tests| tests.iter().any(|test| test == test_identifier)) .unwrap_or_default(); } diff --git a/crates/ef-testing/src/evm_sequencer/constants.rs b/crates/ef-testing/src/evm_sequencer/constants.rs index a423bf22..538466e5 100644 --- a/crates/ef-testing/src/evm_sequencer/constants.rs +++ b/crates/ef-testing/src/evm_sequencer/constants.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use lazy_static::lazy_static; +use reth_primitives::alloy_primitives::{address, Address}; use starknet::core::types::contract::legacy::LegacyContractClass; use starknet::core::types::contract::CompiledClass; use starknet_api::{ @@ -41,9 +42,12 @@ lazy_static! { ("ec_op_builtin".to_string(), 1_f64), ("keccak_builtin".to_string(), 1_f64), ("segment_arena_builtin".to_string(), 1_f64), - ] - .into_iter() - .collect(); + ] + .into_iter() + .collect(); + + // EVM Addresses + pub static ref BEACON_ROOT_ADDRESS: Address = address!("000f3df6d732807ef1319fb7b8bb8522d0beac02"); // Main addresses pub static ref ETH_FEE_TOKEN_ADDRESS: ContractAddress = contract_address!("0x049D36570D4e46f48e99674bd3fcc84644DdD6b96F7C741B1562B82f9e004dC7"); diff --git a/crates/ef-testing/src/models/case.rs b/crates/ef-testing/src/models/case.rs index 3581030e..f8eaefcb 100644 --- a/crates/ef-testing/src/models/case.rs +++ b/crates/ef-testing/src/models/case.rs @@ -2,7 +2,8 @@ use super::error::RunnerError; use super::result::{extract_output_and_log_execution_result, EVMOutput}; use crate::evm_sequencer::constants::{ - CONTRACT_ACCOUNT_CLASS_HASH, EOA_CLASS_HASH, KAKAROT_ADDRESS, PROXY_CLASS_HASH, + BEACON_ROOT_ADDRESS, CONTRACT_ACCOUNT_CLASS_HASH, EOA_CLASS_HASH, KAKAROT_ADDRESS, + PROXY_CLASS_HASH, }; use crate::evm_sequencer::evm_state::Evm; use crate::evm_sequencer::sequencer::{ @@ -80,29 +81,35 @@ impl BlockchainTestCase { &self, sequencer: &mut KakarotSequencer, ) -> Result { - // we extract the transaction from the block + // we extract the transactions from the block let block = &self.block; let block = SealedBlock::decode(&mut block.rlp.as_ref()).map_err(RunnerError::RlpDecodeError)?; - // Encode body as transaction - let mut tx_signed = block.body.first().cloned().ok_or_else(|| { - RunnerError::Other(vec!["No transaction in pre state block".into()].into()) - })?; - - tx_signed.transaction.set_chain_id(CHAIN_ID); - let signature = sign_message(self.secret_key, tx_signed.signature_hash()) - .map_err(|err| RunnerError::Other(vec![err.to_string()].into()))?; - - tx_signed.signature = signature; + let mut output = EVMOutput::default(); + + // Iterate over all transactions in the block + for tx in block.body.iter() { + // Encode body as transaction + let mut tx_signed = tx.clone(); + tx_signed.transaction.set_chain_id(CHAIN_ID); + // TODO: this will not support blocks with transactions from different senders (different secret key) + let signature = sign_message(self.secret_key, tx_signed.signature_hash()) + .map_err(|err| RunnerError::Other(vec![err.to_string()].into()))?; + tx_signed.signature = signature; + + let execution_result = sequencer.execute_transaction(tx_signed); + + // Update the output with the execution result of the current transaction + let tx_output = extract_output_and_log_execution_result( + &execution_result, + &self.case_name, + &self.case_category, + ) + .unwrap_or_default(); - let execution_result = sequencer.execute_transaction(tx_signed); - let output = extract_output_and_log_execution_result( - &execution_result, - &self.case_name, - &self.case_category, - ) - .unwrap_or_default(); + output.merge(&tx_output); + } Ok(output) } @@ -169,6 +176,13 @@ impl BlockchainTestCase { } for (address, expected_state) in post_state.iter() { + //TODO: this should not be a part of the post-state of EF-Tests and can + // be removed once we base ourself on the next EF-Tests release, which fixes this issue + // Beacon-related features are not supported in Kakarot + if *address == *BEACON_ROOT_ADDRESS { + continue; + } + // Storage for (k, v) in expected_state.storage.iter() { let actual = sequencer.storage_at(address, *k)?; diff --git a/crates/ef-testing/src/models/result.rs b/crates/ef-testing/src/models/result.rs index 47a0dca9..4fe18167 100644 --- a/crates/ef-testing/src/models/result.rs +++ b/crates/ef-testing/src/models/result.rs @@ -19,6 +19,14 @@ pub struct EVMOutput { pub success: bool, } +impl EVMOutput { + pub fn merge(&mut self, other: &Self) { + self.return_data.extend_from_slice(&other.return_data); + self.gas_used += other.gas_used; + self.success &= other.success; + } +} + impl TryFrom<&EventData> for EVMOutput { type Error = eyre::Report; diff --git a/scripts/generate_skip_file.py b/scripts/generate_skip_file.py index d4566ed3..45b453f1 100644 --- a/scripts/generate_skip_file.py +++ b/scripts/generate_skip_file.py @@ -3,31 +3,44 @@ from collections import defaultdict +def format_into_identifier(s: str) -> str: + if "Pyspec" in s: + test_name = ( + s.split("::")[-1] + .replace("test_", "") + .replace("(", "_lpar_") + .replace(")", "_rpar") + .replace("[", "__") + .replace("]", "") + .replace("-", "_minus_") + .split(",") + ) + test_name = "_".join(part.strip() for part in test_name) + return test_name + else: + return s.replace("-", "_minus_").replace("+", "_plus_").replace("^", "_xor_") + + def extract_runresource_failures(input_file): failing_tests = [] - # Buffer to keep track of the last 8 lines - buffer = [] + last_reverted = None + with open(input_file, "r") as file: for line in file: - buffer.append(line) - if len(buffer) > 8: - buffer.pop(0) - if "RunResources has no remaining steps." in line: - # Extract the test name, which is 7 lines above the error message - try: - test_name_line = buffer[ - -8 - ] # The 8th item from the end is 7 lines above the error line - # Extract the test name from the line - if "reverted:" in test_name_line: - test_name = ( - test_name_line.split("reverted:")[0].split("::")[-1].strip() - ) - failing_tests.append(test_name) - except IndexError: - # This happens if the error is found within the first 7 lines of the file - # or the buffer doesn't have enough lines yet, just skip it - continue + if "reverted:" in line: + test_name_line = line + test_name = test_name_line.split("reverted:")[0].split("::")[-1].strip() + test_name = format_into_identifier(test_name) + last_reverted = test_name + # If we find a line that says "RunResources has no remaining steps." after the last reverted test, + # we know that the test failed due to a RunResources error + elif ( + "RunResources has no remaining steps." in line + and last_reverted is not None + ): + failing_tests.append(last_reverted) + last_reverted = None + return failing_tests @@ -41,10 +54,7 @@ def parse_and_write_to_yaml(input_file, output_file): .replace("_minus_", "-") .replace("_plus_", "+") .replace("_xor_", "^"), - m.split("::")[-1] - .replace("_minus_", "-") - .replace("_plus_", "+") - .replace("_xor_", "^"), + m.split("::")[-1], ) for m in re.findall(r"thread '(.*)' panicked at", data) ]