From a9eaf3ef40fe4032de1dd032e47a237220ef483b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Roycourt?= <11146088+remiroyc@users.noreply.github.com> Date: Fri, 27 Oct 2023 14:52:38 +0200 Subject: [PATCH] feat(ark-metadata): add metadata_updated_at (#162) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description - Introduce `metadata_updated_at` timestamp to record the last modification time in the Metadata property. - Implement compatibility for the `glb` file format. - Ensure graceful handling and avoid panicking when unable to upload files to S3. ## What type of PR is this? (check all applicable) - [X] 🍕 Feature (`feat:`) ## Added tests? - [X] 👍 yes ## Added to documentation? - [X] 🙅 no documentation needed --- Cargo.lock | 1 + crates/ark-metadata/Cargo.toml | 1 + crates/ark-metadata/src/metadata_manager.rs | 32 +++++++++++++++------ crates/ark-metadata/src/types.rs | 1 + crates/ark-metadata/src/utils.rs | 21 ++++++++++++++ examples/pontos_sqlx.rs | 13 +++++++-- 6 files changed, 57 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 74f0168cd..20413cc48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -234,6 +234,7 @@ dependencies = [ "ark-starknet", "async-trait", "base64 0.21.4", + "chrono", "dotenv", "mockall", "reqwest", diff --git a/crates/ark-metadata/Cargo.toml b/crates/ark-metadata/Cargo.toml index e9c3e0eb3..44802ca94 100644 --- a/crates/ark-metadata/Cargo.toml +++ b/crates/ark-metadata/Cargo.toml @@ -18,6 +18,7 @@ starknet.workspace = true ark-starknet.workspace = true async-trait.workspace = true thiserror.workspace = true +chrono = "0.4" [dev-dependencies] ark-starknet = { path = "../ark-starknet", features = ["mock"] } diff --git a/crates/ark-metadata/src/metadata_manager.rs b/crates/ark-metadata/src/metadata_manager.rs index 831d86d7d..2cd2e55b4 100644 --- a/crates/ark-metadata/src/metadata_manager.rs +++ b/crates/ark-metadata/src/metadata_manager.rs @@ -85,11 +85,19 @@ impl<'a, T: Storage, C: StarknetClient, F: FileManager> MetadataManager<'a, T, C ipfs_gateway_uri: &str, image_timeout: Duration, ) -> Result<(), MetadataError> { + trace!( + "refresh_token_metadata(contract_address=0x{:064x}, token_id={})", + contract_address, + token_id.to_decimal(false), + ); + let token_uri = self .get_token_uri(&token_id, contract_address) .await .map_err(|err| MetadataError::ParsingError(err.to_string()))?; + trace!("Token URI: {}", token_uri); + let token_metadata = get_token_metadata( &self.request_client, token_uri.as_str(), @@ -108,15 +116,15 @@ impl<'a, T: Storage, C: StarknetClient, F: FileManager> MetadataManager<'a, T, C .map(|s| s.replace("ipfs://", &ipfs_url)) .unwrap_or_default(); - self.fetch_token_image( - url.as_str(), - cache, - contract_address, - &token_id, - image_timeout, - ) - .await - .map_err(|err| MetadataError::RequestImageError(err.to_string()))?; + let _ = self + .fetch_token_image( + url.as_str(), + cache, + contract_address, + &token_id, + image_timeout, + ) + .await; } self.storage @@ -208,6 +216,12 @@ impl<'a, T: Storage, C: StarknetClient, F: FileManager> MetadataManager<'a, T, C let headers = response.headers().clone(); let bytes = response.bytes().await?; let (content_type, content_length) = extract_metadata_from_headers(&headers)?; + + info!( + "Image: Content-Type={}, Content-Length={}", + content_type, content_length + ); + let file_ext = file_extension_from_mime_type(content_type.as_str()); debug!( diff --git a/crates/ark-metadata/src/types.rs b/crates/ark-metadata/src/types.rs index 0ce4b9fe5..3e0b87a23 100644 --- a/crates/ark-metadata/src/types.rs +++ b/crates/ark-metadata/src/types.rs @@ -65,6 +65,7 @@ pub struct MetadataAttribute { pub struct TokenMetadata { pub normalized: NormalizedMetadata, pub raw: String, + pub metadata_updated_at: Option, } #[derive(Debug, Default, Deserialize, Serialize)] diff --git a/crates/ark-metadata/src/utils.rs b/crates/ark-metadata/src/utils.rs index fabaae4ae..7773a49f4 100644 --- a/crates/ark-metadata/src/utils.rs +++ b/crates/ark-metadata/src/utils.rs @@ -1,6 +1,7 @@ use crate::types::{MetadataType, NormalizedMetadata, TokenMetadata}; use anyhow::{anyhow, Result}; use base64::{engine::general_purpose, Engine as _}; +use chrono::Utc; use reqwest::header::{HeaderMap, CONTENT_LENGTH, CONTENT_TYPE}; use reqwest::Client; use std::time::Duration; @@ -61,9 +62,12 @@ async fn fetch_metadata( Err(_) => NormalizedMetadata::default(), }; + let now = Utc::now(); + Ok(TokenMetadata { raw: raw_metadata, normalized: metadata, + metadata_updated_at: Some(now.timestamp()), }) } else { error!("Failed to get ipfs metadata. URI: {}", uri); @@ -79,6 +83,7 @@ async fn fetch_metadata( pub fn file_extension_from_mime_type(mime_type: &str) -> &str { match mime_type { + "model/gltf-binary" => "glb", "image/png" => "png", "image/jpeg" => "jpg", "image/gif" => "gif", @@ -113,6 +118,7 @@ fn get_onchain_metadata(uri: &str) -> Result { Ok(TokenMetadata { raw: decoded.to_string(), normalized: metadata, + metadata_updated_at: None, }) } Some(("data:application/json", uri)) => { @@ -120,6 +126,7 @@ fn get_onchain_metadata(uri: &str) -> Result { Ok(TokenMetadata { raw: uri.to_string(), normalized: metadata, + metadata_updated_at: None, }) } _ => match serde_json::from_str(uri) { @@ -234,4 +241,18 @@ mod tests { "Failed to extract or parse content length" ); } + + #[tokio::test] + async fn test_fetch_metadata() { + let client = Client::new(); + let uri = "https://example.com"; + let request_timeout_duration = Duration::from_secs(10); + + let metadata = fetch_metadata(uri, &client, request_timeout_duration).await; + assert!(metadata.is_ok()); + + let uri = "invalid_uri"; + let metadata = fetch_metadata(uri, &client, request_timeout_duration).await; + assert!(metadata.is_err()); + } } diff --git a/examples/pontos_sqlx.rs b/examples/pontos_sqlx.rs index bb7f67a56..662afa833 100644 --- a/examples/pontos_sqlx.rs +++ b/examples/pontos_sqlx.rs @@ -82,17 +82,24 @@ impl DefaultEventHandler { #[async_trait] impl EventHandler for DefaultEventHandler { - async fn on_terminated(&self, block_number: u64, indexation_progress: f64) { + async fn on_block_processed(&self, block_number: u64, indexation_progress: f64) { println!( "pontos: block processed: block_number={}, indexation_progress={}", block_number, indexation_progress ); } - async fn on_block_processing(&self, block_number: u64) { + async fn on_block_processing(&self, block_timestamp: u64, block_number: Option) { // TODO: here we want to call some storage if needed from an other object. // But it's totally unrelated to the core process, so we can do whatever we want here. - println!("pontos: processing block: block_number={}", block_number); + println!( + "pontos: processing block: block_timestamp={}, block_number={:?}", + block_timestamp, block_number + ); + } + + async fn on_indexation_range_completed(&self) { + println!("pontos: indexation range completed"); } async fn on_token_registered(&self, token: TokenInfo) {