From e43a2e2c4b0fa9a933d18921e61e716596ee9f0b Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Tue, 22 Oct 2024 16:16:06 +0200 Subject: [PATCH 01/32] HTTP client auth as trait extension --- .gitignore | 1 + src/http.rs | 269 +++++++++++++++++++++++++++++++++++++-- src/sinks/axiom.rs | 1 + src/sinks/http/config.rs | 9 +- 4 files changed, 267 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 24b0593d51d6c..eebe811ef8ab2 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ checkpoints/* miniodat bench_output.txt sample.log +vector.yaml scripts/package-lock.json target node_modules diff --git a/src/http.rs b/src/http.rs index f8e1c939c58c8..8b47fa5023ee4 100644 --- a/src/http.rs +++ b/src/http.rs @@ -1,4 +1,5 @@ #![allow(missing_docs)] +use axum::async_trait; use futures::future::BoxFuture; use headers::{Authorization, HeaderMapExt}; use http::{ @@ -13,14 +14,13 @@ use hyper::{ use hyper_openssl::HttpsConnector; use hyper_proxy::ProxyConnector; use rand::Rng; +use serde::Deserialize; use serde_with::serde_as; use snafu::{ResultExt, Snafu}; use std::{ - fmt, - net::SocketAddr, - task::{Context, Poll}, - time::Duration, + error::Error, fmt, net::SocketAddr, sync::{Arc, Mutex}, task::{Context, Poll}, time::{Duration, SystemTime, UNIX_EPOCH} }; +use bytes::Buf; use tokio::time::Instant; use tower::{Layer, Service}; use tower_http::{ @@ -28,7 +28,7 @@ use tower_http::{ trace::TraceLayer, }; use tracing::{Instrument, Span}; -use vector_lib::configurable::configurable_component; +use vector_lib::{configurable::configurable_component, tls::{TlsConfig, TlsSettings}}; use vector_lib::sensitive_string::SensitiveString; use crate::{ @@ -72,31 +72,173 @@ impl HttpError { pub type HttpClientFuture = >>::Future; type HttpProxyConnector = ProxyConnector>; +#[async_trait] +trait AuthExtension: Send + Sync +where + B: fmt::Debug + HttpBody + Send + 'static, + B::Data: Send, + B::Error: Into + Send, +{ + async fn modify_request(&self, req: &mut Request); +} + +#[derive(Clone)] +struct OAuth2Extension +{ + token_endpoint: String, + client_id: String, + client_secret: SensitiveString, + http_client: HttpClient, + token: Arc>> +} + +#[derive(Clone)] +struct BasicAuthExtension { + user: String, + password: SensitiveString, +} + +#[derive(Debug, Deserialize)] +struct Token { + access_token: String, + expires_in: u32 +} + +#[derive(Debug, Clone)] +struct ExpirableToken { + access_token: String, + expires_after_ms: u128 +} + +impl OAuth2Extension +{ + async fn get_token(&self) -> String { + if let Some(token) = self.acquire_token_from_cache() { + return token.access_token; + } + + //no valid token in cache (or no token at all) + let new_token = self.request_token().await.unwrap(); + let token_to_return = new_token.access_token.clone(); + self.save_into_cache(new_token); + + token_to_return + } + + fn acquire_token_from_cache(&self) -> Option { + let now = SystemTime::now(); + let since_the_epoch = now.duration_since(UNIX_EPOCH).unwrap(); + let maybe_token = self.token.lock().unwrap(); + match &*maybe_token { + Some(token) => { + if since_the_epoch.as_millis() < token.expires_after_ms { + //we have token, token is valid for at least 1min, we can use it. + return Some(token.clone()); + } + + return None + }, + _ => None, + } + } + + fn save_into_cache(&self, token: ExpirableToken) { + self.token.lock().unwrap().replace(token); + } + + async fn request_token(&self) -> Result> { + let request_body = format!("client_secret={}&grant_type=client_credentials&response_type=token&client_id={}", self.client_secret.inner(), self.client_id); + + let builder = Request::post(self.token_endpoint.clone()); + let builder = builder.header("Content-Type", "application/x-www-form-urlencoded"); + let request = builder.body(Body::from(request_body)).expect("error creating request"); + + let response = self.http_client.send(request).await.unwrap(); + + let body = hyper::body::aggregate(response).await.unwrap(); + let token: Token = serde_json::from_reader(body.reader()).unwrap(); + + //expires_in means, in seconds, for how long it will be valid, lets say 5min, + //to not cause some random 4xx, because token expired in the meantime, we will make some + //room for token refreshing, this room is 1min (60seconds) + let token_is_valid_for_ms = (token.expires_in - 60) * 1000; + let now = SystemTime::now(); + let since_the_epoch = now.duration_since(UNIX_EPOCH).unwrap(); + let token_will_expire_after_ms = since_the_epoch.as_millis() + (token_is_valid_for_ms as u128); + + Ok(ExpirableToken{access_token:token.access_token, expires_after_ms: token_will_expire_after_ms}) + } +} + +#[async_trait] +impl AuthExtension for OAuth2Extension +where + B: fmt::Debug + HttpBody + Send + 'static, + B::Data: Send, + B::Error: Into + Send, +{ + async fn modify_request(&self, req: &mut Request) + { + let token = self.get_token().await; + let auth = Auth::Bearer{ token: SensitiveString::from(token)}; + auth.apply(req); + } +} + + +#[async_trait] +impl AuthExtension for BasicAuthExtension +where + B: fmt::Debug + HttpBody + Send + 'static, + B::Data: Send, + B::Error: Into + Send, +{ + async fn modify_request(&self, req: &mut Request) + { + let user = self.user.clone(); + let password = self.password.clone(); + + let auth = Auth::Basic{ user, password }; + auth.apply(req); + } +} + pub struct HttpClient { client: Client, user_agent: HeaderValue, proxy_connector: HttpProxyConnector, + auth_extension: Option>>> } impl HttpClient where B: fmt::Debug + HttpBody + Send + 'static, B::Data: Send, - B::Error: Into, + B::Error: Into + Send, { pub fn new( tls_settings: impl Into, proxy_config: &ProxyConfig, ) -> Result, HttpError> { - HttpClient::new_with_custom_client(tls_settings, proxy_config, &mut Client::builder()) + HttpClient::new_with_custom_client(tls_settings, proxy_config, &mut Client::builder(), None) + } + + pub fn new_with_auth_extension( + tls_settings: impl Into, + proxy_config: &ProxyConfig, + auth_config: Option + ) -> Result, HttpError> { + HttpClient::new_with_custom_client(tls_settings, proxy_config, &mut Client::builder(), auth_config) } pub fn new_with_custom_client( tls_settings: impl Into, proxy_config: &ProxyConfig, client_builder: &mut client::Builder, + auth_config: Option, ) -> Result, HttpError> { let proxy_connector = build_proxy_connector(tls_settings.into(), proxy_config)?; + let auth_extension = build_auth_extension(auth_config, proxy_config); let client = client_builder.build(proxy_connector.clone()); let app_name = crate::get_app_name(); @@ -108,6 +250,7 @@ where client, user_agent, proxy_connector, + auth_extension }) } @@ -119,13 +262,21 @@ where let _enter = span.enter(); default_request_headers(&mut request, &self.user_agent); - self.maybe_add_proxy_headers(&mut request); - - emit!(http_client::AboutToSendHttpRequest { request: &request }); + self.maybe_add_proxy_headers(&mut request); - let response = self.client.request(request); + let client = self.client.clone(); + let request_extension = self.auth_extension.clone(); let fut = async move { + + emit!(http_client::AboutToSendHttpRequest { request: &request }); + + if let Some(request_extension) = request_extension { + request_extension.modify_request(&mut request).await; + } + + let response: client::ResponseFuture = client.request(request); + // Capture the time right before we issue the request. // Request doesn't start the processing until we start polling it. let before = std::time::Instant::now(); @@ -169,6 +320,42 @@ where } } +fn build_auth_extension(http_client_authorization_strategy: Option, + proxy_config: &ProxyConfig, +) -> Option>>> +where + B: fmt::Debug + HttpBody + Send + 'static, + B::Data: Send, + B::Error: Into + Send, +{ + if let Some(http_client_authorization_strategy) = http_client_authorization_strategy { + match http_client_authorization_strategy.auth { + HttpClientAuthorizationStrategy::Basic { user, password } => { + let basic_auth_extension = BasicAuthExtension{user, password}; + return Some(Arc::new(Box::new(basic_auth_extension))); + }, + HttpClientAuthorizationStrategy::OAuth2 { token_endpoint, client_id, client_secret } => { + let tls_for_auth = http_client_authorization_strategy.tls.clone(); + let tls_for_auth: TlsSettings = TlsSettings::from_options(&tls_for_auth).unwrap(); + let empty_token = Arc::new(Mutex::new(None)); + let auth_client = HttpClient::new(tls_for_auth, proxy_config).unwrap(); + + let oauth2_extension = OAuth2Extension { + token_endpoint, + client_id, + client_secret, + http_client: auth_client, + token: empty_token + }; + + return Some(Arc::new(Box::new(oauth2_extension))); + }, + } + } + + None +} + pub fn build_proxy_connector( tls_settings: MaybeTlsSettings, proxy_config: &ProxyConfig, @@ -249,6 +436,7 @@ impl Clone for HttpClient { client: self.client.clone(), user_agent: self.user_agent.clone(), proxy_connector: self.proxy_connector.clone(), + auth_extension: self.auth_extension.clone() } } } @@ -262,6 +450,65 @@ impl fmt::Debug for HttpClient { } } +/// Configuration for HTTP client providing an authentication mechanism. +#[configurable_component] +#[configurable(metadata(docs::advanced))] +#[derive(Clone, Debug)] +#[serde(deny_unknown_fields)] +pub struct HttpClientAuthorizationConfig { + /// Define how to authorize against an upstream. + #[configurable] + auth: HttpClientAuthorizationStrategy, + + /// The TLS settings for the http client's connection. + /// + /// Optional, constrains TLS settings for this http client. + #[configurable(derived)] + tls: Option, +} + +/// Configuration of the authentication strategy for HTTP requests. +/// +/// HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an +/// HTTP header without any additional encryption beyond what is provided by the transport itself. +#[configurable_component] +#[derive(Clone, Debug)] +#[serde(deny_unknown_fields, rename_all = "snake_case", tag = "strategy")] +#[configurable(metadata(docs::enum_tag_description = "The authentication strategy to use."))] +pub enum HttpClientAuthorizationStrategy { + /// Basic authentication. + /// + /// The username and password are concatenated and encoded via [base64][base64]. + /// + /// [base64]: https://en.wikipedia.org/wiki/Base64 + Basic { + /// The basic authentication username. + #[configurable(metadata(docs::examples = "username"))] + user: String, + + /// The basic authentication password. + #[configurable(metadata(docs::examples = "password"))] + password: SensitiveString, + }, + + /// Authentication based on OAuth 2.0 protocol. + /// + /// This strategy allows to dynamically acquire and use token based on provided parameters. + OAuth2 { + /// Token endpoint location, required for token acquisition. + #[configurable(metadata(docs::examples = "https://auth.provider/oauth/token"))] + token_endpoint: String, + + /// The client id. + #[configurable(metadata(docs::examples = "client_id"))] + client_id: String, + + /// The sensitive client secret. + #[configurable(metadata(docs::examples = "client_secret"))] + client_secret: SensitiveString, + }, +} + /// Configuration of the authentication strategy for HTTP requests. /// /// HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an diff --git a/src/sinks/axiom.rs b/src/sinks/axiom.rs index dfc4ab124dcc1..0e149aa123693 100644 --- a/src/sinks/axiom.rs +++ b/src/sinks/axiom.rs @@ -119,6 +119,7 @@ impl SinkConfig for AxiomConfig { }), method: HttpMethod::Post, tls: self.tls.clone(), + http_client_authorization_strategy: None, request, acknowledgements: self.acknowledgements, batch: self.batch, diff --git a/src/sinks/http/config.rs b/src/sinks/http/config.rs index ccc9ace780046..e09699c77f438 100644 --- a/src/sinks/http/config.rs +++ b/src/sinks/http/config.rs @@ -8,9 +8,10 @@ use vector_lib::codecs::{ CharacterDelimitedEncoder, }; + use crate::{ codecs::{EncodingConfigWithFraming, SinkType}, - http::{Auth, HttpClient, MaybeAuth}, + http::{Auth, HttpClient, MaybeAuth, HttpClientAuthorizationConfig}, sinks::{ prelude::*, util::{ @@ -90,6 +91,9 @@ pub struct HttpSinkConfig { #[configurable(derived)] pub tls: Option, + #[configurable(derived)] + pub http_client_authorization_strategy: Option, + #[configurable(derived)] #[serde( default, @@ -153,7 +157,8 @@ impl From for Method { impl HttpSinkConfig { fn build_http_client(&self, cx: &SinkContext) -> crate::Result { let tls = TlsSettings::from_options(&self.tls)?; - Ok(HttpClient::new(tls, cx.proxy())?) + let auth_strategy = self.http_client_authorization_strategy.clone(); + Ok(HttpClient::new_with_auth_extension(tls, cx.proxy(), auth_strategy)?) } pub(super) fn build_encoder(&self) -> crate::Result> { From 887532924e3c325a66685051ee3ac8ddba22c858 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Tue, 22 Oct 2024 17:20:37 +0200 Subject: [PATCH 02/32] Use Client directly --- src/http.rs | 48 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/src/http.rs b/src/http.rs index 8b47fa5023ee4..a2b67d6bef233 100644 --- a/src/http.rs +++ b/src/http.rs @@ -88,7 +88,7 @@ struct OAuth2Extension token_endpoint: String, client_id: String, client_secret: SensitiveString, - http_client: HttpClient, + client: Client, token: Arc>> } @@ -153,7 +153,20 @@ impl OAuth2Extension let builder = builder.header("Content-Type", "application/x-www-form-urlencoded"); let request = builder.body(Body::from(request_body)).expect("error creating request"); - let response = self.http_client.send(request).await.unwrap(); + let before = std::time::Instant::now(); + let response_result = self.client.request(request).await; + let roundtrip = before.elapsed(); + + let response = response_result + .inspect_err(|error| { + emit!(http_client::GotHttpWarning { error, roundtrip }); + }) + .context(CallRequestSnafu)?; + + emit!(http_client::GotHttpResponse { + response: &response, + roundtrip + }); let body = hyper::body::aggregate(response).await.unwrap(); let token: Token = serde_json::from_reader(body.reader()).unwrap(); @@ -185,7 +198,6 @@ where } } - #[async_trait] impl AuthExtension for BasicAuthExtension where @@ -207,7 +219,7 @@ pub struct HttpClient { client: Client, user_agent: HeaderValue, proxy_connector: HttpProxyConnector, - auth_extension: Option>>> + auth_extension: Option>> } impl HttpClient @@ -238,7 +250,7 @@ where auth_config: Option, ) -> Result, HttpError> { let proxy_connector = build_proxy_connector(tls_settings.into(), proxy_config)?; - let auth_extension = build_auth_extension(auth_config, proxy_config); + let auth_extension = build_auth_extension(auth_config, proxy_config, client_builder); let client = client_builder.build(proxy_connector.clone()); let app_name = crate::get_app_name(); @@ -265,16 +277,19 @@ where self.maybe_add_proxy_headers(&mut request); let client = self.client.clone(); - let request_extension = self.auth_extension.clone(); + let auth_extension = self.auth_extension.clone(); let fut = async move { - emit!(http_client::AboutToSendHttpRequest { request: &request }); - - if let Some(request_extension) = request_extension { - request_extension.modify_request(&mut request).await; + //should request for token influence upstream service latency ? + if let Some(auth_extension) = auth_extension { + let auth_span = tracing::info_span!("auth_extension"); + auth_extension.modify_request(&mut request) + .instrument(auth_span.clone().or_current()) + .await; } + emit!(http_client::AboutToSendHttpRequest { request: &request }); let response: client::ResponseFuture = client.request(request); // Capture the time right before we issue the request. @@ -322,7 +337,8 @@ where fn build_auth_extension(http_client_authorization_strategy: Option, proxy_config: &ProxyConfig, -) -> Option>>> + client_builder: &mut client::Builder, +) -> Option>> where B: fmt::Debug + HttpBody + Send + 'static, B::Data: Send, @@ -332,23 +348,25 @@ where match http_client_authorization_strategy.auth { HttpClientAuthorizationStrategy::Basic { user, password } => { let basic_auth_extension = BasicAuthExtension{user, password}; - return Some(Arc::new(Box::new(basic_auth_extension))); + return Some(Arc::new(basic_auth_extension)); }, HttpClientAuthorizationStrategy::OAuth2 { token_endpoint, client_id, client_secret } => { let tls_for_auth = http_client_authorization_strategy.tls.clone(); let tls_for_auth: TlsSettings = TlsSettings::from_options(&tls_for_auth).unwrap(); let empty_token = Arc::new(Mutex::new(None)); - let auth_client = HttpClient::new(tls_for_auth, proxy_config).unwrap(); + + let auth_proxy_connector = build_proxy_connector(tls_for_auth.into(), proxy_config).unwrap(); + let auth_client = client_builder.build(auth_proxy_connector.clone()); let oauth2_extension = OAuth2Extension { token_endpoint, client_id, client_secret, - http_client: auth_client, + client: auth_client, token: empty_token }; - return Some(Arc::new(Box::new(oauth2_extension))); + return Some(Arc::new(oauth2_extension)); }, } } From 0f15ea340a55da6aab899791e7229efdd2cf7198 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Tue, 22 Oct 2024 23:52:16 +0200 Subject: [PATCH 03/32] Unavailable oauth server should not crash vector --- src/http.rs | 60 +++++++++++++++++++++++------- src/internal_events/http_client.rs | 18 +++++++++ 2 files changed, 65 insertions(+), 13 deletions(-) diff --git a/src/http.rs b/src/http.rs index a2b67d6bef233..a9420791eba44 100644 --- a/src/http.rs +++ b/src/http.rs @@ -56,6 +56,8 @@ pub enum HttpError { CallRequest { source: hyper::Error }, #[snafu(display("Failed to build HTTP request: {}", source))] BuildRequest { source: http::Error }, + #[snafu(display("Failed to acquire authentication resource."))] + ExtensionAuthentication, } impl HttpError { @@ -64,6 +66,7 @@ impl HttpError { HttpError::BuildRequest { .. } | HttpError::MakeProxyConnector { .. } => false, HttpError::CallRequest { .. } | HttpError::BuildTlsConnector { .. } + | HttpError::ExtensionAuthentication { .. } | HttpError::MakeHttpsConnector { .. } => true, } } @@ -79,7 +82,7 @@ where B::Data: Send, B::Error: Into + Send, { - async fn modify_request(&self, req: &mut Request); + async fn modify_request(&self, req: &mut Request) -> Result<(), Box>; } #[derive(Clone)] @@ -112,17 +115,17 @@ struct ExpirableToken { impl OAuth2Extension { - async fn get_token(&self) -> String { + async fn get_token(&self) -> Result> { if let Some(token) = self.acquire_token_from_cache() { - return token.access_token; + return Ok(token.access_token); } //no valid token in cache (or no token at all) - let new_token = self.request_token().await.unwrap(); + let new_token = self.request_token().await?; let token_to_return = new_token.access_token.clone(); self.save_into_cache(new_token); - token_to_return + Ok(token_to_return) } fn acquire_token_from_cache(&self) -> Option { @@ -168,8 +171,14 @@ impl OAuth2Extension roundtrip }); - let body = hyper::body::aggregate(response).await.unwrap(); - let token: Token = serde_json::from_reader(body.reader()).unwrap(); + if !response.status().is_success() { + let body_bytes = hyper::body::aggregate(response).await?; + let body_str = std::str::from_utf8(body_bytes.chunk())?.to_string(); + return Err(Box::new(AcquireTokenError{message: body_str})); + } + + let body = hyper::body::aggregate(response).await?; + let token: Token = serde_json::from_reader(body.reader())?; //expires_in means, in seconds, for how long it will be valid, lets say 5min, //to not cause some random 4xx, because token expired in the meantime, we will make some @@ -183,6 +192,19 @@ impl OAuth2Extension } } +#[derive(Debug)] +pub struct AcquireTokenError { + message: String +} + +impl fmt::Display for AcquireTokenError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Server error from authentication server: {}", self.message) + } +} + +impl Error for AcquireTokenError {} + #[async_trait] impl AuthExtension for OAuth2Extension where @@ -190,11 +212,13 @@ where B::Data: Send, B::Error: Into + Send, { - async fn modify_request(&self, req: &mut Request) + async fn modify_request(&self, req: &mut Request) -> Result<(), Box> { - let token = self.get_token().await; + let token = self.get_token().await?; let auth = Auth::Bearer{ token: SensitiveString::from(token)}; auth.apply(req); + + Ok(()) } } @@ -205,13 +229,15 @@ where B::Data: Send, B::Error: Into + Send, { - async fn modify_request(&self, req: &mut Request) + async fn modify_request(&self, req: &mut Request) -> Result<(), Box> { let user = self.user.clone(); let password = self.password.clone(); let auth = Auth::Basic{ user, password }; auth.apply(req); + + Ok(()) } } @@ -280,13 +306,21 @@ where let auth_extension = self.auth_extension.clone(); let fut = async move { - //should request for token influence upstream service latency ? if let Some(auth_extension) = auth_extension { let auth_span = tracing::info_span!("auth_extension"); - auth_extension.modify_request(&mut request) + let res = auth_extension.modify_request(&mut request) .instrument(auth_span.clone().or_current()) - .await; + .await + .inspect_err(|error| { + // Emit the error into the internal events system. + emit!(http_client::GotAuthExtensionError{error}); + }) + .map_err(|_err| HttpError::ExtensionAuthentication); + + if let Err(e) = res { + return Err(e) + } } emit!(http_client::AboutToSendHttpRequest { request: &request }); diff --git a/src/internal_events/http_client.rs b/src/internal_events/http_client.rs index 2584ef9c1254d..03974f34aadb3 100644 --- a/src/internal_events/http_client.rs +++ b/src/internal_events/http_client.rs @@ -95,6 +95,24 @@ impl<'a> InternalEvent for GotHttpWarning<'a> { } } +#[derive(Debug)] +pub struct GotAuthExtensionError<'a> { + pub error: &'a Box, +} + +impl<'a> InternalEvent for GotAuthExtensionError<'a> { + fn emit(self) { + warn!( + message = "HTTP Auth extension error.", + error = %self.error, + error_type = error_type::REQUEST_FAILED, + stage = error_stage::PROCESSING, + internal_log_rate_limit = true, + ); + counter!("http_auth_ext_client_errors_total", "error_kind" => self.error.to_string()).increment(1); + } +} + /// Newtype placeholder to provide a formatter for the request and response body. struct FormatBody<'a, B>(&'a B); From 3c87e8d9bc97f5dcad6808e86402838f24e1a1ab Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Wed, 23 Oct 2024 14:28:47 +0200 Subject: [PATCH 04/32] Fix snafu error to by sync + send --- src/http.rs | 25 +++++++++++-------------- src/internal_events/http_client.rs | 2 +- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/http.rs b/src/http.rs index a9420791eba44..67eb19f3d19bd 100644 --- a/src/http.rs +++ b/src/http.rs @@ -57,7 +57,9 @@ pub enum HttpError { #[snafu(display("Failed to build HTTP request: {}", source))] BuildRequest { source: http::Error }, #[snafu(display("Failed to acquire authentication resource."))] - ExtensionAuthentication, + ExtensionAuthentication { + source: Box, + }, } impl HttpError { @@ -82,7 +84,7 @@ where B::Data: Send, B::Error: Into + Send, { - async fn modify_request(&self, req: &mut Request) -> Result<(), Box>; + async fn modify_request(&self, req: &mut Request) -> Result<(), Box>; } #[derive(Clone)] @@ -115,7 +117,7 @@ struct ExpirableToken { impl OAuth2Extension { - async fn get_token(&self) -> Result> { + async fn get_token(&self) -> Result> { if let Some(token) = self.acquire_token_from_cache() { return Ok(token.access_token); } @@ -149,7 +151,7 @@ impl OAuth2Extension self.token.lock().unwrap().replace(token); } - async fn request_token(&self) -> Result> { + async fn request_token(&self) -> Result> { let request_body = format!("client_secret={}&grant_type=client_credentials&response_type=token&client_id={}", self.client_secret.inner(), self.client_id); let builder = Request::post(self.token_endpoint.clone()); @@ -163,8 +165,7 @@ impl OAuth2Extension let response = response_result .inspect_err(|error| { emit!(http_client::GotHttpWarning { error, roundtrip }); - }) - .context(CallRequestSnafu)?; + })?; emit!(http_client::GotHttpResponse { response: &response, @@ -212,7 +213,7 @@ where B::Data: Send, B::Error: Into + Send, { - async fn modify_request(&self, req: &mut Request) -> Result<(), Box> + async fn modify_request(&self, req: &mut Request) -> Result<(), Box> { let token = self.get_token().await?; let auth = Auth::Bearer{ token: SensitiveString::from(token)}; @@ -229,7 +230,7 @@ where B::Data: Send, B::Error: Into + Send, { - async fn modify_request(&self, req: &mut Request) -> Result<(), Box> + async fn modify_request(&self, req: &mut Request) -> Result<(), Box> { let user = self.user.clone(); let password = self.password.clone(); @@ -309,18 +310,14 @@ where //should request for token influence upstream service latency ? if let Some(auth_extension) = auth_extension { let auth_span = tracing::info_span!("auth_extension"); - let res = auth_extension.modify_request(&mut request) + auth_extension.modify_request(&mut request) .instrument(auth_span.clone().or_current()) .await .inspect_err(|error| { // Emit the error into the internal events system. emit!(http_client::GotAuthExtensionError{error}); }) - .map_err(|_err| HttpError::ExtensionAuthentication); - - if let Err(e) = res { - return Err(e) - } + .context(ExtensionAuthenticationSnafu)?; } emit!(http_client::AboutToSendHttpRequest { request: &request }); diff --git a/src/internal_events/http_client.rs b/src/internal_events/http_client.rs index 03974f34aadb3..0dd23b3d0434e 100644 --- a/src/internal_events/http_client.rs +++ b/src/internal_events/http_client.rs @@ -97,7 +97,7 @@ impl<'a> InternalEvent for GotHttpWarning<'a> { #[derive(Debug)] pub struct GotAuthExtensionError<'a> { - pub error: &'a Box, + pub error: &'a Box, } impl<'a> InternalEvent for GotAuthExtensionError<'a> { From e37eaf1b8c67ca3358003c345c7dfbc71b5b7799 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Thu, 24 Oct 2024 01:33:04 +0200 Subject: [PATCH 05/32] review fixes --- .gitignore | 1 - src/http.rs | 41 ++++++++++++++++++++++++------ src/internal_events/http_client.rs | 13 +++++++--- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index eebe811ef8ab2..24b0593d51d6c 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ checkpoints/* miniodat bench_output.txt sample.log -vector.yaml scripts/package-lock.json target node_modules diff --git a/src/http.rs b/src/http.rs index 67eb19f3d19bd..aaa70ed1a6ab6 100644 --- a/src/http.rs +++ b/src/http.rs @@ -1,5 +1,5 @@ #![allow(missing_docs)] -use axum::async_trait; +use async_trait::async_trait; use futures::future::BoxFuture; use headers::{Authorization, HeaderMapExt}; use http::{ @@ -57,7 +57,7 @@ pub enum HttpError { #[snafu(display("Failed to build HTTP request: {}", source))] BuildRequest { source: http::Error }, #[snafu(display("Failed to acquire authentication resource."))] - ExtensionAuthentication { + AuthenticationExtension { source: Box, }, } @@ -68,7 +68,7 @@ impl HttpError { HttpError::BuildRequest { .. } | HttpError::MakeProxyConnector { .. } => false, HttpError::CallRequest { .. } | HttpError::BuildTlsConnector { .. } - | HttpError::ExtensionAuthentication { .. } + | HttpError::AuthenticationExtension { .. } | HttpError::MakeHttpsConnector { .. } => true, } } @@ -93,6 +93,7 @@ struct OAuth2Extension token_endpoint: String, client_id: String, client_secret: SensitiveString, + grace_period: u32, client: Client, token: Arc>> } @@ -183,8 +184,17 @@ impl OAuth2Extension //expires_in means, in seconds, for how long it will be valid, lets say 5min, //to not cause some random 4xx, because token expired in the meantime, we will make some - //room for token refreshing, this room is 1min (60seconds) - let token_is_valid_for_ms = (token.expires_in - 60) * 1000; + //room for token refreshing, this room is a grace_period. + let (mut grace_period_seconds, overflow) = token.expires_in.overflowing_sub(self.grace_period); + + //if time for grace period exceed an expire_in, it basically means: always use new token. + if overflow { + grace_period_seconds = 0; + } + + let token_is_valid_for_ms : u128 = grace_period_seconds as u128 * 1000; + //we are multiplying by 1000 becuase expires_in field is in seconds, grace_period also, + //but later we oparate on miliseconds. let now = SystemTime::now(); let since_the_epoch = now.duration_since(UNIX_EPOCH).unwrap(); let token_will_expire_after_ms = since_the_epoch.as_millis() + (token_is_valid_for_ms as u128); @@ -315,9 +325,9 @@ where .await .inspect_err(|error| { // Emit the error into the internal events system. - emit!(http_client::GotAuthExtensionError{error}); + emit!(http_client::AuthExtensionError{error}); }) - .context(ExtensionAuthenticationSnafu)?; + .context(AuthenticationExtensionSnafu)?; } emit!(http_client::AboutToSendHttpRequest { request: &request }); @@ -381,7 +391,7 @@ where let basic_auth_extension = BasicAuthExtension{user, password}; return Some(Arc::new(basic_auth_extension)); }, - HttpClientAuthorizationStrategy::OAuth2 { token_endpoint, client_id, client_secret } => { + HttpClientAuthorizationStrategy::OAuth2 { token_endpoint, client_id, client_secret, grace_period } => { let tls_for_auth = http_client_authorization_strategy.tls.clone(); let tls_for_auth: TlsSettings = TlsSettings::from_options(&tls_for_auth).unwrap(); let empty_token = Arc::new(Mutex::new(None)); @@ -393,6 +403,7 @@ where token_endpoint, client_id, client_secret, + grace_period, client: auth_client, token: empty_token }; @@ -555,9 +566,23 @@ pub enum HttpClientAuthorizationStrategy { /// The sensitive client secret. #[configurable(metadata(docs::examples = "client_secret"))] client_secret: SensitiveString, + + /// The grace period configuration for a bearer token. + /// To avoid random authorization failures caused by expired token exception, + /// we will acquire new token, some time (grace period) before current token will be expired, + /// because of that, we will always execute request with fresh enought token. + #[serde(default = "default_oauth2_token_grace_period")] + #[configurable(metadata(docs::examples = 300))] + #[configurable(metadata(docs::type_unit = "seconds"))] + #[configurable(metadata(docs::human_name = "Grace period for bearer token."))] + grace_period: u32, }, } +const fn default_oauth2_token_grace_period() -> u32 { + 300 // 5 minutes +} + /// Configuration of the authentication strategy for HTTP requests. /// /// HTTP authentication should be used with HTTPS only, as the authentication credentials are passed as an diff --git a/src/internal_events/http_client.rs b/src/internal_events/http_client.rs index 0dd23b3d0434e..e231ac5971f5d 100644 --- a/src/internal_events/http_client.rs +++ b/src/internal_events/http_client.rs @@ -96,20 +96,25 @@ impl<'a> InternalEvent for GotHttpWarning<'a> { } #[derive(Debug)] -pub struct GotAuthExtensionError<'a> { +pub struct AuthExtensionError<'a> { pub error: &'a Box, } -impl<'a> InternalEvent for GotAuthExtensionError<'a> { +impl<'a> InternalEvent for AuthExtensionError<'a> { fn emit(self) { - warn!( + error!( message = "HTTP Auth extension error.", error = %self.error, error_type = error_type::REQUEST_FAILED, stage = error_stage::PROCESSING, internal_log_rate_limit = true, ); - counter!("http_auth_ext_client_errors_total", "error_kind" => self.error.to_string()).increment(1); + counter!( + "component_errors_total", + "error_type" => error_type::CONFIGURATION_FAILED, + "stage" => error_stage::SENDING, + ) + .increment(1); } } From 0c4480963f14c76500c0ab165b1ddbe312e4e420 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Thu, 24 Oct 2024 01:40:49 +0200 Subject: [PATCH 06/32] review fixes --- src/http.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/http.rs b/src/http.rs index aaa70ed1a6ab6..93586688db4f8 100644 --- a/src/http.rs +++ b/src/http.rs @@ -193,8 +193,8 @@ impl OAuth2Extension } let token_is_valid_for_ms : u128 = grace_period_seconds as u128 * 1000; - //we are multiplying by 1000 becuase expires_in field is in seconds, grace_period also, - //but later we oparate on miliseconds. + //we are multiplying by 1000 because expires_in field is in seconds, grace_period also, + //but later we operate on milliseconds. let now = SystemTime::now(); let since_the_epoch = now.duration_since(UNIX_EPOCH).unwrap(); let token_will_expire_after_ms = since_the_epoch.as_millis() + (token_is_valid_for_ms as u128); From 31d0327d1cbb9af826b5e826a1b290ce0d089131 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Thu, 24 Oct 2024 01:44:05 +0200 Subject: [PATCH 07/32] review fixes --- src/http.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http.rs b/src/http.rs index 93586688db4f8..d29ad70141e56 100644 --- a/src/http.rs +++ b/src/http.rs @@ -570,7 +570,7 @@ pub enum HttpClientAuthorizationStrategy { /// The grace period configuration for a bearer token. /// To avoid random authorization failures caused by expired token exception, /// we will acquire new token, some time (grace period) before current token will be expired, - /// because of that, we will always execute request with fresh enought token. + /// because of that, we will always execute request with fresh enough token. #[serde(default = "default_oauth2_token_grace_period")] #[configurable(metadata(docs::examples = 300))] #[configurable(metadata(docs::type_unit = "seconds"))] From 8f84adf92ea2ff990e1531641c14a3466d806cd3 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Mon, 28 Oct 2024 20:24:18 +0100 Subject: [PATCH 08/32] acquire token with oauth2 and mtls as described in rfc --- src/http.rs | 57 +++++++++++++++++++++++++++++++--------- src/sinks/axiom.rs | 2 +- src/sinks/http/config.rs | 6 ++--- 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/http.rs b/src/http.rs index d29ad70141e56..6d5ad85b03d76 100644 --- a/src/http.rs +++ b/src/http.rs @@ -92,7 +92,7 @@ struct OAuth2Extension { token_endpoint: String, client_id: String, - client_secret: SensitiveString, + client_secret: Option, grace_period: u32, client: Client, token: Arc>> @@ -153,8 +153,15 @@ impl OAuth2Extension } async fn request_token(&self) -> Result> { - let request_body = format!("client_secret={}&grant_type=client_credentials&response_type=token&client_id={}", self.client_secret.inner(), self.client_id); - + let mut request_body = format!("grant_type=client_credentials&client_id={}", self.client_id); + + //in case of oauth2 with mtls (https://datatracker.ietf.org/doc/html/rfc8705) we only pass client_id, + //so secret can be considiered as optional. + if let Some(client_secret) = &self.client_secret { + let secret_param = format!("&client_secret={}", client_secret.inner()); + request_body.push_str(&secret_param); + } + let builder = Request::post(self.token_endpoint.clone()); let builder = builder.header("Content-Type", "application/x-www-form-urlencoded"); let request = builder.body(Body::from(request_body)).expect("error creating request"); @@ -275,7 +282,7 @@ where pub fn new_with_auth_extension( tls_settings: impl Into, proxy_config: &ProxyConfig, - auth_config: Option + auth_config: Option ) -> Result, HttpError> { HttpClient::new_with_custom_client(tls_settings, proxy_config, &mut Client::builder(), auth_config) } @@ -284,7 +291,7 @@ where tls_settings: impl Into, proxy_config: &ProxyConfig, client_builder: &mut client::Builder, - auth_config: Option, + auth_config: Option, ) -> Result, HttpError> { let proxy_connector = build_proxy_connector(tls_settings.into(), proxy_config)?; let auth_extension = build_auth_extension(auth_config, proxy_config, client_builder); @@ -376,7 +383,7 @@ where } } -fn build_auth_extension(http_client_authorization_strategy: Option, +fn build_auth_extension(authorization_config: Option, proxy_config: &ProxyConfig, client_builder: &mut client::Builder, ) -> Option>> @@ -385,14 +392,14 @@ where B::Data: Send, B::Error: Into + Send, { - if let Some(http_client_authorization_strategy) = http_client_authorization_strategy { - match http_client_authorization_strategy.auth { + if let Some(authorization_config) = authorization_config { + match authorization_config.strategy { HttpClientAuthorizationStrategy::Basic { user, password } => { let basic_auth_extension = BasicAuthExtension{user, password}; return Some(Arc::new(basic_auth_extension)); }, HttpClientAuthorizationStrategy::OAuth2 { token_endpoint, client_id, client_secret, grace_period } => { - let tls_for_auth = http_client_authorization_strategy.tls.clone(); + let tls_for_auth = authorization_config.tls.clone(); let tls_for_auth: TlsSettings = TlsSettings::from_options(&tls_for_auth).unwrap(); let empty_token = Arc::new(Mutex::new(None)); @@ -515,10 +522,10 @@ impl fmt::Debug for HttpClient { #[configurable(metadata(docs::advanced))] #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] -pub struct HttpClientAuthorizationConfig { +pub struct AuthorizationConfig { /// Define how to authorize against an upstream. #[configurable] - auth: HttpClientAuthorizationStrategy, + strategy: HttpClientAuthorizationStrategy, /// The TLS settings for the http client's connection. /// @@ -554,6 +561,32 @@ pub enum HttpClientAuthorizationStrategy { /// Authentication based on OAuth 2.0 protocol. /// /// This strategy allows to dynamically acquire and use token based on provided parameters. + /// Both standard client_credentials and mtls extension is supported, for standard client_credentials just provide both + /// client_id and client_secret parameters: + /// + /// # Example + /// + /// ``` + /// strategy: + /// strategy: "o_auth2" + /// client_id: "client.id" + /// client_secret: "secret-value" + /// token_endpoint: "https://yourendpoint.com/oauth/token" + /// ``` + /// In case you want to use mtls extension [rfc8705](https://datatracker.ietf.org/doc/html/rfc8705), provide desired key and certificate, + /// together with client_id (with no client_secret parameter). + /// + /// # Example + /// + /// ``` + /// strategy: + /// strategy: "o_auth2" + /// client_id: "client.id" + /// token_endpoint: "https://yourendpoint.com/oauth/token" + /// tls: + /// crt_path: cert.pem + /// key_file: key.pem + /// ``` OAuth2 { /// Token endpoint location, required for token acquisition. #[configurable(metadata(docs::examples = "https://auth.provider/oauth/token"))] @@ -565,7 +598,7 @@ pub enum HttpClientAuthorizationStrategy { /// The sensitive client secret. #[configurable(metadata(docs::examples = "client_secret"))] - client_secret: SensitiveString, + client_secret: Option, /// The grace period configuration for a bearer token. /// To avoid random authorization failures caused by expired token exception, diff --git a/src/sinks/axiom.rs b/src/sinks/axiom.rs index 0e149aa123693..979fe1241bd82 100644 --- a/src/sinks/axiom.rs +++ b/src/sinks/axiom.rs @@ -119,7 +119,7 @@ impl SinkConfig for AxiomConfig { }), method: HttpMethod::Post, tls: self.tls.clone(), - http_client_authorization_strategy: None, + authorization_config: None, request, acknowledgements: self.acknowledgements, batch: self.batch, diff --git a/src/sinks/http/config.rs b/src/sinks/http/config.rs index e09699c77f438..2dedea2f302c0 100644 --- a/src/sinks/http/config.rs +++ b/src/sinks/http/config.rs @@ -11,7 +11,7 @@ use vector_lib::codecs::{ use crate::{ codecs::{EncodingConfigWithFraming, SinkType}, - http::{Auth, HttpClient, MaybeAuth, HttpClientAuthorizationConfig}, + http::{Auth, HttpClient, MaybeAuth, AuthorizationConfig}, sinks::{ prelude::*, util::{ @@ -92,7 +92,7 @@ pub struct HttpSinkConfig { pub tls: Option, #[configurable(derived)] - pub http_client_authorization_strategy: Option, + pub authorization_config: Option, #[configurable(derived)] #[serde( @@ -157,7 +157,7 @@ impl From for Method { impl HttpSinkConfig { fn build_http_client(&self, cx: &SinkContext) -> crate::Result { let tls = TlsSettings::from_options(&self.tls)?; - let auth_strategy = self.http_client_authorization_strategy.clone(); + let auth_strategy = self.authorization_config.clone(); Ok(HttpClient::new_with_auth_extension(tls, cx.proxy(), auth_strategy)?) } From 9b7fd998fd09d8c0d12fb01fed2fbde4d2b60bd3 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Mon, 28 Oct 2024 20:29:17 +0100 Subject: [PATCH 09/32] fix spell issues --- src/http.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/http.rs b/src/http.rs index 6d5ad85b03d76..6098f7d288446 100644 --- a/src/http.rs +++ b/src/http.rs @@ -155,8 +155,8 @@ impl OAuth2Extension async fn request_token(&self) -> Result> { let mut request_body = format!("grant_type=client_credentials&client_id={}", self.client_id); - //in case of oauth2 with mtls (https://datatracker.ietf.org/doc/html/rfc8705) we only pass client_id, - //so secret can be considiered as optional. + //in case of oauth2 with mTLS (https://datatracker.ietf.org/doc/html/rfc8705) we only pass client_id, + //so secret can be considered as optional. if let Some(client_secret) = &self.client_secret { let secret_param = format!("&client_secret={}", client_secret.inner()); request_body.push_str(&secret_param); @@ -561,7 +561,7 @@ pub enum HttpClientAuthorizationStrategy { /// Authentication based on OAuth 2.0 protocol. /// /// This strategy allows to dynamically acquire and use token based on provided parameters. - /// Both standard client_credentials and mtls extension is supported, for standard client_credentials just provide both + /// Both standard client_credentials and mTLS extension is supported, for standard client_credentials just provide both /// client_id and client_secret parameters: /// /// # Example @@ -573,7 +573,7 @@ pub enum HttpClientAuthorizationStrategy { /// client_secret: "secret-value" /// token_endpoint: "https://yourendpoint.com/oauth/token" /// ``` - /// In case you want to use mtls extension [rfc8705](https://datatracker.ietf.org/doc/html/rfc8705), provide desired key and certificate, + /// In case you want to use mTLS extension [rfc8705](https://datatracker.ietf.org/doc/html/rfc8705), provide desired key and certificate, /// together with client_id (with no client_secret parameter). /// /// # Example From 00ebf03ece1fe470be1ed01b7d56c22f2fba58f0 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Wed, 30 Oct 2024 23:45:51 +0100 Subject: [PATCH 10/32] fix review comments --- src/http.rs | 35 +++++++++++++++++------------------ src/sinks/http/config.rs | 1 + src/sinks/http/tests.rs | 1 + 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/http.rs b/src/http.rs index 6098f7d288446..fe27e90457729 100644 --- a/src/http.rs +++ b/src/http.rs @@ -118,7 +118,7 @@ struct ExpirableToken { impl OAuth2Extension { - async fn get_token(&self) -> Result> { + async fn get_token(&self) -> Result { if let Some(token) = self.acquire_token_from_cache() { return Ok(token.access_token); } @@ -133,7 +133,7 @@ impl OAuth2Extension fn acquire_token_from_cache(&self) -> Option { let now = SystemTime::now(); - let since_the_epoch = now.duration_since(UNIX_EPOCH).unwrap(); + let since_the_epoch = now.duration_since(UNIX_EPOCH).expect("time went backwards"); let maybe_token = self.token.lock().unwrap(); match &*maybe_token { Some(token) => { @@ -149,14 +149,14 @@ impl OAuth2Extension } fn save_into_cache(&self, token: ExpirableToken) { - self.token.lock().unwrap().replace(token); + self.token.lock().expect("poisoned token lock").replace(token); } async fn request_token(&self) -> Result> { let mut request_body = format!("grant_type=client_credentials&client_id={}", self.client_id); - //in case of oauth2 with mTLS (https://datatracker.ietf.org/doc/html/rfc8705) we only pass client_id, - //so secret can be considered as optional. + // in case of oauth2 with mTLS (https://datatracker.ietf.org/doc/html/rfc8705) we only pass client_id, + // so secret can be considered as optional. if let Some(client_secret) = &self.client_secret { let secret_param = format!("&client_secret={}", client_secret.inner()); request_body.push_str(&secret_param); @@ -189,9 +189,9 @@ impl OAuth2Extension let body = hyper::body::aggregate(response).await?; let token: Token = serde_json::from_reader(body.reader())?; - //expires_in means, in seconds, for how long it will be valid, lets say 5min, - //to not cause some random 4xx, because token expired in the meantime, we will make some - //room for token refreshing, this room is a grace_period. + // expires_in means, in seconds, for how long it will be valid, lets say 5min, + // to not cause some random 4xx, because token expired in the meantime, we will make some + // room for token refreshing, this room is a grace_period. let (mut grace_period_seconds, overflow) = token.expires_in.overflowing_sub(self.grace_period); //if time for grace period exceed an expire_in, it basically means: always use new token. @@ -200,11 +200,11 @@ impl OAuth2Extension } let token_is_valid_for_ms : u128 = grace_period_seconds as u128 * 1000; - //we are multiplying by 1000 because expires_in field is in seconds, grace_period also, - //but later we operate on milliseconds. + // we are multiplying by 1000 because expires_in field is in seconds, grace_period also, + // but later we operate on milliseconds. let now = SystemTime::now(); - let since_the_epoch = now.duration_since(UNIX_EPOCH).unwrap(); - let token_will_expire_after_ms = since_the_epoch.as_millis() + (token_is_valid_for_ms as u128); + let since_the_epoch = now.duration_since(UNIX_EPOCH).expect("time went backwards"); + let token_will_expire_after_ms = since_the_epoch.as_millis() + token_is_valid_for_ms; Ok(ExpirableToken{access_token:token.access_token, expires_after_ms: token_will_expire_after_ms}) } @@ -230,7 +230,7 @@ where B::Data: Send, B::Error: Into + Send, { - async fn modify_request(&self, req: &mut Request) -> Result<(), Box> + async fn modify_request(&self, req: &mut Request) -> Result<(), vector_lib::Error> { let token = self.get_token().await?; let auth = Auth::Bearer{ token: SensitiveString::from(token)}; @@ -247,7 +247,7 @@ where B::Data: Send, B::Error: Into + Send, { - async fn modify_request(&self, req: &mut Request) -> Result<(), Box> + async fn modify_request(&self, req: &mut Request) -> Result<(), vector_lib::Error> { let user = self.user.clone(); let password = self.password.clone(); @@ -318,13 +318,12 @@ where let _enter = span.enter(); default_request_headers(&mut request, &self.user_agent); - self.maybe_add_proxy_headers(&mut request); + self.maybe_add_proxy_headers(&mut request); let client = self.client.clone(); let auth_extension = self.auth_extension.clone(); let fut = async move { - //should request for token influence upstream service latency ? if let Some(auth_extension) = auth_extension { let auth_span = tracing::info_span!("auth_extension"); auth_extension.modify_request(&mut request) @@ -566,7 +565,7 @@ pub enum HttpClientAuthorizationStrategy { /// /// # Example /// - /// ``` + /// ```yaml /// strategy: /// strategy: "o_auth2" /// client_id: "client.id" @@ -578,7 +577,7 @@ pub enum HttpClientAuthorizationStrategy { /// /// # Example /// - /// ``` + /// ```yaml /// strategy: /// strategy: "o_auth2" /// client_id: "client.id" diff --git a/src/sinks/http/config.rs b/src/sinks/http/config.rs index 2dedea2f302c0..cf842194fc06b 100644 --- a/src/sinks/http/config.rs +++ b/src/sinks/http/config.rs @@ -343,6 +343,7 @@ mod tests { batch: BatchConfig::default(), request: RequestConfig::default(), tls: None, + authorization_config: None, acknowledgements: AcknowledgementsConfig::default(), payload_prefix: String::new(), payload_suffix: String::new(), diff --git a/src/sinks/http/tests.rs b/src/sinks/http/tests.rs index 363877380c308..65beca27574ec 100644 --- a/src/sinks/http/tests.rs +++ b/src/sinks/http/tests.rs @@ -59,6 +59,7 @@ fn default_cfg(encoding: EncodingConfigWithFraming) -> HttpSinkConfig { batch: Default::default(), request: Default::default(), tls: Default::default(), + authorization_config: None, acknowledgements: Default::default(), } } From e840a2eae4987684d3f355bda44d8feb5d657740 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Wed, 30 Oct 2024 23:50:14 +0100 Subject: [PATCH 11/32] fix review comments --- src/http.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/http.rs b/src/http.rs index fe27e90457729..9e4b601bcfc75 100644 --- a/src/http.rs +++ b/src/http.rs @@ -133,7 +133,7 @@ impl OAuth2Extension fn acquire_token_from_cache(&self) -> Option { let now = SystemTime::now(); - let since_the_epoch = now.duration_since(UNIX_EPOCH).expect("time went backwards"); + let since_the_epoch = now.duration_since(UNIX_EPOCH).expect("Time went backwards"); let maybe_token = self.token.lock().unwrap(); match &*maybe_token { Some(token) => { @@ -164,7 +164,7 @@ impl OAuth2Extension let builder = Request::post(self.token_endpoint.clone()); let builder = builder.header("Content-Type", "application/x-www-form-urlencoded"); - let request = builder.body(Body::from(request_body)).expect("error creating request"); + let request = builder.body(Body::from(request_body)).expect("Error creating request"); let before = std::time::Instant::now(); let response_result = self.client.request(request).await; @@ -203,7 +203,7 @@ impl OAuth2Extension // we are multiplying by 1000 because expires_in field is in seconds, grace_period also, // but later we operate on milliseconds. let now = SystemTime::now(); - let since_the_epoch = now.duration_since(UNIX_EPOCH).expect("time went backwards"); + let since_the_epoch = now.duration_since(UNIX_EPOCH).expect("Time went backwards"); let token_will_expire_after_ms = since_the_epoch.as_millis() + token_is_valid_for_ms; Ok(ExpirableToken{access_token:token.access_token, expires_after_ms: token_will_expire_after_ms}) From b92f58733f500f4c1c49b54f29843869618af9a0 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Thu, 31 Oct 2024 00:16:01 +0100 Subject: [PATCH 12/32] fix review comments --- src/http.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/http.rs b/src/http.rs index 9e4b601bcfc75..b0b5c23d9b835 100644 --- a/src/http.rs +++ b/src/http.rs @@ -134,7 +134,7 @@ impl OAuth2Extension fn acquire_token_from_cache(&self) -> Option { let now = SystemTime::now(); let since_the_epoch = now.duration_since(UNIX_EPOCH).expect("Time went backwards"); - let maybe_token = self.token.lock().unwrap(); + let maybe_token = self.token.lock().expect("Poisoned token lock"); match &*maybe_token { Some(token) => { if since_the_epoch.as_millis() < token.expires_after_ms { @@ -149,7 +149,7 @@ impl OAuth2Extension } fn save_into_cache(&self, token: ExpirableToken) { - self.token.lock().expect("poisoned token lock").replace(token); + self.token.lock().expect("Poisoned token lock").replace(token); } async fn request_token(&self) -> Result> { From 5ef76fda4d4b6308f0db1df464b6a9ac9517ef80 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Fri, 1 Nov 2024 13:36:30 +0100 Subject: [PATCH 13/32] fix review comments --- src/http.rs | 227 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 206 insertions(+), 21 deletions(-) diff --git a/src/http.rs b/src/http.rs index b0b5c23d9b835..7609d061c4346 100644 --- a/src/http.rs +++ b/src/http.rs @@ -84,18 +84,19 @@ where B::Data: Send, B::Error: Into + Send, { - async fn modify_request(&self, req: &mut Request) -> Result<(), Box>; + async fn modify_request(&self, req: &mut Request) -> Result<(), vector_lib::Error>; } #[derive(Clone)] -struct OAuth2Extension +struct OAuth2Extension { token_endpoint: String, client_id: String, client_secret: Option, grace_period: u32, client: Client, - token: Arc>> + token: Arc>>, + get_time_now_fn: Arc Duration + Send + Sync + 'static> } #[derive(Clone)] @@ -118,6 +119,40 @@ struct ExpirableToken { impl OAuth2Extension { + /// Creates a new `OAuth2Extension`. + fn new(token_endpoint: String, client_id: String, client_secret: Option, grace_period: u32, client: Client) -> OAuth2Extension { + let get_time_now_fn = || SystemTime::now().duration_since(UNIX_EPOCH).expect("Time went backwards"); + + OAuth2Extension::new_internal( + token_endpoint, + client_id, + client_secret, + grace_period, + client, + Arc::new(get_time_now_fn) + ) + } + + /// Creates a new `OAuth2Extension` without default get_time_now_fn argument. + /// This method should be used only in tests. + fn new_internal(token_endpoint: String, client_id: String, client_secret: Option, grace_period: u32, client: Client, get_time_now_fn: Arc Duration + Send + Sync + 'static>) -> OAuth2Extension { + let initial_empty_token = Arc::new(Mutex::new(None)); + + OAuth2Extension { + token_endpoint, + client_id, + client_secret, + grace_period, + client, + token: initial_empty_token, + get_time_now_fn + } + } + + fn get_time_now(&self) -> Duration { + (self.get_time_now_fn)() + } + async fn get_token(&self) -> Result { if let Some(token) = self.acquire_token_from_cache() { return Ok(token.access_token); @@ -132,12 +167,10 @@ impl OAuth2Extension } fn acquire_token_from_cache(&self) -> Option { - let now = SystemTime::now(); - let since_the_epoch = now.duration_since(UNIX_EPOCH).expect("Time went backwards"); let maybe_token = self.token.lock().expect("Poisoned token lock"); match &*maybe_token { Some(token) => { - if since_the_epoch.as_millis() < token.expires_after_ms { + if self.get_time_now().as_millis() < token.expires_after_ms { //we have token, token is valid for at least 1min, we can use it. return Some(token.clone()); } @@ -199,12 +232,11 @@ impl OAuth2Extension grace_period_seconds = 0; } - let token_is_valid_for_ms : u128 = grace_period_seconds as u128 * 1000; - // we are multiplying by 1000 because expires_in field is in seconds, grace_period also, + // we are multiplying by 1000 because expires_in field is in seconds(oauth standard), grace_period also, // but later we operate on milliseconds. - let now = SystemTime::now(); - let since_the_epoch = now.duration_since(UNIX_EPOCH).expect("Time went backwards"); - let token_will_expire_after_ms = since_the_epoch.as_millis() + token_is_valid_for_ms; + let token_is_valid_until_ms : u128 = grace_period_seconds as u128 * 1000; + let now_millis = self.get_time_now().as_millis(); + let token_will_expire_after_ms = now_millis + token_is_valid_until_ms; Ok(ExpirableToken{access_token:token.access_token, expires_after_ms: token_will_expire_after_ms}) } @@ -400,20 +432,11 @@ where HttpClientAuthorizationStrategy::OAuth2 { token_endpoint, client_id, client_secret, grace_period } => { let tls_for_auth = authorization_config.tls.clone(); let tls_for_auth: TlsSettings = TlsSettings::from_options(&tls_for_auth).unwrap(); - let empty_token = Arc::new(Mutex::new(None)); let auth_proxy_connector = build_proxy_connector(tls_for_auth.into(), proxy_config).unwrap(); let auth_client = client_builder.build(auth_proxy_connector.clone()); - let oauth2_extension = OAuth2Extension { - token_endpoint, - client_id, - client_secret, - grace_period, - client: auth_client, - token: empty_token - }; - + let oauth2_extension = OAuth2Extension::new(token_endpoint, client_id, client_secret, grace_period, auth_client); return Some(Arc::new(oauth2_extension)); }, } @@ -913,6 +936,8 @@ mod tests { use hyper::{server::conn::AddrStream, service::make_service_fn, Server}; use proptest::prelude::*; + use rand::distributions::DistString; + use rand_distr::Alphanumeric; use tower::ServiceBuilder; use crate::test_util::next_addr; @@ -1164,4 +1189,164 @@ mod tests { let response = client.send(req).await.unwrap(); assert_eq!(response.headers().get("Connection"), None); } + + #[tokio::test] + async fn test_caching_of_tokens_in_oauth2extension_with_hyper_server() { + let addr: SocketAddr = next_addr(); + // This hyper service expose a fake oauth2 server, each request will return a response with new + // bearer token, where expires_in property is 5seconds. + let make_svc = make_service_fn(move |_: &AddrStream| { + let svc = ServiceBuilder::new() + .service(tower::service_fn(|req: Request| async move { + assert_eq!( + req.headers().get("Content-Type"), + Some(&HeaderValue::from_static("application/x-www-form-urlencoded")), + ); + + let body_bytes = hyper::body::to_bytes(req.into_body()).await.unwrap(); + let request_body = String::from_utf8(body_bytes.to_vec()).unwrap(); + + assert_eq!( + // Based on the (later) OAuth2Extension configuration. + "grant_type=client_credentials&client_id=some_client_secret&client_secret=some_secret", + request_body, + ); + + let token_valid_for_seconds: u32 = 5; + let random_token = Alphanumeric.sample_string(&mut rand::thread_rng(), 16); + let token = format!(r#" + {{ + "access_token": "{}", + "token_type": "bearer", + "expires_in": {}, + "scope": "some-scope" + }} + "#, random_token, token_valid_for_seconds); + Ok::, hyper::Error>(Response::new(Body::from(token))) + })); + futures_util::future::ok::<_, Infallible>(svc) + }); + + tokio::spawn(async move { + Server::bind(&addr).serve(make_svc).await.unwrap(); + }); + + // Wait for the server to start. + tokio::time::sleep(Duration::from_millis(10)).await; + + // Simplest possible configuration for oauth's client connector. + let tls: vector_lib::tls::MaybeTls<(), TlsSettings> = MaybeTlsSettings::from_config(&None, false).unwrap(); + let proxy_connector = build_proxy_connector(tls, &ProxyConfig::default()).unwrap(); + let auth_client = Client::builder().build(proxy_connector); + + let token_endpoint = format!("http://{}", addr); + let client_id = String::from("some_client_secret"); + let client_secret = Some(SensitiveString::from(String::from("some_secret"))); + // Fake oauth server returns token which expires in 5sec, with grace period equal 2 seconds + // we will have each token cached for next 3seconds (after this time token will be treat as expired). + let two_seconds_grace_period: u32 = 2; + + // That can looks tricky for the first time, but idea is simple, we mock get_now_fn, + // which is used internally by OAuth2Extension to decidy whether token is eligible for refreshing. + // In real life Duration since epoch in seconds, can be for example 1730460289 (November 1, 2024), + // but to simplify understanding we wil start with 11 seconds sicne epoch, and can progress. + // Each value (index) in vec, means invocation of get_now_fn by OAuth2Extension, so + // first call returns Duration::from_secs(11), second, Duration::from_secs(12) and so on, + // because of that we have full controll over time here. + let mocked_seconds_since_epoch = vec![11, 12, 20, 21, 22, 23]; + let counter = Arc::new(Mutex::new(0)); + let get_now_fn = move || { + let counter = counter.clone(); + let mut counter = counter.lock().unwrap(); + let i = *counter; + *counter += 1; + Duration::from_secs(mocked_seconds_since_epoch[i]) + }; + + // Setup an OAuth2Extension and mocked time function + let get_now_fn = Arc::new(get_now_fn); + let extension = OAuth2Extension::new_internal(token_endpoint, client_id, client_secret, two_seconds_grace_period, auth_client, get_now_fn); + + // First token is acquired because cache is empty. + let first_acquisition = extension.get_token().await.unwrap(); + // Seconds will be taken from cache because first valid until (in ms) is + // 14000ms = (11000ms + (5000ms - 2000ms)) + // where 5000ms because of token is valid 5seconds, + // and grace period is 2seconds. + let second_acquisition = extension.get_token().await.unwrap(); + assert_eq!( + first_acquisition, + second_acquisition, + ); + + // This time 20000ms since epoch is after 14000ms (until token is valid) + // so we expect new token acquired. + let third_acquisition = extension.get_token().await.unwrap(); + let fourth_acquisition = extension.get_token().await.unwrap(); + // Ensure new token requested. + assert_ne!( + first_acquisition, + third_acquisition, + ); + assert_eq!( + third_acquisition, + fourth_acquisition, + ); + + // Becuase third token is valid until 24000ms all acquisitions should return from cache. + let fifth_acquisition = extension.get_token().await.unwrap(); + assert_eq!( + fourth_acquisition, + fifth_acquisition, + ); + } + + #[tokio::test] + async fn test_oauth2extension_handle_errors_gently_with_hyper_server() { + let addr: SocketAddr = next_addr(); + // Simplest possible configuration for oauth's client connector. + let tls: vector_lib::tls::MaybeTls<(), TlsSettings> = MaybeTlsSettings::from_config(&None, false).unwrap(); + let proxy_connector = build_proxy_connector(tls, &ProxyConfig::default()).unwrap(); + let auth_client = Client::builder().build(proxy_connector); + + let token_endpoint = format!("http://{}", addr); + let client_id = String::from("some_client_secret"); + let client_secret = Some(SensitiveString::from(String::from("some_secret"))); + let two_seconds_grace_period: u32 = 2; + + // Setup an OAuth2Extension. + let extension = OAuth2Extension::new(token_endpoint, client_id, client_secret, two_seconds_grace_period, auth_client); + + // First token is acquired because cache is empty. + let failed_acquisition = extension.get_token().await; + assert!(failed_acquisition.is_err()); + let err_msg = failed_acquisition.err().unwrap().to_string(); + assert!(err_msg.contains("Connection refused")); + + let make_svc = make_service_fn(move |_: &AddrStream| { + let svc = ServiceBuilder::new() + .service(tower::service_fn(|_req: Request| async move { + let not_a_valid_token = r#" + { + "definetly" : "not a vald response" + } + "#; + + Ok::, hyper::Error>(Response::new(Body::from(not_a_valid_token))) + })); + futures_util::future::ok::<_, Infallible>(svc) + }); + + tokio::spawn(async move { + Server::bind(&addr).serve(make_svc).await.unwrap(); + }); + + // Wait for the server to start. + tokio::time::sleep(Duration::from_millis(10)).await; + + let failed_acquisition = extension.get_token().await; + assert!(failed_acquisition.is_err()); + let err_msg = failed_acquisition.err().unwrap().to_string(); + assert!(err_msg.contains("missing field")); + } } From 4d6b2c235dcf1deaadf253ebc3ae84023e57f1af Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Sat, 2 Nov 2024 02:17:10 +0100 Subject: [PATCH 14/32] fix review comments --- src/http.rs | 156 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 149 insertions(+), 7 deletions(-) diff --git a/src/http.rs b/src/http.rs index 7609d061c4346..cbbf90d4d0c89 100644 --- a/src/http.rs +++ b/src/http.rs @@ -222,23 +222,29 @@ impl OAuth2Extension let body = hyper::body::aggregate(response).await?; let token: Token = serde_json::from_reader(body.reader())?; - // expires_in means, in seconds, for how long it will be valid, lets say 5min, + let now = self.get_time_now(); + let token_will_expire_after_ms = OAuth2Extension::calculate_valid_until(now, self.grace_period, &token); + + Ok(ExpirableToken{access_token:token.access_token, expires_after_ms: token_will_expire_after_ms}) + } + + fn calculate_valid_until(now: Duration, grace_period: u32, token: &Token) -> u128 { + // 'expires_in' means, in seconds, for how long it will be valid, lets say 5min, // to not cause some random 4xx, because token expired in the meantime, we will make some // room for token refreshing, this room is a grace_period. - let (mut grace_period_seconds, overflow) = token.expires_in.overflowing_sub(self.grace_period); + let (mut grace_period_seconds, overflow) = token.expires_in.overflowing_sub(grace_period); - //if time for grace period exceed an expire_in, it basically means: always use new token. + // If time for grace period exceed an expire_in, it basically means: always use new token. if overflow { grace_period_seconds = 0; } - // we are multiplying by 1000 because expires_in field is in seconds(oauth standard), grace_period also, + // We are multiplying by 1000 because expires_in field is in seconds(oauth standard), grace_period also, // but later we operate on milliseconds. let token_is_valid_until_ms : u128 = grace_period_seconds as u128 * 1000; - let now_millis = self.get_time_now().as_millis(); - let token_will_expire_after_ms = now_millis + token_is_valid_until_ms; + let now_millis = now.as_millis(); - Ok(ExpirableToken{access_token:token.access_token, expires_after_ms: token_will_expire_after_ms}) + now_millis + token_is_valid_until_ms } } @@ -1349,4 +1355,140 @@ mod tests { let err_msg = failed_acquisition.err().unwrap().to_string(); assert!(err_msg.contains("missing field")); } + + #[tokio::test] + async fn test_oauth2_strategy_with_hyper_server() { + let oauth_addr: SocketAddr = next_addr(); + let oauth_make_svc = make_service_fn(move |_: &AddrStream| { + let svc = ServiceBuilder::new() + .service(tower::service_fn(|req: Request| async move { + assert_eq!( + req.headers().get("Content-Type"), + Some(&HeaderValue::from_static("application/x-www-form-urlencoded")), + ); + + let body_bytes = hyper::body::to_bytes(req.into_body()).await.unwrap(); + let request_body = String::from_utf8(body_bytes.to_vec()).unwrap(); + + assert_eq!( + // Based on the (later) OAuth2Extension configuration. + "grant_type=client_credentials&client_id=some_client_secret&client_secret=some_secret", + request_body, + ); + + let token = r#" + { + "access_token": "some.jwt.token", + "token_type": "bearer", + "expires_in": 60, + "scope": "some-scope" + } + "#; + + Ok::, hyper::Error>(Response::new(Body::from(token))) + })); + futures_util::future::ok::<_, Infallible>(svc) + }); + + // Server a Http client will request together with acquired bearer token. + let addr: SocketAddr = next_addr(); + let make_svc = make_service_fn(move |_conn: &AddrStream| { + let svc = ServiceBuilder::new() + .service(tower::service_fn(|req: Request| async move { + assert_eq!( + req.headers().get("authorization"), + Some(&HeaderValue::from_static("Bearer some.jwt.token")), + ); + + Ok::, hyper::Error>(Response::new(Body::empty())) + })); + futures_util::future::ok::<_, Infallible>(svc) + }); + + tokio::spawn(async move { + Server::bind(&oauth_addr).serve(oauth_make_svc).await.unwrap(); + }); + + tokio::spawn(async move { + Server::bind(&addr).serve(make_svc).await.unwrap(); + }); + + // Wait for the server to start. + tokio::time::sleep(Duration::from_millis(10)).await; + + // Http client to test + let token_endpoint = format!("http://{}", oauth_addr); + let client_id: String = String::from("some_client_secret"); + let client_secret = Some(SensitiveString::from(String::from("some_secret"))); + let grace_period = 5; + + let oauth2_strategy = HttpClientAuthorizationStrategy::OAuth2 { + token_endpoint, + client_id, + client_secret, + grace_period + }; + + let auth_config = AuthorizationConfig { + strategy: oauth2_strategy, + tls: None + }; + + let client = HttpClient::new_with_auth_extension(None, &ProxyConfig::default(), Some(auth_config)).unwrap(); + + let req = Request::get(format!("http://{}/", addr)) + .body(Body::empty()) + .unwrap(); + + let response = client.send(req).await.unwrap(); + assert!(response.status().is_success()); + } + + #[tokio::test] + async fn test_basic_auth_strategy_with_hyper_server() { + // Server a Http client will request together with acquired bearer token. + let addr: SocketAddr = next_addr(); + let make_svc = make_service_fn(move |_conn: &AddrStream| { + let svc = ServiceBuilder::new() + .service(tower::service_fn(|req: Request| async move { + assert_eq!( + req.headers().get("authorization"), + Some(&HeaderValue::from_static("Basic dXNlcjpwYXNzd29yZA==")), + ); + + Ok::, hyper::Error>(Response::new(Body::empty())) + })); + futures_util::future::ok::<_, Infallible>(svc) + }); + + tokio::spawn(async move { + Server::bind(&addr).serve(make_svc).await.unwrap(); + }); + + // Wait for the server to start. + tokio::time::sleep(Duration::from_millis(10)).await; + + // Http client to test + let user = String::from("user"); + let password = SensitiveString::from(String::from("password")); + + let basic_strategy = HttpClientAuthorizationStrategy::Basic { + user, + password + }; + + let auth_config = AuthorizationConfig { + strategy: basic_strategy, + tls: None + }; + + let client = HttpClient::new_with_auth_extension(None, &ProxyConfig::default(), Some(auth_config)).unwrap(); + + let req = Request::get(format!("http://{}/", addr)) + .body(Body::empty()) + .unwrap(); + + let response = client.send(req).await.unwrap(); + assert!(response.status().is_success()); + } } From 349dd354ef7f2d4844f428459897bee2c5ed3320 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Sat, 2 Nov 2024 02:19:52 +0100 Subject: [PATCH 15/32] vdev fmt --- src/http.rs | 323 +++++++++++++++++------------ src/internal_events/http_client.rs | 4 +- src/sinks/http/config.rs | 9 +- 3 files changed, 202 insertions(+), 134 deletions(-) diff --git a/src/http.rs b/src/http.rs index cbbf90d4d0c89..f3ed8b58f7e4f 100644 --- a/src/http.rs +++ b/src/http.rs @@ -1,5 +1,6 @@ #![allow(missing_docs)] use async_trait::async_trait; +use bytes::Buf; use futures::future::BoxFuture; use headers::{Authorization, HeaderMapExt}; use http::{ @@ -18,9 +19,13 @@ use serde::Deserialize; use serde_with::serde_as; use snafu::{ResultExt, Snafu}; use std::{ - error::Error, fmt, net::SocketAddr, sync::{Arc, Mutex}, task::{Context, Poll}, time::{Duration, SystemTime, UNIX_EPOCH} + error::Error, + fmt, + net::SocketAddr, + sync::{Arc, Mutex}, + task::{Context, Poll}, + time::{Duration, SystemTime, UNIX_EPOCH}, }; -use bytes::Buf; use tokio::time::Instant; use tower::{Layer, Service}; use tower_http::{ @@ -28,8 +33,11 @@ use tower_http::{ trace::TraceLayer, }; use tracing::{Instrument, Span}; -use vector_lib::{configurable::configurable_component, tls::{TlsConfig, TlsSettings}}; use vector_lib::sensitive_string::SensitiveString; +use vector_lib::{ + configurable::configurable_component, + tls::{TlsConfig, TlsSettings}, +}; use crate::{ config::ProxyConfig, @@ -78,8 +86,8 @@ pub type HttpClientFuture = >>::Future type HttpProxyConnector = ProxyConnector>; #[async_trait] -trait AuthExtension: Send + Sync -where +trait AuthExtension: Send + Sync +where B: fmt::Debug + HttpBody + Send + 'static, B::Data: Send, B::Error: Into + Send, @@ -88,15 +96,14 @@ where } #[derive(Clone)] -struct OAuth2Extension -{ +struct OAuth2Extension { token_endpoint: String, client_id: String, client_secret: Option, grace_period: u32, client: Client, token: Arc>>, - get_time_now_fn: Arc Duration + Send + Sync + 'static> + get_time_now_fn: Arc Duration + Send + Sync + 'static>, } #[derive(Clone)] @@ -108,20 +115,29 @@ struct BasicAuthExtension { #[derive(Debug, Deserialize)] struct Token { access_token: String, - expires_in: u32 + expires_in: u32, } #[derive(Debug, Clone)] struct ExpirableToken { access_token: String, - expires_after_ms: u128 + expires_after_ms: u128, } -impl OAuth2Extension -{ +impl OAuth2Extension { /// Creates a new `OAuth2Extension`. - fn new(token_endpoint: String, client_id: String, client_secret: Option, grace_period: u32, client: Client) -> OAuth2Extension { - let get_time_now_fn = || SystemTime::now().duration_since(UNIX_EPOCH).expect("Time went backwards"); + fn new( + token_endpoint: String, + client_id: String, + client_secret: Option, + grace_period: u32, + client: Client, + ) -> OAuth2Extension { + let get_time_now_fn = || { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + }; OAuth2Extension::new_internal( token_endpoint, @@ -129,13 +145,20 @@ impl OAuth2Extension client_secret, grace_period, client, - Arc::new(get_time_now_fn) + Arc::new(get_time_now_fn), ) } /// Creates a new `OAuth2Extension` without default get_time_now_fn argument. /// This method should be used only in tests. - fn new_internal(token_endpoint: String, client_id: String, client_secret: Option, grace_period: u32, client: Client, get_time_now_fn: Arc Duration + Send + Sync + 'static>) -> OAuth2Extension { + fn new_internal( + token_endpoint: String, + client_id: String, + client_secret: Option, + grace_period: u32, + client: Client, + get_time_now_fn: Arc Duration + Send + Sync + 'static>, + ) -> OAuth2Extension { let initial_empty_token = Arc::new(Mutex::new(None)); OAuth2Extension { @@ -145,9 +168,9 @@ impl OAuth2Extension grace_period, client, token: initial_empty_token, - get_time_now_fn + get_time_now_fn, } - } + } fn get_time_now(&self) -> Duration { (self.get_time_now_fn)() @@ -160,7 +183,7 @@ impl OAuth2Extension //no valid token in cache (or no token at all) let new_token = self.request_token().await?; - let token_to_return = new_token.access_token.clone(); + let token_to_return = new_token.access_token.clone(); self.save_into_cache(new_token); Ok(token_to_return) @@ -175,36 +198,43 @@ impl OAuth2Extension return Some(token.clone()); } - return None - }, + return None; + } _ => None, } } fn save_into_cache(&self, token: ExpirableToken) { - self.token.lock().expect("Poisoned token lock").replace(token); + self.token + .lock() + .expect("Poisoned token lock") + .replace(token); } - async fn request_token(&self) -> Result> { - let mut request_body = format!("grant_type=client_credentials&client_id={}", self.client_id); - + async fn request_token( + &self, + ) -> Result> { + let mut request_body = + format!("grant_type=client_credentials&client_id={}", self.client_id); + // in case of oauth2 with mTLS (https://datatracker.ietf.org/doc/html/rfc8705) we only pass client_id, // so secret can be considered as optional. if let Some(client_secret) = &self.client_secret { let secret_param = format!("&client_secret={}", client_secret.inner()); request_body.push_str(&secret_param); } - + let builder = Request::post(self.token_endpoint.clone()); let builder = builder.header("Content-Type", "application/x-www-form-urlencoded"); - let request = builder.body(Body::from(request_body)).expect("Error creating request"); + let request = builder + .body(Body::from(request_body)) + .expect("Error creating request"); let before = std::time::Instant::now(); let response_result = self.client.request(request).await; let roundtrip = before.elapsed(); - let response = response_result - .inspect_err(|error| { + let response = response_result.inspect_err(|error| { emit!(http_client::GotHttpWarning { error, roundtrip }); })?; @@ -216,20 +246,24 @@ impl OAuth2Extension if !response.status().is_success() { let body_bytes = hyper::body::aggregate(response).await?; let body_str = std::str::from_utf8(body_bytes.chunk())?.to_string(); - return Err(Box::new(AcquireTokenError{message: body_str})); + return Err(Box::new(AcquireTokenError { message: body_str })); } let body = hyper::body::aggregate(response).await?; let token: Token = serde_json::from_reader(body.reader())?; let now = self.get_time_now(); - let token_will_expire_after_ms = OAuth2Extension::calculate_valid_until(now, self.grace_period, &token); + let token_will_expire_after_ms = + OAuth2Extension::calculate_valid_until(now, self.grace_period, &token); - Ok(ExpirableToken{access_token:token.access_token, expires_after_ms: token_will_expire_after_ms}) + Ok(ExpirableToken { + access_token: token.access_token, + expires_after_ms: token_will_expire_after_ms, + }) } fn calculate_valid_until(now: Duration, grace_period: u32, token: &Token) -> u128 { - // 'expires_in' means, in seconds, for how long it will be valid, lets say 5min, + // 'expires_in' means, in seconds, for how long it will be valid, lets say 5min, // to not cause some random 4xx, because token expired in the meantime, we will make some // room for token refreshing, this room is a grace_period. let (mut grace_period_seconds, overflow) = token.expires_in.overflowing_sub(grace_period); @@ -239,9 +273,9 @@ impl OAuth2Extension grace_period_seconds = 0; } - // We are multiplying by 1000 because expires_in field is in seconds(oauth standard), grace_period also, - // but later we operate on milliseconds. - let token_is_valid_until_ms : u128 = grace_period_seconds as u128 * 1000; + // We are multiplying by 1000 because expires_in field is in seconds(oauth standard), grace_period also, + // but later we operate on milliseconds. + let token_is_valid_until_ms: u128 = grace_period_seconds as u128 * 1000; let now_millis = now.as_millis(); now_millis + token_is_valid_until_ms @@ -250,28 +284,33 @@ impl OAuth2Extension #[derive(Debug)] pub struct AcquireTokenError { - message: String + message: String, } impl fmt::Display for AcquireTokenError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "Server error from authentication server: {}", self.message) + write!( + f, + "Server error from authentication server: {}", + self.message + ) } } impl Error for AcquireTokenError {} #[async_trait] -impl AuthExtension for OAuth2Extension -where +impl AuthExtension for OAuth2Extension +where B: fmt::Debug + HttpBody + Send + 'static, B::Data: Send, B::Error: Into + Send, { - async fn modify_request(&self, req: &mut Request) -> Result<(), vector_lib::Error> - { + async fn modify_request(&self, req: &mut Request) -> Result<(), vector_lib::Error> { let token = self.get_token().await?; - let auth = Auth::Bearer{ token: SensitiveString::from(token)}; + let auth = Auth::Bearer { + token: SensitiveString::from(token), + }; auth.apply(req); Ok(()) @@ -279,18 +318,17 @@ where } #[async_trait] -impl AuthExtension for BasicAuthExtension -where +impl AuthExtension for BasicAuthExtension +where B: fmt::Debug + HttpBody + Send + 'static, B::Data: Send, B::Error: Into + Send, { - async fn modify_request(&self, req: &mut Request) -> Result<(), vector_lib::Error> - { + async fn modify_request(&self, req: &mut Request) -> Result<(), vector_lib::Error> { let user = self.user.clone(); let password = self.password.clone(); - let auth = Auth::Basic{ user, password }; + let auth = Auth::Basic { user, password }; auth.apply(req); Ok(()) @@ -301,7 +339,7 @@ pub struct HttpClient { client: Client, user_agent: HeaderValue, proxy_connector: HttpProxyConnector, - auth_extension: Option>> + auth_extension: Option>>, } impl HttpClient @@ -320,9 +358,14 @@ where pub fn new_with_auth_extension( tls_settings: impl Into, proxy_config: &ProxyConfig, - auth_config: Option + auth_config: Option, ) -> Result, HttpError> { - HttpClient::new_with_custom_client(tls_settings, proxy_config, &mut Client::builder(), auth_config) + HttpClient::new_with_custom_client( + tls_settings, + proxy_config, + &mut Client::builder(), + auth_config, + ) } pub fn new_with_custom_client( @@ -344,7 +387,7 @@ where client, user_agent, proxy_connector, - auth_extension + auth_extension, }) } @@ -364,12 +407,13 @@ where let fut = async move { if let Some(auth_extension) = auth_extension { let auth_span = tracing::info_span!("auth_extension"); - auth_extension.modify_request(&mut request) + auth_extension + .modify_request(&mut request) .instrument(auth_span.clone().or_current()) .await .inspect_err(|error| { // Emit the error into the internal events system. - emit!(http_client::AuthExtensionError{error}); + emit!(http_client::AuthExtensionError { error }); }) .context(AuthenticationExtensionSnafu)?; } @@ -420,11 +464,12 @@ where } } -fn build_auth_extension(authorization_config: Option, +fn build_auth_extension( + authorization_config: Option, proxy_config: &ProxyConfig, client_builder: &mut client::Builder, ) -> Option>> -where +where B: fmt::Debug + HttpBody + Send + 'static, B::Data: Send, B::Error: Into + Send, @@ -432,19 +477,31 @@ where if let Some(authorization_config) = authorization_config { match authorization_config.strategy { HttpClientAuthorizationStrategy::Basic { user, password } => { - let basic_auth_extension = BasicAuthExtension{user, password}; + let basic_auth_extension = BasicAuthExtension { user, password }; return Some(Arc::new(basic_auth_extension)); - }, - HttpClientAuthorizationStrategy::OAuth2 { token_endpoint, client_id, client_secret, grace_period } => { + } + HttpClientAuthorizationStrategy::OAuth2 { + token_endpoint, + client_id, + client_secret, + grace_period, + } => { let tls_for_auth = authorization_config.tls.clone(); let tls_for_auth: TlsSettings = TlsSettings::from_options(&tls_for_auth).unwrap(); - let auth_proxy_connector = build_proxy_connector(tls_for_auth.into(), proxy_config).unwrap(); + let auth_proxy_connector = + build_proxy_connector(tls_for_auth.into(), proxy_config).unwrap(); let auth_client = client_builder.build(auth_proxy_connector.clone()); - let oauth2_extension = OAuth2Extension::new(token_endpoint, client_id, client_secret, grace_period, auth_client); + let oauth2_extension = OAuth2Extension::new( + token_endpoint, + client_id, + client_secret, + grace_period, + auth_client, + ); return Some(Arc::new(oauth2_extension)); - }, + } } } @@ -531,7 +588,7 @@ impl Clone for HttpClient { client: self.client.clone(), user_agent: self.user_agent.clone(), proxy_connector: self.proxy_connector.clone(), - auth_extension: self.auth_extension.clone() + auth_extension: self.auth_extension.clone(), } } } @@ -545,7 +602,7 @@ impl fmt::Debug for HttpClient { } } -/// Configuration for HTTP client providing an authentication mechanism. +/// Configuration for HTTP client providing an authentication mechanism. #[configurable_component] #[configurable(metadata(docs::advanced))] #[derive(Clone, Debug)] @@ -591,9 +648,9 @@ pub enum HttpClientAuthorizationStrategy { /// This strategy allows to dynamically acquire and use token based on provided parameters. /// Both standard client_credentials and mTLS extension is supported, for standard client_credentials just provide both /// client_id and client_secret parameters: - /// + /// /// # Example - /// + /// /// ```yaml /// strategy: /// strategy: "o_auth2" @@ -603,9 +660,9 @@ pub enum HttpClientAuthorizationStrategy { /// ``` /// In case you want to use mTLS extension [rfc8705](https://datatracker.ietf.org/doc/html/rfc8705), provide desired key and certificate, /// together with client_id (with no client_secret parameter). - /// + /// /// # Example - /// + /// /// ```yaml /// strategy: /// strategy: "o_auth2" @@ -627,9 +684,9 @@ pub enum HttpClientAuthorizationStrategy { /// The sensitive client secret. #[configurable(metadata(docs::examples = "client_secret"))] client_secret: Option, - + /// The grace period configuration for a bearer token. - /// To avoid random authorization failures caused by expired token exception, + /// To avoid random authorization failures caused by expired token exception, /// we will acquire new token, some time (grace period) before current token will be expired, /// because of that, we will always execute request with fresh enough token. #[serde(default = "default_oauth2_token_grace_period")] @@ -1195,7 +1252,7 @@ mod tests { let response = client.send(req).await.unwrap(); assert_eq!(response.headers().get("Connection"), None); } - + #[tokio::test] async fn test_caching_of_tokens_in_oauth2extension_with_hyper_server() { let addr: SocketAddr = next_addr(); @@ -1203,7 +1260,7 @@ mod tests { // bearer token, where expires_in property is 5seconds. let make_svc = make_service_fn(move |_: &AddrStream| { let svc = ServiceBuilder::new() - .service(tower::service_fn(|req: Request| async move { + .service(tower::service_fn(|req: Request| async move { assert_eq!( req.headers().get("Content-Type"), Some(&HeaderValue::from_static("application/x-www-form-urlencoded")), @@ -1238,10 +1295,11 @@ mod tests { }); // Wait for the server to start. - tokio::time::sleep(Duration::from_millis(10)).await; - + tokio::time::sleep(Duration::from_millis(10)).await; + // Simplest possible configuration for oauth's client connector. - let tls: vector_lib::tls::MaybeTls<(), TlsSettings> = MaybeTlsSettings::from_config(&None, false).unwrap(); + let tls: vector_lib::tls::MaybeTls<(), TlsSettings> = + MaybeTlsSettings::from_config(&None, false).unwrap(); let proxy_connector = build_proxy_connector(tls, &ProxyConfig::default()).unwrap(); let auth_client = Client::builder().build(proxy_connector); @@ -1252,7 +1310,7 @@ mod tests { // we will have each token cached for next 3seconds (after this time token will be treat as expired). let two_seconds_grace_period: u32 = 2; - // That can looks tricky for the first time, but idea is simple, we mock get_now_fn, + // That can looks tricky for the first time, but idea is simple, we mock get_now_fn, // which is used internally by OAuth2Extension to decidy whether token is eligible for refreshing. // In real life Duration since epoch in seconds, can be for example 1730460289 (November 1, 2024), // but to simplify understanding we wil start with 11 seconds sicne epoch, and can progress. @@ -1263,7 +1321,7 @@ mod tests { let counter = Arc::new(Mutex::new(0)); let get_now_fn = move || { let counter = counter.clone(); - let mut counter = counter.lock().unwrap(); + let mut counter = counter.lock().unwrap(); let i = *counter; *counter += 1; Duration::from_secs(mocked_seconds_since_epoch[i]) @@ -1271,47 +1329,43 @@ mod tests { // Setup an OAuth2Extension and mocked time function let get_now_fn = Arc::new(get_now_fn); - let extension = OAuth2Extension::new_internal(token_endpoint, client_id, client_secret, two_seconds_grace_period, auth_client, get_now_fn); - + let extension = OAuth2Extension::new_internal( + token_endpoint, + client_id, + client_secret, + two_seconds_grace_period, + auth_client, + get_now_fn, + ); + // First token is acquired because cache is empty. let first_acquisition = extension.get_token().await.unwrap(); - // Seconds will be taken from cache because first valid until (in ms) is - // 14000ms = (11000ms + (5000ms - 2000ms)) + // Seconds will be taken from cache because first valid until (in ms) is + // 14000ms = (11000ms + (5000ms - 2000ms)) // where 5000ms because of token is valid 5seconds, // and grace period is 2seconds. let second_acquisition = extension.get_token().await.unwrap(); - assert_eq!( - first_acquisition, - second_acquisition, - ); + assert_eq!(first_acquisition, second_acquisition,); // This time 20000ms since epoch is after 14000ms (until token is valid) // so we expect new token acquired. let third_acquisition = extension.get_token().await.unwrap(); let fourth_acquisition = extension.get_token().await.unwrap(); // Ensure new token requested. - assert_ne!( - first_acquisition, - third_acquisition, - ); - assert_eq!( - third_acquisition, - fourth_acquisition, - ); + assert_ne!(first_acquisition, third_acquisition,); + assert_eq!(third_acquisition, fourth_acquisition,); // Becuase third token is valid until 24000ms all acquisitions should return from cache. let fifth_acquisition = extension.get_token().await.unwrap(); - assert_eq!( - fourth_acquisition, - fifth_acquisition, - ); + assert_eq!(fourth_acquisition, fifth_acquisition,); } - + #[tokio::test] async fn test_oauth2extension_handle_errors_gently_with_hyper_server() { - let addr: SocketAddr = next_addr(); + let addr: SocketAddr = next_addr(); // Simplest possible configuration for oauth's client connector. - let tls: vector_lib::tls::MaybeTls<(), TlsSettings> = MaybeTlsSettings::from_config(&None, false).unwrap(); + let tls: vector_lib::tls::MaybeTls<(), TlsSettings> = + MaybeTlsSettings::from_config(&None, false).unwrap(); let proxy_connector = build_proxy_connector(tls, &ProxyConfig::default()).unwrap(); let auth_client = Client::builder().build(proxy_connector); @@ -1321,8 +1375,14 @@ mod tests { let two_seconds_grace_period: u32 = 2; // Setup an OAuth2Extension. - let extension = OAuth2Extension::new(token_endpoint, client_id, client_secret, two_seconds_grace_period, auth_client); - + let extension = OAuth2Extension::new( + token_endpoint, + client_id, + client_secret, + two_seconds_grace_period, + auth_client, + ); + // First token is acquired because cache is empty. let failed_acquisition = extension.get_token().await; assert!(failed_acquisition.is_err()); @@ -1330,8 +1390,8 @@ mod tests { assert!(err_msg.contains("Connection refused")); let make_svc = make_service_fn(move |_: &AddrStream| { - let svc = ServiceBuilder::new() - .service(tower::service_fn(|_req: Request| async move { + let svc = ServiceBuilder::new().service(tower::service_fn( + |_req: Request| async move { let not_a_valid_token = r#" { "definetly" : "not a vald response" @@ -1339,16 +1399,17 @@ mod tests { "#; Ok::, hyper::Error>(Response::new(Body::from(not_a_valid_token))) - })); + }, + )); futures_util::future::ok::<_, Infallible>(svc) }); - + tokio::spawn(async move { Server::bind(&addr).serve(make_svc).await.unwrap(); }); // Wait for the server to start. - tokio::time::sleep(Duration::from_millis(10)).await; + tokio::time::sleep(Duration::from_millis(10)).await; let failed_acquisition = extension.get_token().await; assert!(failed_acquisition.is_err()); @@ -1361,7 +1422,7 @@ mod tests { let oauth_addr: SocketAddr = next_addr(); let oauth_make_svc = make_service_fn(move |_: &AddrStream| { let svc = ServiceBuilder::new() - .service(tower::service_fn(|req: Request| async move { + .service(tower::service_fn(|req: Request| async move { assert_eq!( req.headers().get("Content-Type"), Some(&HeaderValue::from_static("application/x-www-form-urlencoded")), @@ -1393,8 +1454,8 @@ mod tests { // Server a Http client will request together with acquired bearer token. let addr: SocketAddr = next_addr(); let make_svc = make_service_fn(move |_conn: &AddrStream| { - let svc = ServiceBuilder::new() - .service(tower::service_fn(|req: Request| async move { + let svc = + ServiceBuilder::new().service(tower::service_fn(|req: Request| async move { assert_eq!( req.headers().get("authorization"), Some(&HeaderValue::from_static("Bearer some.jwt.token")), @@ -1406,7 +1467,10 @@ mod tests { }); tokio::spawn(async move { - Server::bind(&oauth_addr).serve(oauth_make_svc).await.unwrap(); + Server::bind(&oauth_addr) + .serve(oauth_make_svc) + .await + .unwrap(); }); tokio::spawn(async move { @@ -1414,32 +1478,34 @@ mod tests { }); // Wait for the server to start. - tokio::time::sleep(Duration::from_millis(10)).await; - + tokio::time::sleep(Duration::from_millis(10)).await; + // Http client to test let token_endpoint = format!("http://{}", oauth_addr); let client_id: String = String::from("some_client_secret"); let client_secret = Some(SensitiveString::from(String::from("some_secret"))); let grace_period = 5; - + let oauth2_strategy = HttpClientAuthorizationStrategy::OAuth2 { token_endpoint, client_id, client_secret, - grace_period + grace_period, }; let auth_config = AuthorizationConfig { strategy: oauth2_strategy, - tls: None + tls: None, }; - let client = HttpClient::new_with_auth_extension(None, &ProxyConfig::default(), Some(auth_config)).unwrap(); + let client = + HttpClient::new_with_auth_extension(None, &ProxyConfig::default(), Some(auth_config)) + .unwrap(); let req = Request::get(format!("http://{}/", addr)) .body(Body::empty()) .unwrap(); - + let response = client.send(req).await.unwrap(); assert!(response.status().is_success()); } @@ -1449,8 +1515,8 @@ mod tests { // Server a Http client will request together with acquired bearer token. let addr: SocketAddr = next_addr(); let make_svc = make_service_fn(move |_conn: &AddrStream| { - let svc = ServiceBuilder::new() - .service(tower::service_fn(|req: Request| async move { + let svc = + ServiceBuilder::new().service(tower::service_fn(|req: Request| async move { assert_eq!( req.headers().get("authorization"), Some(&HeaderValue::from_static("Basic dXNlcjpwYXNzd29yZA==")), @@ -1466,28 +1532,27 @@ mod tests { }); // Wait for the server to start. - tokio::time::sleep(Duration::from_millis(10)).await; - + tokio::time::sleep(Duration::from_millis(10)).await; + // Http client to test let user = String::from("user"); let password = SensitiveString::from(String::from("password")); - let basic_strategy = HttpClientAuthorizationStrategy::Basic { - user, - password - }; + let basic_strategy = HttpClientAuthorizationStrategy::Basic { user, password }; let auth_config = AuthorizationConfig { strategy: basic_strategy, - tls: None + tls: None, }; - let client = HttpClient::new_with_auth_extension(None, &ProxyConfig::default(), Some(auth_config)).unwrap(); + let client = + HttpClient::new_with_auth_extension(None, &ProxyConfig::default(), Some(auth_config)) + .unwrap(); let req = Request::get(format!("http://{}/", addr)) .body(Body::empty()) .unwrap(); - + let response = client.send(req).await.unwrap(); assert!(response.status().is_success()); } diff --git a/src/internal_events/http_client.rs b/src/internal_events/http_client.rs index e231ac5971f5d..10b42850c0f62 100644 --- a/src/internal_events/http_client.rs +++ b/src/internal_events/http_client.rs @@ -96,11 +96,11 @@ impl<'a> InternalEvent for GotHttpWarning<'a> { } #[derive(Debug)] -pub struct AuthExtensionError<'a> { +pub struct AuthExtensionError<'a> { pub error: &'a Box, } -impl<'a> InternalEvent for AuthExtensionError<'a> { +impl<'a> InternalEvent for AuthExtensionError<'a> { fn emit(self) { error!( message = "HTTP Auth extension error.", diff --git a/src/sinks/http/config.rs b/src/sinks/http/config.rs index cf842194fc06b..55648f35b3bd1 100644 --- a/src/sinks/http/config.rs +++ b/src/sinks/http/config.rs @@ -8,10 +8,9 @@ use vector_lib::codecs::{ CharacterDelimitedEncoder, }; - use crate::{ codecs::{EncodingConfigWithFraming, SinkType}, - http::{Auth, HttpClient, MaybeAuth, AuthorizationConfig}, + http::{Auth, AuthorizationConfig, HttpClient, MaybeAuth}, sinks::{ prelude::*, util::{ @@ -158,7 +157,11 @@ impl HttpSinkConfig { fn build_http_client(&self, cx: &SinkContext) -> crate::Result { let tls = TlsSettings::from_options(&self.tls)?; let auth_strategy = self.authorization_config.clone(); - Ok(HttpClient::new_with_auth_extension(tls, cx.proxy(), auth_strategy)?) + Ok(HttpClient::new_with_auth_extension( + tls, + cx.proxy(), + auth_strategy, + )?) } pub(super) fn build_encoder(&self) -> crate::Result> { From 36ac20c5d3208dfae48447148cbebb9906950d94 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Sat, 2 Nov 2024 16:57:07 +0100 Subject: [PATCH 16/32] vdev fmt --- .github/semantic.yml | 1 + src/http.rs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/.github/semantic.yml b/.github/semantic.yml index 825848de985e0..af40f1ffdc218 100644 --- a/.github/semantic.yml +++ b/.github/semantic.yml @@ -19,6 +19,7 @@ types: - revert # A revert of a previous change. scopes: + - http - new source - new transform - new sink diff --git a/src/http.rs b/src/http.rs index f3ed8b58f7e4f..061d3e0852a36 100644 --- a/src/http.rs +++ b/src/http.rs @@ -1556,4 +1556,35 @@ mod tests { let response = client.send(req).await.unwrap(); assert!(response.status().is_success()); } + + #[tokio::test] + async fn test_grace_period_calculation() { + let now = Duration::from_secs(100); + let grace_period_seconds = 5; + let fake_token = Token { + access_token: String::from("some-jwt"), + expires_in: 20, + }; + + let expires_after_ms = + OAuth2Extension::calculate_valid_until(now, grace_period_seconds, &fake_token); + + assert_eq!(115000, expires_after_ms); + } + + #[tokio::test] + async fn test_grace_period_calculation_with_overflow() { + let now = Duration::from_secs(100); + let grace_period_seconds = 30; + let fake_token = Token { + access_token: String::from("some-jwt"), + expires_in: 20, + }; + + let expires_after_ms = + OAuth2Extension::calculate_valid_until(now, grace_period_seconds, &fake_token); + + // When overflow, we expect grace_period be 0 (so, now + grace = now) + assert_eq!(100000, expires_after_ms); + } } From 95b15643b812eeeb5558600c82c25dd6f6f60f19 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Sat, 2 Nov 2024 17:04:50 +0100 Subject: [PATCH 17/32] review --- .github/semantic.yml | 254 --------------------------------- .github/workflows/semantic.yml | 1 + 2 files changed, 1 insertion(+), 254 deletions(-) delete mode 100644 .github/semantic.yml diff --git a/.github/semantic.yml b/.github/semantic.yml deleted file mode 100644 index af40f1ffdc218..0000000000000 --- a/.github/semantic.yml +++ /dev/null @@ -1,254 +0,0 @@ -# WARNING! THIS FILE IS AUTOGENERATED! DO NOT EDIT! -# Original file: https://github.com/timberio/resources/tree/master/vector/github/semantic_pull_requests.tf -# ------------------------------------------------------------------------------ -# -# Configuration for probot/semantic-pull-requests GitHub check bot -# -# Installation/configuration docs here -# https://github.com/probot/semantic-pull-requests#configuration -# -# Always validate the PR title, and ignore the commits -titleOnly: true - -types: - - chore # An internal change this is not observable by users. - - enhancement # Any user observable enhancement to an existing feature. - - feat # Any user observable new feature, such as a new platform, source, transform, or sink. - - fix # Any user observable bug fix. - - docs # A documentation change. - - revert # A revert of a previous change. - -scopes: - - http - - new source - - new transform - - new sink - - # domains - - ARC # Anything related to adaptive request concurrency - - administration # Anything related to administration/operation - - api # Anything related to Vector's GraphQL API - - architecture # Anything related to architecture - - auth # Anything related to authentication/authorization - - buffers # Anything related to Vector's memory/disk buffers - - ci # Anything related to Vector's CI environment - - cli # Anything related to Vector's CLI - - codecs # Anything related to Vector's codecs (encoding/decoding) - - compression # Anything related to compressing data within Vector - - config # Anything related to configuring Vector - - core # Anything related to core crates i.e. vector-core, core-common, etc - - data model # Anything related to Vector's internal data model - - delivery # Anything related to delivering events within Vector such as end-to-end acknowledgements - - deployment # Anything related to deploying Vector - - deps # Anything related to Vector's dependencies - - dev # Anything related to Vector's development environment - - durability # Anything related to the durability of Vector events - - enriching # Anything related to enriching Vector's events with context data - - enrichment_tables # Anything related to the Vector's enrichment tables - - exceptions # Anything related to Vector's exception events. - - external docs # Anything related to Vector's external, public documentation - - filtering # Anything related to filtering within Vector - - healthchecks # Anything related to Vector's healthchecks - - internal docs # Anything related to Vector's internal documentation - - logs # Anything related to Vector's log events - - metrics # Anything related to Vector's metrics events - - networking # Anything related to Vector's networking - - observability # Anything related to monitoring/observing Vector - - parsing # Anything related to parsing within Vector - - performance # Anything related to Vector's performance - - platforms # Anything related to Vector's supported platforms - - privacy # Anything related to privacy/compliance - - processing # Anything related to processing Vector's events (parsing, merging, reducing, etc.) - - releasing # Anything related to releasing Vector - - reliability # Anything related to Vector's reliability - - reload # Anything related to reloading Vector (updating configuration) - - replay # Anything related to replaying data - - schemas # Anything related to internal Vector event schemas - - security # Anything related to security - - setup # Anything related to setting up or installing Vector - - shutdown # Anything related to the shutdown of Vector - - sinks # Anything related to the Vector's sinks - - soak tests # Anything related to Vector's soak tests - - sources # Anything related to the Vector's sources - - startup # Anything related to the startup of Vector - - templating # Anything related to templating Vector's configuration values - - tests # Anything related to Vector's internal tests - - topology # Anything related to Vector's topology code - - traces # Anything related to Vectors' trace events - - transforms # Anything related to Vector's transform components - - unit tests # Anything related to Vector's unit testing feature - - vdev # Anything related to the vdev tooling - - vrl # Anything related to the Vector Remap Language - - # platforms - - amazon-linux platform # Anything `amazon-linux` platform related - - apt platform # Anything `apt` platform related - - arm platform # Anything `arm` platform related - - arm64 platform # Anything `arm64` platform related - - centos platform # Anything `centos` platform related - - debian platform # Anything `debian` platform related - - docker platform # Anything `docker` platform related - - dpkg platform # Anything `dpkg` platform related - - helm platform # Anything `helm` platform related - - heroku platform # Anything `heroku` platform related - - homebrew platform # Anything `homebrew` platform related - - kubernetes platform # Anything `kubernetes` platform related - - macos platform # Anything `macos` platform related - - msi platform # Anything `msi` platform related - - nix platform # Anything `nix` platform related - - nixos platform # Anything `nixos` platform related - - raspbian platform # Anything `raspbian` platform related - - rhel platform # Anything `rhel` platform related - - rpm platform # Anything `rpm` platform related - - ubuntu platform # Anything `ubuntu` platform related - - windows platform # Anything `windows` platform related - - x86_64 platform # Anything `x86_64` platform related - - yum platform # Anything `yum` platform related - - # service providers - - aws service # Anything `aws` service provider related - - azure service # Anything `azure` service provider related - - confluent service # Anything `confluent` service provider related - - datadog service # Anything `datadog` service provider related - - elastic service # Anything `elastic` service provider related - - gcp service # Anything `gcp` service provider related - - grafana service # Anything `grafana` service provider related - - heroku service # Anything `heroku` service provider related - - honeycomb service # Anything `honeycomb` service provider related - - humio service # Anything `humio` service provider related - - influxdata service # Anything `influxdata` service provider related - - mezmo service # Anything `mezmo` service provider related - - new relic service # Anything `new relic` service provider related - - papertrail service # Anything `papertrail` service provider related - - sematext service # Anything `sematext` service provider related - - splunk service # Anything `splunk` service provider related - - yandex service # Anything `yandex` service provider related - - # sources - - amqp source # Anything `amqp` source related - - apache_metrics source # Anything `apache_metrics` source related - - aws_ecs_metrics source # Anything `aws_ecs_metrics` source related - - aws_kinesis_firehose source # Anything `aws_kinesis_firehose` source related - - aws_s3 source # Anything `aws_s3` source related - - aws_sqs source # Anything `aws_sqs` source related - - datadog_agent source # Anything `datadog_agent` source related - - demo_logs source # Anything `demo_logs` source related - - dnstap source # Anything `dnstap` source related - - docker_logs source # Anything `docker_logs` source related - - eventstoredb_metrics source # Anything `eventstoredb_metrics` source related - - exec source # Anything `exec` source related - - file source # Anything `file` source related - - file_descriptor source # Anything `file_descriptor` source related - - fluent source # Anything `fluent` source related - - gcp_pubsub source # Anything `gcp_pubsub` source related - - heroku_logs source # Anything `heroku_logs` source related - - host_metrics source # Anything `host_metrics` source related - - http_client source # Anything `http_client` source related - - http_server source # Anything `http_server` source related - - internal_logs source # Anything `internal_logs` source related - - internal_metrics source # Anything `internal_metrics` source related - - journald source # Anything `journald` source related - - kafka source # Anything `kafka` source related - - kubernetes_logs source # Anything `kubernetes_logs` source related - - logstash source # Anything `logstash` source related - - mongodb_metrics source # Anything `mongodb_metrics` source related - - new source # A request for a new source - - nginx_metrics source # Anything `nginx_metrics` source related - - opentelemetry source # Anything `opentelemetry` source related - - postgresql_metrics source # Anything `postgresql_metrics` source related - - prometheus_remote_write source # Anything `prometheus_remote_write` source related - - prometheus_scrape source # Anything `prometheus_scrape` source related - - pulsar source # Anything `pulsar` source related - - redis source # Anything `redis` source related - - socket source # Anything `socket` source related - - splunk_hec source # Anything `splunk_hec` source related - - statsd source # Anything `statsd` source related - - stdin source # Anything `stdin` source related - - syslog source # Anything `syslog` source related - - vector source # Anything `vector` source related - - # transforms - - aggregate transform # Anything `aggregate` transform related - - aws_ec2_metadata transform # Anything `aws_ec2_metadata` transform related - - dedupe transform # Anything `dedupe` transform related - - filter transform # Anything `filter` transform related - - log_to_metric transform # Anything `log_to_metric` transform related - - lua transform # Anything `lua` transform related - - metric_to_log transform # Anything `metric_to_log` transform related - - new transform # A request for a new transform - - pipelines transform # Anything `pipelines` transform related - - reduce transform # Anything `reduce` transform related - - remap transform # Anything `remap` transform related - - route transform # Anything `route` transform related - - sample transform # Anything `sample` transform related - - tag_cardinality_limit transform # Anything `tag_cardinality_limit` transform related - - throttle transform # Anything `throttle` transform related - - # sinks - - amqp sink # Anything `amqp` sink related - - appsignal sink # Anything `appsignal` sink related - - aws_cloudwatch_logs sink # Anything `aws_cloudwatch_logs` sink related - - aws_cloudwatch_metrics sink # Anything `aws_cloudwatch_metrics` sink related - - aws_kinesis_firehose sink # Anything `aws_kinesis_firehose` sink related - - aws_kinesis_streams sink # Anything `aws_kinesis_streams` sink related - - aws_s3 sink # Anything `aws_s3` sink related - - aws_sqs sink # Anything `aws_sqs` sink related - - axiom sink # Anything `axiom` sink related - - azure_blob sink # Anything `azure_blob` sink related - - azure_monitor_logs sink # Anything `azure_monitor_logs` sink related - - blackhole sink # Anything `blackhole` sink related - - clickhouse sink # Anything `clickhouse` sink related - - console sink # Anything `console` sink related - - databend sink # Anything `databend` sink related - - datadog_events sink # Anything `datadog_events` sink related - - datadog_logs sink # Anything `datadog_logs` sink related - - datadog_metrics sink # Anything `datadog_metrics` sink related - - elasticsearch sink # Anything `elasticsearch` sink related - - file sink # Anything `file` sink related - - gcp_chronicle sink # Anything `gcp_chronicle` sink related - - gcp_cloud_storage sink # Anything `gcp_cloud_storage` sink related - - gcp_pubsub sink # Anything `gcp_pubsub` sink related - - gcp_stackdriver_logs sink # Anything `gcp_stackdriver_logs` sink related - - gcp_stackdriver_metrics sink # Anything `gcp_stackdriver_metrics` sink related - - honeycomb sink # Anything `honeycomb` sink related - - http sink # Anything `http` sink related - - humio_logs sink # Anything `humio_logs` sink related - - humio_metrics sink # Anything `humio_metrics` sink related - - influxdb_logs sink # Anything `influxdb_logs` sink related - - influxdb_metrics sink # Anything `influxdb_metrics` sink related - - kafka sink # Anything `kafka` sink related - - loki sink # Anything `loki` sink related - - mezmo sink # Anything `mezmo` sink related - - nats sink # Anything `nats` sink related - - new sink # A request for a new sink - - new_relic sink # Anything `new_relic` sink related - - new_relic_logs sink # Anything `new_relic_logs` sink related - - opentelemetry sink # Anything `opentelemetry` sink related - - papertrail sink # Anything `papertrail` sink related - - prometheus_exporter sink # Anything `prometheus_exporter` sink related - - prometheus_remote_write sink # Anything `prometheus_remote_write` sink related - - pulsar sink # Anything `pulsar` sink related - - redis sink # Anything `redis` sink related - - sematext_logs sink # Anything `sematext_logs` sink related - - sematext_metrics sink # Anything `sematext_metrics` sink related - - socket sink # Anything `socket` sink related - - splunk_hec sink # Anything `splunk_hec` sink related - - statsd sink # Anything `statsd` sink related - - vector sink # Anything `vector` sink related - - websocket sink # Anything `websocket` sink related - - # website - - SEO website # Anything related to search engine optimization (SEO) - - analytics website # Anything related to website analytics - - blog website # Anything related to the Vector blog - - copy website # Anything related to website copy - - css website # Anything related to vector.dev's CSS and aesthetics - - guides website # Anything related to Vector's guides - - highlights website # Anything related to vector.dev highlights - - javascript website # Anything related to the JavaScript functionality on vector.dev - - navigation website # Anything related to navigating the site - - operations website # Anything related to site operations: deploying, etc - - releases website # Anything related to the press page - - search website # Anything related to the website's Algolia search indexing/config - - sitemap website # Anything related to the press page - - template website # Anything related to website HTML and other templates diff --git a/.github/workflows/semantic.yml b/.github/workflows/semantic.yml index 6c2342bca29be..bfc3cce3f4c0c 100644 --- a/.github/workflows/semantic.yml +++ b/.github/workflows/semantic.yml @@ -29,6 +29,7 @@ jobs: revert scopes: | + http new source new transform new sink From faf34336b45d924b9193d50c7bf578fa5fe6f70a Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Sat, 2 Nov 2024 17:54:40 +0100 Subject: [PATCH 18/32] vdev fmt --- src/http.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/http.rs b/src/http.rs index 061d3e0852a36..aef2dc4be8258 100644 --- a/src/http.rs +++ b/src/http.rs @@ -198,7 +198,7 @@ impl OAuth2Extension { return Some(token.clone()); } - return None; + None } _ => None, } @@ -262,7 +262,7 @@ impl OAuth2Extension { }) } - fn calculate_valid_until(now: Duration, grace_period: u32, token: &Token) -> u128 { + const fn calculate_valid_until(now: Duration, grace_period: u32, token: &Token) -> u128 { // 'expires_in' means, in seconds, for how long it will be valid, lets say 5min, // to not cause some random 4xx, because token expired in the meantime, we will make some // room for token refreshing, this room is a grace_period. @@ -1317,10 +1317,10 @@ mod tests { // Each value (index) in vec, means invocation of get_now_fn by OAuth2Extension, so // first call returns Duration::from_secs(11), second, Duration::from_secs(12) and so on, // because of that we have full controll over time here. - let mocked_seconds_since_epoch = vec![11, 12, 20, 21, 22, 23]; + let mocked_seconds_since_epoch = [11, 12, 20, 21, 22, 23]; let counter = Arc::new(Mutex::new(0)); let get_now_fn = move || { - let counter = counter.clone(); + let counter = Arc::clone(&counter); let mut counter = counter.lock().unwrap(); let i = *counter; *counter += 1; From 74c24f85f177b1127334fb1b556e5357e3391680 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Sat, 2 Nov 2024 20:33:20 +0100 Subject: [PATCH 19/32] changelog --- ...tic_bearer_token_acquisition_in_http_client.enhancement.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog.d/add_automatic_bearer_token_acquisition_in_http_client.enhancement.md diff --git a/changelog.d/add_automatic_bearer_token_acquisition_in_http_client.enhancement.md b/changelog.d/add_automatic_bearer_token_acquisition_in_http_client.enhancement.md new file mode 100644 index 0000000000000..085542594bd4c --- /dev/null +++ b/changelog.d/add_automatic_bearer_token_acquisition_in_http_client.enhancement.md @@ -0,0 +1,4 @@ +The `http_client` can now acquire a bearer token using oauth2 protocol, cache it and refresh before token expires. +OAuth2 and mTLS extension are supported in this implementation. + +authors: KowalczykBartek \ No newline at end of file From ec8d56f9c32b22eefac981ce5c1803ee31089d65 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Sat, 2 Nov 2024 20:34:59 +0100 Subject: [PATCH 20/32] changelog --- ...matic_bearer_token_acquisition_in_http_client.enhancement.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/add_automatic_bearer_token_acquisition_in_http_client.enhancement.md b/changelog.d/add_automatic_bearer_token_acquisition_in_http_client.enhancement.md index 085542594bd4c..daf3799bbb44a 100644 --- a/changelog.d/add_automatic_bearer_token_acquisition_in_http_client.enhancement.md +++ b/changelog.d/add_automatic_bearer_token_acquisition_in_http_client.enhancement.md @@ -1,4 +1,4 @@ -The `http_client` can now acquire a bearer token using oauth2 protocol, cache it and refresh before token expires. +The `http_client` can now acquire a bearer token using OAuth2 protocol, cache it and refresh before token expires. OAuth2 and mTLS extension are supported in this implementation. authors: KowalczykBartek \ No newline at end of file From e18305aa9a4a3145248b22da261dd523f41776f0 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Tue, 5 Nov 2024 15:44:06 +0100 Subject: [PATCH 21/32] vdev fmt and check rust --- .github/workflows/semantic.yml | 2 +- ...tic_bearer_token_acquisition_in_http_client.enhancement.md | 4 ++-- src/internal_events/http_client.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/semantic.yml b/.github/workflows/semantic.yml index bfc3cce3f4c0c..219b38ba7c86a 100644 --- a/.github/workflows/semantic.yml +++ b/.github/workflows/semantic.yml @@ -29,7 +29,7 @@ jobs: revert scopes: | - http + http new source new transform new sink diff --git a/changelog.d/add_automatic_bearer_token_acquisition_in_http_client.enhancement.md b/changelog.d/add_automatic_bearer_token_acquisition_in_http_client.enhancement.md index daf3799bbb44a..5795f8504ced2 100644 --- a/changelog.d/add_automatic_bearer_token_acquisition_in_http_client.enhancement.md +++ b/changelog.d/add_automatic_bearer_token_acquisition_in_http_client.enhancement.md @@ -1,4 +1,4 @@ -The `http_client` can now acquire a bearer token using OAuth2 protocol, cache it and refresh before token expires. +The `http_client` can now acquire a bearer token using OAuth2 protocol, cache it and refresh before token expires. OAuth2 and mTLS extension are supported in this implementation. -authors: KowalczykBartek \ No newline at end of file +authors: KowalczykBartek diff --git a/src/internal_events/http_client.rs b/src/internal_events/http_client.rs index 10b42850c0f62..d69752c11df06 100644 --- a/src/internal_events/http_client.rs +++ b/src/internal_events/http_client.rs @@ -97,7 +97,7 @@ impl<'a> InternalEvent for GotHttpWarning<'a> { #[derive(Debug)] pub struct AuthExtensionError<'a> { - pub error: &'a Box, + pub error: &'a vector_lib::Error, } impl<'a> InternalEvent for AuthExtensionError<'a> { From 6afc388e4c508f585925e888495011d1663f22bc Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Tue, 5 Nov 2024 22:48:15 +0100 Subject: [PATCH 22/32] provide test for mtls scenario --- Cargo.lock | 88 +++++++++++++++++++++++++---- Cargo.toml | 4 ++ src/http.rs | 155 +++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 234 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1fcc21e1019f6..417797b0123c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -624,7 +624,7 @@ dependencies = [ "once_cell", "rand 0.8.5", "regex", - "ring", + "ring 0.17.5", "rustls 0.21.11", "rustls-native-certs 0.6.3", "rustls-pemfile 1.0.3", @@ -786,7 +786,7 @@ dependencies = [ "hex", "http 0.2.9", "hyper 0.14.28", - "ring", + "ring 0.17.5", "time", "tokio", "tracing 0.1.40", @@ -8012,6 +8012,21 @@ dependencies = [ "subtle", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted 0.7.1", + "web-sys", + "winapi", +] + [[package]] name = "ring" version = "0.17.5" @@ -8022,7 +8037,7 @@ dependencies = [ "getrandom 0.2.15", "libc", "spin 0.9.8", - "untrusted", + "untrusted 0.9.0", "windows-sys 0.48.0", ] @@ -8262,6 +8277,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" +dependencies = [ + "log", + "ring 0.16.20", + "sct", + "webpki", +] + [[package]] name = "rustls" version = "0.21.11" @@ -8269,7 +8296,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" dependencies = [ "log", - "ring", + "ring 0.17.5", "rustls-webpki 0.101.7", "sct", ] @@ -8281,7 +8308,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" dependencies = [ "log", - "ring", + "ring 0.17.5", "rustls-pki-types", "rustls-webpki 0.102.2", "subtle", @@ -8313,6 +8340,15 @@ dependencies = [ "security-framework", ] +[[package]] +name = "rustls-pemfile" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee86d63972a7c661d1536fefe8c3c8407321c3df668891286de28abcd087360" +dependencies = [ + "base64 0.13.1", +] + [[package]] name = "rustls-pemfile" version = "1.0.3" @@ -8344,8 +8380,8 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring", - "untrusted", + "ring 0.17.5", + "untrusted 0.9.0", ] [[package]] @@ -8354,9 +8390,9 @@ version = "0.102.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" dependencies = [ - "ring", + "ring 0.17.5", "rustls-pki-types", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -8469,8 +8505,8 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring", - "untrusted", + "ring 0.17.5", + "untrusted 0.9.0", ] [[package]] @@ -9653,6 +9689,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +dependencies = [ + "rustls 0.20.9", + "tokio", + "webpki", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -10395,6 +10442,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" @@ -10642,6 +10695,8 @@ dependencies = [ "roaring", "rstest", "rumqttc", + "rustls 0.20.9", + "rustls-pemfile 0.3.0", "seahash", "semver 1.0.23", "serde", @@ -10665,6 +10720,7 @@ dependencies = [ "tokio", "tokio-openssl", "tokio-postgres", + "tokio-rustls 0.23.4", "tokio-stream", "tokio-test", "tokio-tungstenite 0.20.1", @@ -11381,6 +11437,16 @@ dependencies = [ "web-sys", ] +[[package]] +name = "webpki" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" +dependencies = [ + "ring 0.17.5", + "untrusted 0.9.0", +] + [[package]] name = "webpki-roots" version = "0.25.2" diff --git a/Cargo.toml b/Cargo.toml index e251541c82c9d..4619ea522b17c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -186,6 +186,10 @@ tokio-stream = { version = "0.1.16", default-features = false, features = ["net" tokio-util = { version = "0.7", default-features = false, features = ["io", "time"] } console-subscriber = { version = "0.4.1", default-features = false, optional = true } +tokio-rustls = "0.23" # where should I put this ? +rustls = "0.20" # where should I put this ? +rustls-pemfile = "0.3" # where should I put this ? + # Tracing tracing = { version = "0.1.34", default-features = false } tracing-core = { version = "0.1.26", default-features = false } diff --git a/src/http.rs b/src/http.rs index aef2dc4be8258..72f54b414bd85 100644 --- a/src/http.rs +++ b/src/http.rs @@ -995,12 +995,19 @@ where #[cfg(test)] mod tests { - use std::convert::Infallible; + use std::{convert::Infallible, fs::File, io::BufReader}; - use hyper::{server::conn::AddrStream, service::make_service_fn, Server}; + use hyper::{ + server::conn::AddrStream, + service::{make_service_fn, service_fn}, + Server, + }; use proptest::prelude::*; use rand::distributions::DistString; use rand_distr::Alphanumeric; + use rustls::{Certificate, PrivateKey, RootCertStore, ServerConfig}; + use tokio::net::TcpListener; + use tokio_rustls::TlsAcceptor; use tower::ServiceBuilder; use crate::test_util::next_addr; @@ -1510,6 +1517,150 @@ mod tests { assert!(response.status().is_success()); } + #[tokio::test] + async fn test_oauth2_with_mtls_strategy_with_hyper_server() { + let oauth_addr: SocketAddr = next_addr(); + let addr: SocketAddr = next_addr(); + let make_svc = make_service_fn(move |_conn: &AddrStream| { + let svc = + ServiceBuilder::new().service(tower::service_fn(|req: Request| async move { + assert_eq!( + req.headers().get("authorization"), + Some(&HeaderValue::from_static("Bearer some.jwt.token")), + ); + + Ok::, hyper::Error>(Response::new(Body::empty())) + })); + futures_util::future::ok::<_, Infallible>(svc) + }); + + // Load a certificates. + fn load_certs(path: &str) -> Vec { + let certfile = File::open(path).unwrap(); + let mut reader = BufReader::new(certfile); + rustls_pemfile::certs(&mut reader) + .unwrap() + .into_iter() + .map(Certificate) + .collect() + } + + // Load a private key. + fn load_private_key(path: &str) -> PrivateKey { + let keyfile = File::open(path).unwrap(); + let mut reader = BufReader::new(keyfile); + let keys = rustls_pemfile::rsa_private_keys(&mut reader).unwrap(); + PrivateKey(keys[0].clone()) + } + + // Load a server tls context to validate client. + let certs = load_certs("tests/data/ca/certs/ca.cert.pem"); + let key = load_private_key("tests/data/ca/private/ca.key.pem"); + let client_certs = load_certs("tests/data/ca/intermediate_client/certs/ca-chain.cert.pem"); + let mut root_store = RootCertStore::empty(); + for cert in client_certs { + root_store.add(&cert).unwrap(); + } + + tokio::spawn(async move { + let tls_config = ServerConfig::builder() + .with_safe_defaults() + .with_client_cert_verifier(rustls::server::AllowAnyAuthenticatedClient::new( + root_store, + )) + .with_single_cert(certs, key) + .unwrap(); + + let tls_acceptor = TlsAcceptor::from(Arc::new(tls_config)); + let acceptor = Arc::new(tls_acceptor); + let http = hyper::server::conn::Http::new(); + let listener: TcpListener = TcpListener::bind(&oauth_addr).await.unwrap(); + + loop { + let (conn, _) = listener.accept().await.unwrap(); + let acceptor = Arc::::clone(&acceptor); + let http = http.clone(); + let fut = async move { + let stream = acceptor.accept(conn).await.unwrap(); + let service = service_fn(|req: Request| async { + assert_eq!( + req.headers().get("Content-Type"), + Some(&HeaderValue::from_static( + "application/x-www-form-urlencoded" + )), + ); + + let body_bytes = hyper::body::to_bytes(req.into_body()).await.unwrap(); + let request_body = String::from_utf8(body_bytes.to_vec()).unwrap(); + + assert_eq!( + // Based on the (later) OAuth2Extension configuration. + "grant_type=client_credentials&client_id=some_client_secret", + request_body, + ); + + let token = r#" + { + "access_token": "some.jwt.token", + "token_type": "bearer", + "expires_in": 60, + "scope": "some-scope" + } + "#; + + Ok::<_, hyper::Error>(Response::new(Body::from(token))) + }); + + http.serve_connection(stream, service).await.unwrap(); + }; + tokio::spawn(fut); + } + }); + + tokio::spawn(async move { + Server::bind(&addr).serve(make_svc).await.unwrap(); + }); + + // Wait for the server to start. + tokio::time::sleep(Duration::from_millis(10)).await; + + // Http client to test + let token_endpoint = format!("https://{}", oauth_addr); + let client_id: String = String::from("some_client_secret"); + let grace_period = 5; + + let oauth2_strategy = HttpClientAuthorizationStrategy::OAuth2 { + token_endpoint, + client_id, + client_secret: None, + grace_period, + }; + + let auth_config = AuthorizationConfig { + strategy: oauth2_strategy, + tls: Some(TlsConfig { + verify_hostname: Some(false), + ca_file: Some("tests/data/ca/certs/ca.cert.pem".into()), + crt_file: Some("tests/data/ca/intermediate_client/certs/localhost.cert.pem".into()), + key_file: Some( + "tests/data/ca/intermediate_client/private/localhost.key.pem".into(), + ), + ..Default::default() + }), + }; + + let client = + HttpClient::new_with_auth_extension(None, &ProxyConfig::default(), Some(auth_config)) + .unwrap(); + + let req = Request::get(format!("http://{}/", addr)) + .body(Body::empty()) + .unwrap(); + + let response = client.send(req).await.unwrap(); + assert!(response.status().is_success()); + } + #[tokio::test] async fn test_basic_auth_strategy_with_hyper_server() { // Server a Http client will request together with acquired bearer token. From b674e00198bc1052c5bb9e8a399c252da17617f6 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Wed, 6 Nov 2024 16:48:41 +0100 Subject: [PATCH 23/32] dd-rust-license-tool write --- LICENSE-3rdparty.csv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index b84a8cd01e0ef..f22370475c54e 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -504,6 +504,7 @@ rustix,https://github.com/bytecodealliance/rustix,Apache-2.0 WITH LLVM-exception rustls,https://github.com/rustls/rustls,Apache-2.0 OR ISC OR MIT,The rustls Authors rustls-native-certs,https://github.com/ctz/rustls-native-certs,Apache-2.0 OR ISC OR MIT,The rustls-native-certs Authors rustls-native-certs,https://github.com/rustls/rustls-native-certs,Apache-2.0 OR ISC OR MIT,The rustls-native-certs Authors +rustls-pemfile,https://github.com/rustls/pemfile,Apache-2.0 OR ISC OR MIT,Joseph Birr-Pixton rustls-pemfile,https://github.com/rustls/pemfile,Apache-2.0 OR ISC OR MIT,The rustls-pemfile Authors rustls-pki-types,https://github.com/rustls/pki-types,MIT OR Apache-2.0,The rustls-pki-types Authors rustls-webpki,https://github.com/rustls/webpki,ISC,The rustls-webpki Authors @@ -603,6 +604,7 @@ tokio-openssl,https://github.com/tokio-rs/tokio-openssl,MIT OR Apache-2.0,Alex C tokio-postgres,https://github.com/sfackler/rust-postgres,MIT OR Apache-2.0,Steven Fackler tokio-retry,https://github.com/srijs/rust-tokio-retry,MIT,Sam Rijs tokio-rustls,https://github.com/rustls/tokio-rustls,MIT OR Apache-2.0,The tokio-rustls Authors +tokio-rustls,https://github.com/tokio-rs/tls,MIT OR Apache-2.0,quininer kel tokio-tungstenite,https://github.com/snapview/tokio-tungstenite,MIT,"Daniel Abramov , Alexey Galakhov " toml,https://github.com/toml-rs/toml,MIT OR Apache-2.0,Alex Crichton toml_edit,https://github.com/toml-rs/toml,MIT OR Apache-2.0,"Andronik Ordian , Ed Page " @@ -669,6 +671,7 @@ wasm-bindgen-shared,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/ wasm-streams,https://github.com/MattiasBuelens/wasm-streams,MIT OR Apache-2.0,Mattias Buelens web-sys,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/web-sys,MIT OR Apache-2.0,The wasm-bindgen Developers webbrowser,https://github.com/amodm/webbrowser-rs,MIT OR Apache-2.0,Amod Malviya @amodm +webpki,https://github.com/briansmith/webpki,ISC,Brian Smith webpki-roots,https://github.com/rustls/webpki-roots,MPL-2.0,The webpki-roots Authors whoami,https://github.com/ardaku/whoami,Apache-2.0 OR BSL-1.0 OR MIT,The whoami Authors widestring,https://github.com/starkat99/widestring-rs,MIT OR Apache-2.0,Kathryn Long From 0245d6bf7d66ec81252ddc58d7565ef12ad3ba09 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Wed, 6 Nov 2024 20:59:47 +0100 Subject: [PATCH 24/32] dd-rust-license-tool write --- Cargo.toml | 8 ++++---- LICENSE-3rdparty.csv | 3 --- src/http.rs | 9 +++++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4619ea522b17c..524eaf0d86c75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -186,10 +186,6 @@ tokio-stream = { version = "0.1.16", default-features = false, features = ["net" tokio-util = { version = "0.7", default-features = false, features = ["io", "time"] } console-subscriber = { version = "0.4.1", default-features = false, optional = true } -tokio-rustls = "0.23" # where should I put this ? -rustls = "0.20" # where should I put this ? -rustls-pemfile = "0.3" # where should I put this ? - # Tracing tracing = { version = "0.1.34", default-features = false } tracing-core = { version = "0.1.26", default-features = false } @@ -415,6 +411,10 @@ vrl.workspace = true wiremock = "0.6.2" zstd = { version = "0.13.0", default-features = false } +tokio-rustls = "0.23" +rustls = "0.20" +rustls-pemfile = "0.3" + [patch.crates-io] # The upgrade for `tokio-util` >= 0.6.9 is blocked on https://github.com/vectordotdev/vector/issues/11257. tokio-util = { git = "https://github.com/vectordotdev/tokio", branch = "tokio-util-0.7.11-framed-read-continue-on-error" } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index f22370475c54e..b84a8cd01e0ef 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -504,7 +504,6 @@ rustix,https://github.com/bytecodealliance/rustix,Apache-2.0 WITH LLVM-exception rustls,https://github.com/rustls/rustls,Apache-2.0 OR ISC OR MIT,The rustls Authors rustls-native-certs,https://github.com/ctz/rustls-native-certs,Apache-2.0 OR ISC OR MIT,The rustls-native-certs Authors rustls-native-certs,https://github.com/rustls/rustls-native-certs,Apache-2.0 OR ISC OR MIT,The rustls-native-certs Authors -rustls-pemfile,https://github.com/rustls/pemfile,Apache-2.0 OR ISC OR MIT,Joseph Birr-Pixton rustls-pemfile,https://github.com/rustls/pemfile,Apache-2.0 OR ISC OR MIT,The rustls-pemfile Authors rustls-pki-types,https://github.com/rustls/pki-types,MIT OR Apache-2.0,The rustls-pki-types Authors rustls-webpki,https://github.com/rustls/webpki,ISC,The rustls-webpki Authors @@ -604,7 +603,6 @@ tokio-openssl,https://github.com/tokio-rs/tokio-openssl,MIT OR Apache-2.0,Alex C tokio-postgres,https://github.com/sfackler/rust-postgres,MIT OR Apache-2.0,Steven Fackler tokio-retry,https://github.com/srijs/rust-tokio-retry,MIT,Sam Rijs tokio-rustls,https://github.com/rustls/tokio-rustls,MIT OR Apache-2.0,The tokio-rustls Authors -tokio-rustls,https://github.com/tokio-rs/tls,MIT OR Apache-2.0,quininer kel tokio-tungstenite,https://github.com/snapview/tokio-tungstenite,MIT,"Daniel Abramov , Alexey Galakhov " toml,https://github.com/toml-rs/toml,MIT OR Apache-2.0,Alex Crichton toml_edit,https://github.com/toml-rs/toml,MIT OR Apache-2.0,"Andronik Ordian , Ed Page " @@ -671,7 +669,6 @@ wasm-bindgen-shared,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/ wasm-streams,https://github.com/MattiasBuelens/wasm-streams,MIT OR Apache-2.0,Mattias Buelens web-sys,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/web-sys,MIT OR Apache-2.0,The wasm-bindgen Developers webbrowser,https://github.com/amodm/webbrowser-rs,MIT OR Apache-2.0,Amod Malviya @amodm -webpki,https://github.com/briansmith/webpki,ISC,Brian Smith webpki-roots,https://github.com/rustls/webpki-roots,MPL-2.0,The webpki-roots Authors whoami,https://github.com/ardaku/whoami,Apache-2.0 OR BSL-1.0 OR MIT,The whoami Authors widestring,https://github.com/starkat99/widestring-rs,MIT OR Apache-2.0,Kathryn Long diff --git a/src/http.rs b/src/http.rs index 72f54b414bd85..90b70d2c92067 100644 --- a/src/http.rs +++ b/src/http.rs @@ -115,7 +115,8 @@ struct BasicAuthExtension { #[derive(Debug, Deserialize)] struct Token { access_token: String, - expires_in: u32, + #[serde(rename = "expires_in")] + expires_after_secs: u32, } #[derive(Debug, Clone)] @@ -266,7 +267,7 @@ impl OAuth2Extension { // 'expires_in' means, in seconds, for how long it will be valid, lets say 5min, // to not cause some random 4xx, because token expired in the meantime, we will make some // room for token refreshing, this room is a grace_period. - let (mut grace_period_seconds, overflow) = token.expires_in.overflowing_sub(grace_period); + let (mut grace_period_seconds, overflow) = token.expires_after_secs.overflowing_sub(grace_period); // If time for grace period exceed an expire_in, it basically means: always use new token. if overflow { @@ -1714,7 +1715,7 @@ mod tests { let grace_period_seconds = 5; let fake_token = Token { access_token: String::from("some-jwt"), - expires_in: 20, + expires_after_secs: 20, }; let expires_after_ms = @@ -1729,7 +1730,7 @@ mod tests { let grace_period_seconds = 30; let fake_token = Token { access_token: String::from("some-jwt"), - expires_in: 20, + expires_after_secs: 20, }; let expires_after_ms = From d0b64393f1d148833de0c1af91399f2c2012d6dd Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Wed, 6 Nov 2024 21:02:35 +0100 Subject: [PATCH 25/32] review comments --- src/http.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/http.rs b/src/http.rs index 90b70d2c92067..cbf47de935e0b 100644 --- a/src/http.rs +++ b/src/http.rs @@ -115,7 +115,7 @@ struct BasicAuthExtension { #[derive(Debug, Deserialize)] struct Token { access_token: String, - #[serde(rename = "expires_in")] + #[serde(rename = "expires_in")] expires_after_secs: u32, } @@ -267,7 +267,8 @@ impl OAuth2Extension { // 'expires_in' means, in seconds, for how long it will be valid, lets say 5min, // to not cause some random 4xx, because token expired in the meantime, we will make some // room for token refreshing, this room is a grace_period. - let (mut grace_period_seconds, overflow) = token.expires_after_secs.overflowing_sub(grace_period); + let (mut grace_period_seconds, overflow) = + token.expires_after_secs.overflowing_sub(grace_period); // If time for grace period exceed an expire_in, it basically means: always use new token. if overflow { From 29a7227c7853727c91f6475c8212895c14281d4b Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Wed, 6 Nov 2024 21:15:55 +0100 Subject: [PATCH 26/32] fixes --- src/http.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/http.rs b/src/http.rs index cbf47de935e0b..90b70d2c92067 100644 --- a/src/http.rs +++ b/src/http.rs @@ -115,7 +115,7 @@ struct BasicAuthExtension { #[derive(Debug, Deserialize)] struct Token { access_token: String, - #[serde(rename = "expires_in")] + #[serde(rename = "expires_in")] expires_after_secs: u32, } @@ -267,8 +267,7 @@ impl OAuth2Extension { // 'expires_in' means, in seconds, for how long it will be valid, lets say 5min, // to not cause some random 4xx, because token expired in the meantime, we will make some // room for token refreshing, this room is a grace_period. - let (mut grace_period_seconds, overflow) = - token.expires_after_secs.overflowing_sub(grace_period); + let (mut grace_period_seconds, overflow) = token.expires_after_secs.overflowing_sub(grace_period); // If time for grace period exceed an expire_in, it basically means: always use new token. if overflow { From 7f316c3c929d5e0faba9f6bbd0055285dc636e18 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Wed, 6 Nov 2024 22:40:24 +0100 Subject: [PATCH 27/32] revert to expires_in --- src/http.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/http.rs b/src/http.rs index 90b70d2c92067..72f54b414bd85 100644 --- a/src/http.rs +++ b/src/http.rs @@ -115,8 +115,7 @@ struct BasicAuthExtension { #[derive(Debug, Deserialize)] struct Token { access_token: String, - #[serde(rename = "expires_in")] - expires_after_secs: u32, + expires_in: u32, } #[derive(Debug, Clone)] @@ -267,7 +266,7 @@ impl OAuth2Extension { // 'expires_in' means, in seconds, for how long it will be valid, lets say 5min, // to not cause some random 4xx, because token expired in the meantime, we will make some // room for token refreshing, this room is a grace_period. - let (mut grace_period_seconds, overflow) = token.expires_after_secs.overflowing_sub(grace_period); + let (mut grace_period_seconds, overflow) = token.expires_in.overflowing_sub(grace_period); // If time for grace period exceed an expire_in, it basically means: always use new token. if overflow { @@ -1715,7 +1714,7 @@ mod tests { let grace_period_seconds = 5; let fake_token = Token { access_token: String::from("some-jwt"), - expires_after_secs: 20, + expires_in: 20, }; let expires_after_ms = @@ -1730,7 +1729,7 @@ mod tests { let grace_period_seconds = 30; let fake_token = Token { access_token: String::from("some-jwt"), - expires_after_secs: 20, + expires_in: 20, }; let expires_after_ms = From 051cc54711fb92c7f6ffb3fb8bb46ad81e5c95b7 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Thu, 7 Nov 2024 08:50:52 +0100 Subject: [PATCH 28/32] review fixes --- src/http.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/http.rs b/src/http.rs index 72f54b414bd85..7d7c986e9f6d3 100644 --- a/src/http.rs +++ b/src/http.rs @@ -115,6 +115,7 @@ struct BasicAuthExtension { #[derive(Debug, Deserialize)] struct Token { access_token: String, + // This property, according to RFC, is expected to be in seconds. expires_in: u32, } From 52f1cc4934c85b8fbe14e57a722f494d87494f7f Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Thu, 7 Nov 2024 18:48:43 +0100 Subject: [PATCH 29/32] review fixes --- src/http.rs | 153 +++------------------------------------------------- 1 file changed, 8 insertions(+), 145 deletions(-) diff --git a/src/http.rs b/src/http.rs index 7d7c986e9f6d3..e051a954d0f15 100644 --- a/src/http.rs +++ b/src/http.rs @@ -103,7 +103,6 @@ struct OAuth2Extension { grace_period: u32, client: Client, token: Arc>>, - get_time_now_fn: Arc Duration + Send + Sync + 'static>, } #[derive(Clone)] @@ -133,35 +132,8 @@ impl OAuth2Extension { client_secret: Option, grace_period: u32, client: Client, - ) -> OAuth2Extension { - let get_time_now_fn = || { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - }; - - OAuth2Extension::new_internal( - token_endpoint, - client_id, - client_secret, - grace_period, - client, - Arc::new(get_time_now_fn), - ) - } - - /// Creates a new `OAuth2Extension` without default get_time_now_fn argument. - /// This method should be used only in tests. - fn new_internal( - token_endpoint: String, - client_id: String, - client_secret: Option, - grace_period: u32, - client: Client, - get_time_now_fn: Arc Duration + Send + Sync + 'static>, ) -> OAuth2Extension { let initial_empty_token = Arc::new(Mutex::new(None)); - OAuth2Extension { token_endpoint, client_id, @@ -169,14 +141,9 @@ impl OAuth2Extension { grace_period, client, token: initial_empty_token, - get_time_now_fn, } } - fn get_time_now(&self) -> Duration { - (self.get_time_now_fn)() - } - async fn get_token(&self) -> Result { if let Some(token) = self.acquire_token_from_cache() { return Ok(token.access_token); @@ -194,7 +161,10 @@ impl OAuth2Extension { let maybe_token = self.token.lock().expect("Poisoned token lock"); match &*maybe_token { Some(token) => { - if self.get_time_now().as_millis() < token.expires_after_ms { + let time_now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards"); + if time_now.as_millis() < token.expires_after_ms { //we have token, token is valid for at least 1min, we can use it. return Some(token.clone()); } @@ -253,9 +223,11 @@ impl OAuth2Extension { let body = hyper::body::aggregate(response).await?; let token: Token = serde_json::from_reader(body.reader())?; - let now = self.get_time_now(); + let time_now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards"); let token_will_expire_after_ms = - OAuth2Extension::calculate_valid_until(now, self.grace_period, &token); + OAuth2Extension::calculate_valid_until(time_now, self.grace_period, &token); Ok(ExpirableToken { access_token: token.access_token, @@ -1004,8 +976,6 @@ mod tests { Server, }; use proptest::prelude::*; - use rand::distributions::DistString; - use rand_distr::Alphanumeric; use rustls::{Certificate, PrivateKey, RootCertStore, ServerConfig}; use tokio::net::TcpListener; use tokio_rustls::TlsAcceptor; @@ -1261,113 +1231,6 @@ mod tests { assert_eq!(response.headers().get("Connection"), None); } - #[tokio::test] - async fn test_caching_of_tokens_in_oauth2extension_with_hyper_server() { - let addr: SocketAddr = next_addr(); - // This hyper service expose a fake oauth2 server, each request will return a response with new - // bearer token, where expires_in property is 5seconds. - let make_svc = make_service_fn(move |_: &AddrStream| { - let svc = ServiceBuilder::new() - .service(tower::service_fn(|req: Request| async move { - assert_eq!( - req.headers().get("Content-Type"), - Some(&HeaderValue::from_static("application/x-www-form-urlencoded")), - ); - - let body_bytes = hyper::body::to_bytes(req.into_body()).await.unwrap(); - let request_body = String::from_utf8(body_bytes.to_vec()).unwrap(); - - assert_eq!( - // Based on the (later) OAuth2Extension configuration. - "grant_type=client_credentials&client_id=some_client_secret&client_secret=some_secret", - request_body, - ); - - let token_valid_for_seconds: u32 = 5; - let random_token = Alphanumeric.sample_string(&mut rand::thread_rng(), 16); - let token = format!(r#" - {{ - "access_token": "{}", - "token_type": "bearer", - "expires_in": {}, - "scope": "some-scope" - }} - "#, random_token, token_valid_for_seconds); - Ok::, hyper::Error>(Response::new(Body::from(token))) - })); - futures_util::future::ok::<_, Infallible>(svc) - }); - - tokio::spawn(async move { - Server::bind(&addr).serve(make_svc).await.unwrap(); - }); - - // Wait for the server to start. - tokio::time::sleep(Duration::from_millis(10)).await; - - // Simplest possible configuration for oauth's client connector. - let tls: vector_lib::tls::MaybeTls<(), TlsSettings> = - MaybeTlsSettings::from_config(&None, false).unwrap(); - let proxy_connector = build_proxy_connector(tls, &ProxyConfig::default()).unwrap(); - let auth_client = Client::builder().build(proxy_connector); - - let token_endpoint = format!("http://{}", addr); - let client_id = String::from("some_client_secret"); - let client_secret = Some(SensitiveString::from(String::from("some_secret"))); - // Fake oauth server returns token which expires in 5sec, with grace period equal 2 seconds - // we will have each token cached for next 3seconds (after this time token will be treat as expired). - let two_seconds_grace_period: u32 = 2; - - // That can looks tricky for the first time, but idea is simple, we mock get_now_fn, - // which is used internally by OAuth2Extension to decidy whether token is eligible for refreshing. - // In real life Duration since epoch in seconds, can be for example 1730460289 (November 1, 2024), - // but to simplify understanding we wil start with 11 seconds sicne epoch, and can progress. - // Each value (index) in vec, means invocation of get_now_fn by OAuth2Extension, so - // first call returns Duration::from_secs(11), second, Duration::from_secs(12) and so on, - // because of that we have full controll over time here. - let mocked_seconds_since_epoch = [11, 12, 20, 21, 22, 23]; - let counter = Arc::new(Mutex::new(0)); - let get_now_fn = move || { - let counter = Arc::clone(&counter); - let mut counter = counter.lock().unwrap(); - let i = *counter; - *counter += 1; - Duration::from_secs(mocked_seconds_since_epoch[i]) - }; - - // Setup an OAuth2Extension and mocked time function - let get_now_fn = Arc::new(get_now_fn); - let extension = OAuth2Extension::new_internal( - token_endpoint, - client_id, - client_secret, - two_seconds_grace_period, - auth_client, - get_now_fn, - ); - - // First token is acquired because cache is empty. - let first_acquisition = extension.get_token().await.unwrap(); - // Seconds will be taken from cache because first valid until (in ms) is - // 14000ms = (11000ms + (5000ms - 2000ms)) - // where 5000ms because of token is valid 5seconds, - // and grace period is 2seconds. - let second_acquisition = extension.get_token().await.unwrap(); - assert_eq!(first_acquisition, second_acquisition,); - - // This time 20000ms since epoch is after 14000ms (until token is valid) - // so we expect new token acquired. - let third_acquisition = extension.get_token().await.unwrap(); - let fourth_acquisition = extension.get_token().await.unwrap(); - // Ensure new token requested. - assert_ne!(first_acquisition, third_acquisition,); - assert_eq!(third_acquisition, fourth_acquisition,); - - // Becuase third token is valid until 24000ms all acquisitions should return from cache. - let fifth_acquisition = extension.get_token().await.unwrap(); - assert_eq!(fourth_acquisition, fifth_acquisition,); - } - #[tokio::test] async fn test_oauth2extension_handle_errors_gently_with_hyper_server() { let addr: SocketAddr = next_addr(); From 4e9500c0993ccac454b2e99a433ccfd469f75001 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Thu, 7 Nov 2024 20:37:35 +0100 Subject: [PATCH 30/32] generate-component-docs --- .../reference/components/sinks/base/http.cue | 203 ++++++++++++++++++ 1 file changed, 203 insertions(+) diff --git a/website/cue/reference/components/sinks/base/http.cue b/website/cue/reference/components/sinks/base/http.cue index 0debd610db4b9..0d829cdfa09d3 100644 --- a/website/cue/reference/components/sinks/base/http.cue +++ b/website/cue/reference/components/sinks/base/http.cue @@ -74,6 +74,209 @@ base: components: sinks: http: configuration: { } } } + authorization_config: { + description: "Configuration for HTTP client providing an authentication mechanism." + required: false + type: object: options: { + strategy: { + description: """ + Configuration of the authentication strategy for HTTP requests. + + Define how to authorize against an upstream. + """ + required: true + type: object: options: { + client_id: { + description: "The client id." + relevant_when: "strategy = \"o_auth2\"" + required: true + type: string: examples: ["client_id"] + } + client_secret: { + description: "The sensitive client secret." + relevant_when: "strategy = \"o_auth2\"" + required: false + type: string: examples: ["client_secret"] + } + grace_period: { + description: """ + The grace period configuration for a bearer token. + To avoid random authorization failures caused by expired token exception, + we will acquire new token, some time (grace period) before current token will be expired, + because of that, we will always execute request with fresh enough token. + """ + relevant_when: "strategy = \"o_auth2\"" + required: false + type: uint: { + default: 300 + examples: [300] + unit: "seconds" + } + } + password: { + description: "The basic authentication password." + relevant_when: "strategy = \"basic\"" + required: true + type: string: examples: ["password"] + } + strategy: { + description: "The authentication strategy to use." + required: true + type: string: enum: { + basic: """ + Basic authentication. + + The username and password are concatenated and encoded via [base64][base64]. + + [base64]: https://en.wikipedia.org/wiki/Base64 + """ + o_auth2: """ + Authentication based on OAuth 2.0 protocol. + + This strategy allows to dynamically acquire and use token based on provided parameters. + Both standard client_credentials and mTLS extension is supported, for standard client_credentials just provide both + client_id and client_secret parameters: + + # Example + + ```yaml + strategy: + strategy: "o_auth2" + client_id: "client.id" + client_secret: "secret-value" + token_endpoint: "https://yourendpoint.com/oauth/token" + ``` + In case you want to use mTLS extension [rfc8705](https://datatracker.ietf.org/doc/html/rfc8705), provide desired key and certificate, + together with client_id (with no client_secret parameter). + + # Example + + ```yaml + strategy: + strategy: "o_auth2" + client_id: "client.id" + token_endpoint: "https://yourendpoint.com/oauth/token" + tls: + crt_path: cert.pem + key_file: key.pem + ``` + """ + } + } + token_endpoint: { + description: "Token endpoint location, required for token acquisition." + relevant_when: "strategy = \"o_auth2\"" + required: true + type: string: examples: ["https://auth.provider/oauth/token"] + } + user: { + description: "The basic authentication username." + relevant_when: "strategy = \"basic\"" + required: true + type: string: examples: ["username"] + } + } + } + tls: { + description: """ + The TLS settings for the http client's connection. + + Optional, constrains TLS settings for this http client. + """ + required: false + type: object: options: { + alpn_protocols: { + description: """ + Sets the list of supported ALPN protocols. + + Declare the supported ALPN protocols, which are used during negotiation with peer. They are prioritized in the order + that they are defined. + """ + required: false + type: array: items: type: string: examples: ["h2"] + } + ca_file: { + description: """ + Absolute path to an additional CA certificate file. + + The certificate must be in the DER or PEM (X.509) format. Additionally, the certificate can be provided as an inline string in PEM format. + """ + required: false + type: string: examples: ["/path/to/certificate_authority.crt"] + } + crt_file: { + description: """ + Absolute path to a certificate file used to identify this server. + + The certificate must be in DER, PEM (X.509), or PKCS#12 format. Additionally, the certificate can be provided as + an inline string in PEM format. + + If this is set, and is not a PKCS#12 archive, `key_file` must also be set. + """ + required: false + type: string: examples: ["/path/to/host_certificate.crt"] + } + key_file: { + description: """ + Absolute path to a private key file used to identify this server. + + The key must be in DER or PEM (PKCS#8) format. Additionally, the key can be provided as an inline string in PEM format. + """ + required: false + type: string: examples: ["/path/to/host_certificate.key"] + } + key_pass: { + description: """ + Passphrase used to unlock the encrypted key file. + + This has no effect unless `key_file` is set. + """ + required: false + type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"] + } + server_name: { + description: """ + Server name to use when using Server Name Indication (SNI). + + Only relevant for outgoing connections. + """ + required: false + type: string: examples: ["www.example.com"] + } + verify_certificate: { + description: """ + Enables certificate verification. For components that create a server, this requires that the + client connections have a valid client certificate. For components that initiate requests, + this validates that the upstream has a valid certificate. + + If enabled, certificates must not be expired and must be issued by a trusted + issuer. This verification operates in a hierarchical manner, checking that the leaf certificate (the + certificate presented by the client/server) is not only valid, but that the issuer of that certificate is also valid, and + so on until the verification process reaches a root certificate. + + Do NOT set this to `false` unless you understand the risks of not verifying the validity of certificates. + """ + required: false + type: bool: {} + } + verify_hostname: { + description: """ + Enables hostname verification. + + If enabled, the hostname used to connect to the remote host must be present in the TLS certificate presented by + the remote host, either as the Common Name or as an entry in the Subject Alternative Name extension. + + Only relevant for outgoing connections. + + Do NOT set this to `false` unless you understand the risks of not verifying the remote hostname. + """ + required: false + type: bool: {} + } + } + } + } + } batch: { description: "Event batching behavior." required: false From aac3949bbdcfb6924e108556dd9e9fc40c78fdd2 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Thu, 7 Nov 2024 21:07:42 +0100 Subject: [PATCH 31/32] remove http component --- .github/workflows/semantic.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/semantic.yml b/.github/workflows/semantic.yml index 219b38ba7c86a..6c2342bca29be 100644 --- a/.github/workflows/semantic.yml +++ b/.github/workflows/semantic.yml @@ -29,7 +29,6 @@ jobs: revert scopes: | - http new source new transform new sink From 9a5170480a29558cf9cb7ff99b8a7cd63a852136 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Thu, 7 Nov 2024 23:53:12 +0100 Subject: [PATCH 32/32] unable to expect a message under windows --- src/http.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/http.rs b/src/http.rs index e051a954d0f15..c35dd396af5cb 100644 --- a/src/http.rs +++ b/src/http.rs @@ -1257,8 +1257,6 @@ mod tests { // First token is acquired because cache is empty. let failed_acquisition = extension.get_token().await; assert!(failed_acquisition.is_err()); - let err_msg = failed_acquisition.err().unwrap().to_string(); - assert!(err_msg.contains("Connection refused")); let make_svc = make_service_fn(move |_: &AddrStream| { let svc = ServiceBuilder::new().service(tower::service_fn( @@ -1284,8 +1282,6 @@ mod tests { let failed_acquisition = extension.get_token().await; assert!(failed_acquisition.is_err()); - let err_msg = failed_acquisition.err().unwrap().to_string(); - assert!(err_msg.contains("missing field")); } #[tokio::test]