From d142eae728c999fdba45a90051a0691b25f02e45 Mon Sep 17 00:00:00 2001 From: Sainath Singineedi Date: Sat, 11 Jan 2025 16:31:42 +0530 Subject: [PATCH] feat(new sink): add changelog and cue references --- changelog.d/21965-add-keep-sink.feature.md | 3 + .../reference/configuration/sinks/keep.md | 14 + .../reference/components/sinks/base/keep.cue | 289 ++++++++++++++++++ .../components/sinks/base/socket.cue | 13 +- 4 files changed, 310 insertions(+), 9 deletions(-) create mode 100644 changelog.d/21965-add-keep-sink.feature.md create mode 100644 website/content/en/docs/reference/configuration/sinks/keep.md create mode 100644 website/cue/reference/components/sinks/base/keep.cue diff --git a/changelog.d/21965-add-keep-sink.feature.md b/changelog.d/21965-add-keep-sink.feature.md new file mode 100644 index 0000000000000..00328d73e84ce --- /dev/null +++ b/changelog.d/21965-add-keep-sink.feature.md @@ -0,0 +1,3 @@ +A new sink for keep was added + +authors: sainad2222 diff --git a/website/content/en/docs/reference/configuration/sinks/keep.md b/website/content/en/docs/reference/configuration/sinks/keep.md new file mode 100644 index 0000000000000..19f84f63de012 --- /dev/null +++ b/website/content/en/docs/reference/configuration/sinks/keep.md @@ -0,0 +1,14 @@ +--- +title: Keep +description: Deliver log events to [Keep](https://www.keephq.dev) +kind: sink +layout: component +tags: ["keep", "component", "sink", "logs", "metrics"] +--- + +{{/* +This doc is generated using: + +1. The template in layouts/docs/component.html +2. The relevant CUE data in cue/reference/components/... +*/}} diff --git a/website/cue/reference/components/sinks/base/keep.cue b/website/cue/reference/components/sinks/base/keep.cue new file mode 100644 index 0000000000000..a50e8b5f0404e --- /dev/null +++ b/website/cue/reference/components/sinks/base/keep.cue @@ -0,0 +1,289 @@ +package metadata + +base: components: sinks: keep: configuration: { + acknowledgements: { + description: """ + Controls how acknowledgements are handled for this sink. + + See [End-to-end Acknowledgements][e2e_acks] for more information on how event acknowledgement is handled. + + [e2e_acks]: https://vector.dev/docs/about/under-the-hood/architecture/end-to-end-acknowledgements/ + """ + required: false + type: object: options: enabled: { + description: """ + Whether or not end-to-end acknowledgements are enabled. + + When enabled for a sink, any source connected to that sink, where the source supports + end-to-end acknowledgements as well, waits for events to be acknowledged by **all + connected** sinks before acknowledging them at the source. + + Enabling or disabling acknowledgements at the sink level takes precedence over any global + [`acknowledgements`][global_acks] configuration. + + [global_acks]: https://vector.dev/docs/reference/configuration/global-options/#acknowledgements + """ + required: false + type: bool: {} + } + } + api_key: { + description: "The API key that is used to authenticate against Keep." + required: true + type: string: examples: ["${KEEP_API_KEY}", "keepappkey"] + } + batch: { + description: "Event batching behavior." + required: false + type: object: options: { + max_bytes: { + description: """ + The maximum size of a batch that is processed by a sink. + + This is based on the uncompressed size of the batched events, before they are + serialized/compressed. + """ + required: false + type: uint: { + default: 100000 + unit: "bytes" + } + } + max_events: { + description: "The maximum size of a batch before it is flushed." + required: false + type: uint: unit: "events" + } + timeout_secs: { + description: "The maximum age of a batch before it is flushed." + required: false + type: float: { + default: 1.0 + unit: "seconds" + } + } + } + } + encoding: { + description: "Transformations to prepare an event for serialization." + required: false + type: object: options: { + except_fields: { + description: "List of fields that are excluded from the encoded event." + required: false + type: array: items: type: string: {} + } + only_fields: { + description: "List of fields that are included in the encoded event." + required: false + type: array: items: type: string: {} + } + timestamp_format: { + description: "Format used for timestamp fields." + required: false + type: string: enum: { + rfc3339: "Represent the timestamp as a RFC 3339 timestamp." + unix: "Represent the timestamp as a Unix timestamp." + unix_float: "Represent the timestamp as a Unix timestamp in floating point." + unix_ms: "Represent the timestamp as a Unix timestamp in milliseconds." + unix_ns: "Represent the timestamp as a Unix timestamp in nanoseconds." + unix_us: "Represent the timestamp as a Unix timestamp in microseconds" + } + } + } + } + endpoint: { + description: "Keep's endpoint to send logs to" + required: false + type: string: { + default: "http://localhost:8080/alerts/event/vectordev?provider_id=test" + examples: ["https://backend.keep.com:8081/alerts/event/vectordev?provider_id=test"] + } + } + request: { + description: """ + Middleware settings for outbound requests. + + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. + """ + required: false + type: object: options: { + adaptive_concurrency: { + description: """ + Configuration of adaptive concurrency parameters. + + These parameters typically do not require changes from the default, and incorrect values can lead to meta-stable or + unstable performance and sink behavior. Proceed with caution. + """ + required: false + type: object: options: { + decrease_ratio: { + description: """ + The fraction of the current value to set the new concurrency limit when decreasing the limit. + + Valid values are greater than `0` and less than `1`. Smaller values cause the algorithm to scale back rapidly + when latency increases. + + Note that the new limit is rounded down after applying this ratio. + """ + required: false + type: float: default: 0.9 + } + ewma_alpha: { + description: """ + The weighting of new measurements compared to older measurements. + + Valid values are greater than `0` and less than `1`. + + ARC uses an exponentially weighted moving average (EWMA) of past RTT measurements as a reference to compare with + the current RTT. Smaller values cause this reference to adjust more slowly, which may be useful if a service has + unusually high response variability. + """ + required: false + type: float: default: 0.4 + } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit will be 1 (no concurrency). + + It is recommended to set this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } + rtt_deviation_scale: { + description: """ + Scale of RTT deviations which are not considered anomalous. + + Valid values are greater than or equal to `0`, and we expect reasonable values to range from `1.0` to `3.0`. + + When calculating the past RTT average, we also compute a secondary “deviation” value that indicates how variable + those values are. We use that deviation when comparing the past RTT average to the current measurements, so we + can ignore increases in RTT that are within an expected range. This factor is used to scale up the deviation to + an appropriate range. Larger values cause the algorithm to ignore larger increases in the RTT. + """ + required: false + type: float: default: 2.5 + } + } + } + concurrency: { + description: """ + Configuration for outbound request concurrency. + + This can be set either to one of the below enum values or to a positive integer, which denotes + a fixed concurrency limit. + """ + required: false + type: { + string: { + default: "adaptive" + enum: { + adaptive: """ + Concurrency will be managed by Vector's [Adaptive Request Concurrency][arc] feature. + + [arc]: https://vector.dev/docs/about/under-the-hood/networking/arc/ + """ + none: """ + A fixed concurrency of 1. + + Only one request can be outstanding at any given time. + """ + } + } + uint: {} + } + } + rate_limit_duration_secs: { + description: "The time window used for the `rate_limit_num` option." + required: false + type: uint: { + default: 1 + unit: "seconds" + } + } + rate_limit_num: { + description: "The maximum number of requests allowed within the `rate_limit_duration_secs` time window." + required: false + type: uint: { + default: 9223372036854775807 + unit: "requests" + } + } + retry_attempts: { + description: "The maximum number of retries to make for failed requests." + required: false + type: uint: { + default: 9223372036854775807 + unit: "retries" + } + } + retry_initial_backoff_secs: { + description: """ + The amount of time to wait before attempting the first retry for a failed request. + + After the first retry has failed, the fibonacci sequence is used to select future backoffs. + """ + required: false + type: uint: { + default: 1 + unit: "seconds" + } + } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } + retry_max_duration_secs: { + description: "The maximum amount of time to wait between retries." + required: false + type: uint: { + default: 30 + unit: "seconds" + } + } + timeout_secs: { + description: """ + The time a request can take before being aborted. + + Datadog highly recommends that you do not lower this value below the service's internal timeout, as this could + create orphaned requests, pile on retries, and result in duplicate data downstream. + """ + required: false + type: uint: { + default: 60 + unit: "seconds" + } + } + } + } +} diff --git a/website/cue/reference/components/sinks/base/socket.cue b/website/cue/reference/components/sinks/base/socket.cue index 001c8a3145897..a95f75b529200 100644 --- a/website/cue/reference/components/sinks/base/socket.cue +++ b/website/cue/reference/components/sinks/base/socket.cue @@ -383,7 +383,7 @@ base: components: sinks: socket: configuration: { } framing: { description: "Framing configuration." - relevant_when: "mode = \"tcp\" or mode = \"unix_stream\" or mode = \"unix_datagram\"" + relevant_when: "mode = \"tcp\" or mode = \"unix_stream\"" required: false type: object: options: { character_delimited: { @@ -453,13 +453,8 @@ base: components: sinks: socket: configuration: { description: "The type of socket to use." required: true type: string: enum: { - tcp: "Send over TCP." - udp: "Send over UDP." - unix_datagram: """ - Send over a Unix domain socket (UDS), in datagram mode. - Unavailable on macOS, due to send(2)'s apparent non-blocking behavior, - resulting in ENOBUFS errors which we currently don't handle. - """ + tcp: "Send over TCP." + udp: "Send over UDP." unix_stream: "Send over a Unix domain socket (UDS), in stream mode." } } @@ -469,7 +464,7 @@ base: components: sinks: socket: configuration: { This should be an absolute path. """ - relevant_when: "mode = \"unix_stream\" or mode = \"unix_datagram\"" + relevant_when: "mode = \"unix_stream\"" required: true type: string: examples: ["/path/to/socket"] }