Merge pull request #2656 from redpanda-data/add-deprecated-fields

Add deprecated fields to splunk output

CHANGELOG.md

@@ -3,15 +3,25 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
-## 4.29.1 - TBD
+## 4.30.0 - 2024-06-13
 
 ### Added
 
-- New experimental `splunk` input.
+- (Benthos) Field `omit_empty` added to the `lines` scanner. (@mihaitodor)
+- (Benthos) New scheme `gcm` added to the `encrypt_aes` and `decrypy_aes` Bloblang methods. (@abergmeier)
+- (Benthos) New Bloblang method `pow`. (@mfamador)
+- (Benthos) New `sin`, `cos`, `tan` and `pi` bloblang methods. (@mfamador)
+- (Benthos) Field `proxy_url` added to the `websocket` input and output. (@mihaitodor)
+- New experimental `splunk` input. (@mihaitodor)
+
+### Fixed
+
+- The `sql_insert` and `sql_raw` components no longer fail when inserting large binary blobs into Oracle `BLOB` columns. (@mihaitodor)
+- (Benthos) The `websocket` input and output now obey the `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` environment variables. (@mihaitodor)
 
 ### Changed
 
-- The `splunk_hec` output is now implemented as a native Go component.
+- The `splunk_hec` output is now implemented as a native Go component. (@mihaitodor)
 
 ## 4.29.0 - 2024-06-04
 

cmd/redpanda-connect/main.go

@@ -5,6 +5,7 @@ import (
 	"log/slog"
 
 	"github.com/redpanda-data/benthos/v4/public/service"
+
 	"github.com/redpanda-data/connect/v4/internal/impl/kafka/enterprise"
 
 	_ "github.com/redpanda-data/connect/v4/public/components/all"

cmd/tools/docs_gen/schema_test.go

@@ -6,6 +6,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/redpanda-data/benthos/v4/public/service"
+
 	_ "github.com/redpanda-data/connect/v4/public/components/all"
 )
 

docs/modules/components/pages/inputs/splunk.adoc

@@ -1,6 +1,6 @@
 = splunk
 :type: input
-:status: experimental
+:status: beta
 :categories: ["Services"]
 
 
@@ -17,7 +17,7 @@ component_type_dropdown::[]
 
 Consumes messages from Splunk.
 
-Introduced in version 4.29.1.
+Introduced in version 4.30.0.
 
 
 [tabs]

docs/modules/components/pages/inputs/websocket.adoc

@@ -44,6 +44,7 @@ input:
   label: ""
   websocket:
     url: ws://localhost:4195/get/ws # No default (required)
+    proxy_url: "" # No default (optional)
     open_message: "" # No default (optional)
     open_message_type: binary
     auto_replay_nacks: true
@@ -95,6 +96,14 @@ The URL to connect to.
 url: ws://localhost:4195/get/ws
 ```
 
+=== `proxy_url`
+
+An optional HTTP proxy URL.
+
+
+*Type*: `string`
+
+
 === `open_message`
 
 An optional message to send to the server upon connection.

docs/modules/components/pages/outputs/splunk_hec.adoc

@@ -17,7 +17,7 @@ component_type_dropdown::[]
 
 Publishes messages to a Splunk HTTP Endpoint Collector (HEC).
 
-Introduced in version 4.29.1.
+Introduced in version 4.30.0.
 
 
 [tabs]

docs/modules/components/pages/outputs/websocket.adoc

@@ -43,6 +43,7 @@ output:
   label: ""
   websocket:
     url: "" # No default (required)
+    proxy_url: "" # No default (optional)
     tls:
       enabled: false
       skip_cert_verify: false
@@ -81,6 +82,14 @@ The URL to connect to.
 *Type*: `string`
 
 
+=== `proxy_url`
+
+An optional HTTP proxy URL.
+
+
+*Type*: `string`
+
+
 === `tls`
 
 Custom TLS settings can be used to override system defaults.

docs/modules/guides/pages/bloblang/functions.adoc

@@ -174,6 +174,27 @@ It is also possible to specify an optional custom alphabet after the length para
 root.id = nanoid(54, "abcde")
 ```
 
+=== `pi`
+
+Returns the value of the mathematical constant Pi.
+
+==== Examples
+
+
+```coffeescript
+root.radians = this.degrees * (pi() / 180)
+
+# In:  {"degrees":45}
+# Out: {"radians":0.7853981633974483}
+```
+
+```coffeescript
+root.degrees = this.radians * (180 / pi())
+
+# In:  {"radians":0.78540}
+# Out: {"degrees":45.00010522957486}
+```
+
 === `random_int`
 
 

docs/modules/guides/pages/bloblang/methods.adoc

@@ -894,6 +894,26 @@ root.new_value = this.value.ceil()
 # Out: {"new_value":-5}
 ```
 
+=== `cos`
+
+Calculates the cosine of a given angle specified in radians.
+
+==== Examples
+
+
+```coffeescript
+root.new_value = (this.value * (pi() / 180)).cos()
+
+# In:  {"value":45}
+# Out: {"new_value":0.7071067811865476}
+
+# In:  {"value":0}
+# Out: {"new_value":1}
+
+# In:  {"value":180}
+# Out: {"new_value":-1}
+```
+
 === `float32`
 
 
@@ -1194,6 +1214,46 @@ root.new_value = this.value.round()
 # Out: {"new_value":6}
 ```
 
+=== `sin`
+
+Calculates the sine of a given angle specified in radians.
+
+==== Examples
+
+
+```coffeescript
+root.new_value = (this.value * (pi() / 180)).sin()
+
+# In:  {"value":45}
+# Out: {"new_value":0.7071067811865475}
+
+# In:  {"value":0}
+# Out: {"new_value":0}
+
+# In:  {"value":90}
+# Out: {"new_value":1}
+```
+
+=== `tan`
+
+Calculates the tangent of a given angle specified in radians.
+
+==== Examples
+
+
+```coffeescript
+root.new_value = "%f".format((this.value * (pi() / 180)).tan())
+
+# In:  {"value":0}
+# Out: {"new_value":"0.000000"}
+
+# In:  {"value":45}
+# Out: {"new_value":"1.000000"}
+
+# In:  {"value":180}
+# Out: {"new_value":"-0.000000"}
+```
+
 === `uint16`
 
 

go.mod

@@ -89,7 +89,7 @@ require (
 	github.com/rabbitmq/amqp091-go v1.9.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
 	github.com/redis/go-redis/v9 v9.4.0
-	github.com/redpanda-data/benthos/v4 v4.29.0
+	github.com/redpanda-data/benthos/v4 v4.30.0
 	github.com/sijms/go-ora/v2 v2.8.19
 	github.com/smira/go-statsd v1.3.3
 	github.com/snowflakedb/gosnowflake v1.7.2

go.sum

@@ -933,8 +933,8 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk=
 github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
-github.com/redpanda-data/benthos/v4 v4.29.0 h1:/BOe02jgAAg4sQrZPbTrshyV8ldYEPs6HRiGqmKJGaI=
-github.com/redpanda-data/benthos/v4 v4.29.0/go.mod h1:veuREp5S8MJ21MXofdfMPVm5qOwQGmymh9c13jax284=
+github.com/redpanda-data/benthos/v4 v4.30.0 h1:JViX1UBMJBB3EXH6vOtKQA2Su9x60LcA+Yb6K1xXkw4=
+github.com/redpanda-data/benthos/v4 v4.30.0/go.mod h1:veuREp5S8MJ21MXofdfMPVm5qOwQGmymh9c13jax284=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rickb777/date v1.20.5 h1:Ybjz7J7ga9ui4VJizQpil0l330r6wkn6CicaoattIxQ=

internal/impl/kafka/enterprise/topic_logger.go

@@ -23,6 +23,7 @@ import (
 	"github.com/twmb/franz-go/pkg/sasl"
 
 	"github.com/redpanda-data/benthos/v4/public/service"
+
 	"github.com/redpanda-data/connect/v4/internal/impl/kafka"
 )
 

internal/impl/splunk/input.go

@@ -35,8 +35,8 @@ const (
 
 func inputSpec() *service.ConfigSpec {
 	return service.NewConfigSpec().
-		// Stable().
-		Version("4.29.1").
+		Beta().
+		Version("4.30.0").
 		Categories("Services").
 		Summary(`Consumes messages from Splunk.`).
 		Fields(

internal/impl/splunk/integration_test.go

@@ -16,10 +16,11 @@ import (
 	"time"
 
 	"github.com/ory/dockertest/v3"
-	_ "github.com/redpanda-data/benthos/v4/public/components/pure"
 	"github.com/redpanda-data/benthos/v4/public/service/integration"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	_ "github.com/redpanda-data/benthos/v4/public/components/pure"
 )
 
 func TestIntegrationSplunk(t *testing.T) {

internal/impl/splunk/output.go

@@ -31,14 +31,20 @@ const (
 	soFieldEventIndex      = "event_index"
 	soFieldSkipCertVerify  = "skip_cert_verify"
 	soFieldBatching        = "batching"
+
+	// Deprecated fields
+	soFieldBatchCount    = "batching_count"
+	soFieldBatchPeriod   = "batching_period"
+	soFieldBatchByteSize = "batching_byte_size"
+	soFieldRateLimit     = "rate_limit"
 )
 
 //------------------------------------------------------------------------------
 
 func outputSpec() *service.ConfigSpec {
 	return service.NewConfigSpec().
 		Beta().
-		Version("4.29.1").
+		Version("4.30.0").
 		Categories("Services").
 		Summary(`Publishes messages to a Splunk HTTP Endpoint Collector (HEC).`).
 		Description(service.OutputPerformanceDocs(true, true)).
@@ -53,6 +59,20 @@ func outputSpec() *service.ConfigSpec {
 			service.NewBoolField(soFieldSkipCertVerify).Description("Whether to skip server side certificate verification.").Advanced().Default(false),
 			service.NewOutputMaxInFlightField(),
 			service.NewBatchPolicyField(soFieldBatching),
+
+			// Old deprecated fields
+			service.NewIntField(soFieldBatchCount).
+				Optional().
+				Deprecated(),
+			service.NewStringField(soFieldBatchPeriod).
+				Optional().
+				Deprecated(),
+			service.NewIntField(soFieldBatchByteSize).
+				Optional().
+				Deprecated(),
+			service.NewStringField(soFieldRateLimit).
+				Optional().
+				Deprecated(),
 		)
 }
 
@@ -65,6 +85,18 @@ func init() {
 			if batchPolicy, err = conf.FieldBatchPolicy(soFieldBatching); err != nil {
 				return
 			}
+
+			// Check for presence of deprecated fields
+			if conf.Contains(soFieldBatchCount) {
+				batchPolicy.Count, _ = conf.FieldInt(soFieldBatchCount)
+			}
+			if conf.Contains(soFieldBatchPeriod) {
+				batchPolicy.Period, _ = conf.FieldString(soFieldBatchPeriod)
+			}
+			if conf.Contains(soFieldBatchByteSize) {
+				batchPolicy.ByteSize, _ = conf.FieldInt(soFieldBatchByteSize)
+			}
+
 			out, err = outputFromParsed(conf, mgr.Logger())
 			return
 		})