contrib: add validation tests using test-agent (DataDog#2047)

This PR adds the APM Test Agent to run alongside unit / contrib tests in CI testing. A testing interface is created in contrib/internal/validationtest, which can be implemented by each contrib integration in order to generate test traces which are emitted to the APM Test Agent using the real Golang tracer. These traces are tested against a number of centrally located checks written within the Test-Agent, including:

Service naming checks
Peer.service checks
Trace header checks

The APM Test Agent is being added to every DD APM Tracer with the primary purpose of providing a centralized location for writing tests which will subsequently be used for asserting on handled traces. This strategy will help bring consistency to Datadog APM Tagging/ integration support.

Author: rarguelloF

.github/workflows/unit-integration-tests.yml

@@ -51,6 +51,8 @@ jobs:
           DD_APM_ENABLED: true
           DD_BIND_HOST: "0.0.0.0"
           DD_API_KEY: "invalid_key_but_this_is_fine"
+          DD_TEST_AGENT_HOST: "localhost"
+          DD_TEST_AGENT_PORT: 9126
         # We need to specify a custom health-check. By default, this container will remain "unhealthy" since
         # we don't fully configure it with a valid API key (and possibly other reasons)
         # This command just checks for our ability to connect to port 8126
@@ -59,6 +61,18 @@ jobs:
         ports:
           - 8125:8125/udp
           - 8126:8126
+      testagent:
+        image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.11.0
+        ports:
+          - 9126:9126
+        env:
+          LOG_LEVEL: DEBUG
+          TRACE_LANGUAGE: golang
+          ENABLED_CHECKS: trace_stall,trace_count_header,trace_peer_service,trace_dd_service
+          PORT: 9126
+          DD_SUPPRESS_TRACE_PARSE_ERRORS: true
+          DD_POOL_TRACE_CHECK_FAILURES: true
+          DD_DISABLE_ERROR_RESPONSES: true
       cassandra:
         image: cassandra:3.7
         env:
@@ -188,6 +202,34 @@ jobs:
         shell: bash
         run: bash <(curl -s https://codecov.io/bash)
 
+      - name: Get Datadog APM Test Agent Logs
+        if: always()
+        shell: bash
+        run: docker logs ${{ job.services.testagent.id }}
+
+      - name: Get Datadog APM Test Agent Trace Check Summary Results
+        if: always()
+        shell: bash
+        run: |
+              RESPONSE=$(curl -s -w "\n%{http_code}" -o response.txt "http://127.0.0.1:9126/test/trace_check/failures?return_all=true")
+              RESPONSE_CODE=$(echo "$RESPONSE" | awk 'END {print $NF}')
+              SUMMARY_RESPONSE=$(curl -s -w "\n%{http_code}" -o summary_response.txt "http://127.0.0.1:9126/test/trace_check/summary?return_all=true")
+              SUMMARY_RESPONSE_CODE=$(echo "$SUMMARY_RESPONSE" | awk 'END {print $NF}')
+              if [[ $RESPONSE_CODE -eq 200 ]]; then
+                  echo " "
+                  cat response.txt
+                  echo " - All APM Test Agent Check Traces returned successful!"
+                  echo "APM Test Agent Check Traces Summary Results:"
+                  cat summary_response.txt | jq "."
+              else
+                  echo "APM Test Agent Check Traces failed with response code: $RESPONSE_CODE"
+                  echo "Failures:"
+                  cat response.txt
+                  echo "APM Test Agent Check Traces Summary Results:"
+                  cat summary_response.txt | jq "."
+                  exit 1
+              fi
+
       - name: Testing outlier google.golang.org/api
         run: |
               go get google.golang.org/api@v0.121.0 # version used to generate code

contrib/internal/validationtest/contrib/bradfitz_memcache.go

@@ -0,0 +1,53 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2023 Datadog, Inc.
+
+package validationtest
+
+import (
+	"testing"
+
+	memcachetrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/bradfitz/gomemcache/memcache"
+
+	"github.com/bradfitz/gomemcache/memcache"
+	"github.com/stretchr/testify/require"
+)
+
+type Memcache struct {
+	client   *memcachetrace.Client
+	opts     []memcachetrace.ClientOption
+	numSpans int
+}
+
+func NewMemcache() *Memcache {
+	return &Memcache{
+		opts: make([]memcachetrace.ClientOption, 0),
+	}
+}
+
+func (i *Memcache) Name() string {
+	return "bradfitz/gomemcache/memcache"
+}
+
+func (i *Memcache) Init(_ *testing.T) {
+	i.client = memcachetrace.WrapClient(memcache.New("127.0.0.1:11211"), i.opts...)
+}
+
+func (i *Memcache) GenSpans(t *testing.T) {
+	t.Helper()
+	err := i.client.Set(&memcache.Item{Key: "myKey", Value: []byte("myValue")})
+	require.NoError(t, err)
+	i.numSpans++
+	t.Cleanup(func() {
+		i.numSpans = 0
+	})
+}
+
+func (i *Memcache) NumSpans() int {
+	return i.numSpans
+}
+
+func (i *Memcache) WithServiceName(name string) {
+	i.opts = append(i.opts, memcachetrace.WithServiceName(name))
+}

contrib/internal/validationtest/contrib/miekg_dns.go

@@ -0,0 +1,125 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2023 Datadog, Inc.
+
+package validationtest
+
+import (
+	"context"
+	"net"
+	"testing"
+	"time"
+
+	dnstrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/miekg/dns"
+
+	"github.com/miekg/dns"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type DNS struct {
+	msg      *dns.Msg
+	mux      *dns.ServeMux
+	addr     string
+	numSpans int
+}
+
+func NewDNS() *DNS {
+	return &DNS{}
+}
+
+func (i *DNS) Name() string {
+	return "miekg/dns"
+}
+
+func (i *DNS) Init(t *testing.T) {
+	// TODO: enable when the integration implements naming schema
+	t.Skip("not implemented yet")
+
+	t.Helper()
+	i.addr = getFreeAddr(t).String()
+	server := &dns.Server{
+		Addr:    i.addr,
+		Net:     "udp",
+		Handler: dnstrace.WrapHandler(&handler{t: t, ig: i}),
+	}
+	// start the traced server
+	go func() {
+		require.NoError(t, server.ListenAndServe())
+	}()
+	// wait for the server to be ready
+	waitServerReady(t, server.Addr)
+	t.Cleanup(func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		assert.NoError(t, server.ShutdownContext(ctx))
+	})
+}
+
+func (i *DNS) GenSpans(t *testing.T) {
+	t.Helper()
+	msg := newMessage()
+	_, err := dnstrace.Exchange(msg, i.addr)
+	require.NoError(t, err)
+	i.numSpans++
+	t.Cleanup(func() {
+		i.numSpans = 0
+	})
+}
+
+func (i *DNS) NumSpans() int {
+	return i.numSpans
+}
+
+func (i *DNS) WithServiceName(_ string) {
+	return
+}
+
+func newMessage() *dns.Msg {
+	m := new(dns.Msg)
+	m.SetQuestion("miek.nl.", dns.TypeMX)
+	return m
+}
+
+type handler struct {
+	t  *testing.T
+	ig *DNS
+}
+
+func (h *handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
+	m := new(dns.Msg)
+	m.SetReply(r)
+	assert.NoError(h.t, w.WriteMsg(m))
+	h.ig.numSpans++
+}
+
+func getFreeAddr(t *testing.T) net.Addr {
+	li, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	addr := li.Addr()
+	require.NoError(t, li.Close())
+	return addr
+}
+
+func waitServerReady(t *testing.T, addr string) {
+	ticker := time.NewTicker(100 * time.Millisecond)
+	defer ticker.Stop()
+	timeoutChan := time.After(5 * time.Second)
+	for {
+		m := new(dns.Msg)
+		m.SetQuestion("miek.nl.", dns.TypeMX)
+		_, err := dns.Exchange(m, addr)
+		if err == nil {
+			break
+		}
+
+		select {
+		case <-ticker.C:
+			continue
+
+		case <-timeoutChan:
+			t.Fatal("timeout waiting for DNS server to be ready")
+		}
+	}
+}