Expand E2E test suite (#2645)

* Expand E2E test suite

Add tests to the stand alone and fleet-server in an agent container test
suite that use autogenerated client code in order to test most API
endpoints (only the artifacts endpoint is missing).
Add a test to the container suite that enrolls another elastic-agent
container instance into fleet and verifies its status.

* Fix unit tests

* Fix typo

* Add basic artifacts test

* Verify artifact body

* Fix comments

Author: michel-laterman

dev-tools/e2e/certs.sh

@@ -38,15 +38,15 @@ openssl req -new \
     -key ${CERT_DIR}/fleet-server.key \
     -passin file:${CERT_DIR}/passphrase \
     -subj "/CN=localhost" \
-    -addext "subjectAltName=IP:127.0.0.1,DNS:localhost" \
+    -addext "subjectAltName=IP:127.0.0.1,DNS:localhost,DNS:fleet-server" \
     -out ${CERT_DIR}/fleet-server.csr \
     2>/dev/null
 
 # Sign CSR with CA
 openssl x509 -req \
     -in ${CERT_DIR}/fleet-server.csr \
     -days 356 \
-    -extfile <(printf "subjectAltName=IP:127.0.0.1,DNS:localhost") \
+    -extfile <(printf "subjectAltName=IP:127.0.0.1,DNS:localhost,DNS:fleet-server") \
     -CA ${CERT_DIR}/e2e-test-ca.crt \
     -CAkey ${CERT_DIR}/e2e-test-ca.key \
     -CAcreateserial \

dev-tools/e2e/kibana.yml

@@ -4,15 +4,26 @@ server.ssl.enabled: false
 elasticsearch.hosts: [ "http://elasticsearch:9200" ]
 elasticsearch.serviceAccountToken: "${KIBANA_TOKEN}"
 
+xpack.securitySolution.packagerTaskInterval: 1s
 xpack.fleet.agents.enabled: true
-xpack.fleet.agents.fleet_server.hosts: ["http://fleet-server:8220"]
+xpack.fleet.agents.fleet_server.hosts: ["https://fleet-server:8220"]
 xpack.fleet.agents.elasticsearch.hosts: ["http://elasticsearch:9200"]
 xpack.fleet.packages:
   - name: elastic_agent
     version: latest
   - name: fleet_server
     version: latest
 xpack.fleet.agentPolicies:
+  - name: dummy-policy
+    id: dummy-policy
+    namespace: default
+    monitoring_enabled: []
+    package_policies: []
+  - name: security-policy
+    id: security-policy
+    namespace: default
+    monitoring_enabled: []
+    package_policies: []
   - name: fleet-server-policy
     id: fleet-server-policy
     monitoring_enabled: []

internal/pkg/api/api.go

@@ -34,6 +34,7 @@ var _ ServerInterface = (*apiServer)(nil)
 
 func (a *apiServer) AgentEnroll(w http.ResponseWriter, r *http.Request, params AgentEnrollParams) {
 	zlog := hlog.FromRequest(r).With().Str("mod", kEnrollMod).Logger()
+	w.Header().Set("Content-Type", "application/json")
 
 	// Error in the scope for deferred rolback function check
 	var err error
@@ -60,6 +61,7 @@ func (a *apiServer) AgentEnroll(w http.ResponseWriter, r *http.Request, params A
 
 func (a *apiServer) AgentAcks(w http.ResponseWriter, r *http.Request, id string, params AgentAcksParams) {
 	zlog := hlog.FromRequest(r).With().Str(LogAgentID, id).Logger()
+	w.Header().Set("Content-Type", "application/json")
 	if err := a.ack.handleAcks(zlog, w, r, id); err != nil {
 		cntAcks.IncError(err)
 		ErrorResp(w, r, err)
@@ -68,6 +70,7 @@ func (a *apiServer) AgentAcks(w http.ResponseWriter, r *http.Request, id string,
 
 func (a *apiServer) AgentCheckin(w http.ResponseWriter, r *http.Request, id string, params AgentCheckinParams) {
 	zlog := hlog.FromRequest(r).With().Str(LogAgentID, id).Logger()
+	w.Header().Set("Content-Type", "application/json")
 	err := a.ct.handleCheckin(zlog, w, r, id, params.UserAgent)
 	if err != nil {
 		cntCheckin.IncError(err)
@@ -84,13 +87,15 @@ func (a *apiServer) Artifact(w http.ResponseWriter, r *http.Request, id string,
 
 	err := a.at.handleArtifacts(zlog, w, r, id, sha2)
 	if err != nil {
+		w.Header().Set("Content-Type", "application/json")
 		cntArtifacts.IncError(err)
 		ErrorResp(w, r, err)
 	}
 }
 
 func (a *apiServer) UploadBegin(w http.ResponseWriter, r *http.Request, params UploadBeginParams) {
 	zlog := hlog.FromRequest(r).With().Logger()
+	w.Header().Set("Content-Type", "application/json")
 	err := a.ut.handleUploadBegin(zlog, w, r)
 	if err != nil {
 		cntUploadStart.IncError(err)
@@ -100,6 +105,7 @@ func (a *apiServer) UploadBegin(w http.ResponseWriter, r *http.Request, params U
 
 func (a *apiServer) UploadComplete(w http.ResponseWriter, r *http.Request, id string, params UploadCompleteParams) {
 	zlog := hlog.FromRequest(r).With().Str(LogAgentID, id).Logger()
+	w.Header().Set("Content-Type", "application/json")
 	err := a.ut.handleUploadComplete(zlog, w, r, id)
 	if err != nil {
 		cntUploadEnd.IncError(err)
@@ -109,6 +115,7 @@ func (a *apiServer) UploadComplete(w http.ResponseWriter, r *http.Request, id st
 
 func (a *apiServer) UploadChunk(w http.ResponseWriter, r *http.Request, id string, chunkNum int, params UploadChunkParams) {
 	zlog := hlog.FromRequest(r).With().Str(LogAgentID, id).Logger()
+	w.Header().Set("Content-Type", "application/json")
 
 	if _, err := a.ut.authAPIKey(r, a.bulker, a.ut.cache); err != nil {
 		cntUploadChunk.IncError(err)
@@ -125,6 +132,7 @@ func (a *apiServer) Status(w http.ResponseWriter, r *http.Request, params Status
 	zlog := hlog.FromRequest(r).With().
 		Str("mod", kStatusMod).
 		Logger()
+	w.Header().Set("Content-Type", "application/json")
 	err := a.st.handleStatus(zlog, a.sm, a.bi, r, w)
 	if err != nil {
 		cntStatus.IncError(err)

internal/pkg/api/handleStatus.go

@@ -68,7 +68,7 @@ func (st StatusT) handleStatus(zlog zerolog.Logger, sm policy.SelfMonitor, bi bu
 	}
 
 	state := sm.State()
-	resp := StatusResponse{
+	resp := StatusAPIResponse{
 		Name:   build.ServiceName,
 		Status: StatusResponseStatus(state.String()), // TODO try to make the oapi codegen less verbose here
 	}

internal/pkg/api/handleStatus_test.go

@@ -108,7 +108,7 @@ func TestHandleStatus(t *testing.T) {
 
 					assert.Equal(t, expectedCode, w.Code)
 
-					var res StatusResponse
+					var res StatusAPIResponse
 					err := json.Unmarshal(w.Body.Bytes(), &res)
 					require.NoError(t, err)
 

internal/pkg/api/handleUpload.go

@@ -95,7 +95,7 @@ func (ut *UploadT) handleUploadBegin(_ zerolog.Logger, w http.ResponseWriter, r
 	}
 
 	// prepare and write response
-	resp := UploadBeginResponse{
+	resp := UploadBeginAPIResponse{
 		ChunkSize: info.ChunkSize,
 		UploadId:  info.ID,
 	}

internal/pkg/api/handleUpload_test.go

@@ -283,13 +283,12 @@ func TestUploadBeginResponse(t *testing.T) {
 	hr.ServeHTTP(rec, req)
 	assert.Equal(t, http.StatusOK, rec.Code)
 
-	var response UploadBeginResponse
+	var response UploadBeginAPIResponse
 	err := json.Unmarshal(rec.Body.Bytes(), &response)
 	assert.NoErrorf(t, err, "upload start should provide valid JSON response")
 
 	assert.NotEmptyf(t, response.UploadId, "upload start response should provide an ID")
 	assert.Greaterf(t, response.ChunkSize, int64(0), "upload start response should provide a chunk size > 0")
-
 }
 
 /*

internal/pkg/api/openapi.gen.go

@@ -4,8 +4,12 @@
 
 //go:generate schema-generate -esdoc -s -cm "{\"Api\": \"API\", \"Id\": \"ID\"}" -o internal/pkg/model/schema.go -p model model/schema.json
 //go:generate oapi-codegen --config model/oapi-cfg.yml model/openapi.yml
+//go:generate oapi-codegen -generate types -package api -o pkg/api/types.gen.go  model/openapi.yml
+//go:generate oapi-codegen -generate client -package api -o pkg/api/client.gen.go  model/openapi.yml
 //go:generate go fmt internal/pkg/model/schema.go
 //go:generate go fmt internal/pkg/api/openapi.gen.go
+//go:generate go fmt pkg/api/types.gen.go
+//go:generate go fmt pkg/api/client.gen.go
 
 package main
 

main.go

@@ -58,6 +58,7 @@ components:
           description: The date-time that the fleet-server binary was created.
           #format: date-time # not using date-time format at the moment because the currently available objects have plain strings
     statusResponse:
+      x-go-name: StatusAPIResponse
       description: Status response information.
       type: object
       required:
@@ -643,6 +644,7 @@ components:
             - endpoint
             - agent
     uploadBeginResponse:
+      x-go-name: UploadBeginAPIResponse
       description: Response to initiating a file upload
       type: object
       required: