Add headless Chrome as an HTTP client to send requests (#246)

* Refactoring

* Fix placeholders

* Refactoring, add Chrome support as HTTP client

* Fix README.md

* Bump required golang version

* Fix bugs

* Fix chromedp logs

* Fix bug with gRPC placeholder

* Fix config for tests

* Add integration tests for Chrome backend

* Change default client back to Chrome

* Fix vulnerable dependency

* go mod tidy

Author: svkirillov

.gitignore

@@ -1,6 +1,6 @@
 .idea
 .DS_store
-gotestwaf
-main
-reports/*
+/gotestwaf
+/main
+/reports/*
 /modsec_stat_*.txt

Dockerfile

@@ -1,14 +1,19 @@
 # syntax=docker/dockerfile:1
 
 # Build Stage ==================================================================
-FROM golang:1.19-alpine AS build
+FROM golang:1.22-alpine AS build
 
 RUN apk --no-cache add git
 
 WORKDIR /app
-COPY . .
 
-RUN go build -o gotestwaf -ldflags "-X github.com/wallarm/gotestwaf/internal/version.Version=$(git describe --tags)" ./cmd/
+COPY ./go.mod ./go.sum ./
+RUN go mod download
+
+COPY . .
+RUN go build -o gotestwaf \
+    -ldflags "-X github.com/wallarm/gotestwaf/internal/version.Version=$(git describe --tags)" \
+    ./cmd/gotestwaf
 
 
 # Main Stage ===================================================================

Makefile

@@ -4,7 +4,9 @@ gotestwaf:
 	DOCKER_BUILDKIT=1 docker build --force-rm -t gotestwaf .
 
 gotestwaf_bin:
-	go build -o gotestwaf -ldflags "-X github.com/wallarm/gotestwaf/internal/version.Version=$(GOTESTWAF_VERSION)" ./cmd
+	go build -o gotestwaf \
+		-ldflags "-X github.com/wallarm/gotestwaf/internal/version.Version=$(GOTESTWAF_VERSION)" \
+		./cmd/gotestwaf
 
 modsec:
 	docker pull mendhak/http-https-echo:31

README.md

@@ -1,7 +1,7 @@
 # GoTestWAF [![Black Hat Arsenal USA 2022](https://img.shields.io/badge/Black%20Hat%20Arsenal-USA%202022-blue)](https://www.blackhat.com/us-22/arsenal/schedule/index.html#gotestwaf---well-known-open-source-waf-tester-now-supports-api-security-hacking-27986)
 
 GoTestWAF is a tool for API and OWASP attack simulation that supports a wide range of API protocols including
-REST, GraphQL, gRPC, WebSockets, SOAP, XMLRPC, and others.
+REST, GraphQL, gRPC, SOAP, XMLRPC, and others.
 
 It was designed to evaluate web application security solutions, such as API security proxies, Web Application Firewalls,
 IPS, API gateways, and others.
@@ -26,7 +26,6 @@ The results of the security solution evaluation are recorded in the report file
 Default conditions for request generation are defined in the `testcases` folder in the YAML files of the following format:
 
 ```yaml
----
 payload:
   - '"union select -7431.1, name, @aaa from u_base--w-'
   - "'or 123.22=123.22"
@@ -40,8 +39,7 @@ placeholder:
   - UrlParam
   - JSUnicode
   - Header
-type: "SQL Injection"
-...
+type: SQL Injection
 ```
 
 * `payload` is a malicious attack sample (e.g XSS payload like ```<script>alert(111)</script>``` or something more sophisticated).
@@ -70,10 +68,6 @@ Since the format of the YAML string is required for payloads, they must be [enco
     * XMLBody
     * URLParam
     * URLPath
-    * NonCrudUrlPath
-    * NonCrudUrlParam
-    * NonCRUDHeader
-    * NonCRUDRequestBody
     * RawRequest
 
     The `RawRequest` placeholder will allow you to do an arbitrary HTTP request. The payload is substituted by replacing the string `{{payload}}` in the URL path, Headers or body. Fields of `RawRequest` placeholder:
@@ -90,7 +84,6 @@ Since the format of the YAML string is required for payloads, they must be [enco
     Example:
 
     ```yaml
-    ---
     payload:
       - test
     encoder:
@@ -113,8 +106,7 @@ Since the format of the YAML string is required for payloads, they must be [enco
             Knock knock.
             {{payload}}
             --boundary--
-    type: "RawRequest test"
-    ...
+    type: RawRequest test
     ```
 
 * `type` is a name of entire group of the payloads in file. It can be arbitrary, but should reflect the type of attacks in the file.
@@ -376,15 +368,16 @@ Options:
       --blockStatusCodes ints   HTTP status code that WAF uses while blocking requests (default [403])
       --configPath string       Path to the config file (default "config.yaml")
       --email string            E-mail to which the report will be sent
-      --followCookies           If true, use cookies sent by the server. May work only with --maxIdleConns=1
+      --followCookies           If true, use cookies sent by the server. May work only with --maxIdleConns=1 (gohttp only)
       --grpcPort uint16         gRPC port to check
-      --idleConnTimeout int     The maximum amount of time a keep-alive connection will live (default 2)
+      --httpClient string       Which HTTP client use to send requests: chrome, gohttp (default "gohttp")
+      --idleConnTimeout int     The maximum amount of time a keep-alive connection will live (gohttp only) (default 2)
       --ignoreUnresolved        If true, unresolved test cases will be considered as bypassed (affect score and results)
       --includePayloads         If true, payloads will be included in HTML/PDF report
       --logFormat string        Set logging format: text, json (default "text")
       --logLevel string         Logging level: panic, fatal, error, warn, info, debug, trace (default "info")
-      --maxIdleConns int        The maximum number of keep-alive connections (default 2)
-      --maxRedirects int        The maximum number of handling redirects (default 50)
+      --maxIdleConns int        The maximum number of keep-alive connections (gohttp only) (default 2)
+      --maxRedirects int        The maximum number of handling redirects (gohttp only) (default 50)
       --noEmailReport           Save report locally
       --nonBlockedAsPassed      If true, count requests that weren't blocked as passed. If false, requests that don't satisfy to PassStatusCodes/PassRegExp as blocked
       --openapiFile string      Path to openAPI file
@@ -393,7 +386,7 @@ Options:
       --proxy string            Proxy URL to use
       --quiet                   If true, disable verbose logging
       --randomDelay int         Random delay in ms in addition to the delay between requests (default 400)
-      --renewSession            Renew cookies before each test. Should be used with --followCookies flag
+      --renewSession            Renew cookies before each test. Should be used with --followCookies flag (gohttp only)
       --reportFormat string     Export report to one of the following formats: none, pdf, html, json (default "pdf")
       --reportName string       Report file name. Supports `time' package template format (default "waf-evaluation-report-2006-January-02-15-04-05")
       --reportPath string       A directory to store reports (default "reports")
@@ -408,7 +401,6 @@ Options:
       --version                 Show GoTestWAF version and exit
       --wafName string          Name of the WAF product (default "generic")
       --workers int             The number of workers to scan (default 5)
-      --wsURL string            WebSocket URL to check
 ```
 
 The listed options can be passed to GoTestWAF as follows:

cmd/flags.go cmd/gotestwaf/flags.go

@@ -34,8 +34,7 @@ const (
 	jsonLogFormat = "json"
 
 	cliDescription = `GoTestWAF is a tool for API and OWASP attack simulation that supports a
-wide range of API protocols including REST, GraphQL, gRPC, WebSockets,
-SOAP, XMLRPC, and others.
+wide range of API protocols including REST, GraphQL, gRPC, SOAP, XMLRPC, and others.
 Homepage: https://github.com/wallarm/gotestwaf
 
 Usage: %s [OPTIONS] --url <URL>
@@ -67,21 +66,44 @@ func parseFlags() (args []string, err error) {
 
 	flag.Usage = usage
 
+	// General parameters
 	flag.StringVar(&configPath, "configPath", defaultConfigPath, "Path to the config file")
 	flag.BoolVar(&quiet, "quiet", false, "If true, disable verbose logging")
 	logLvl := flag.String("logLevel", "info", "Logging level: panic, fatal, error, warn, info, debug, trace")
 	flag.StringVar(&logFormat, "logFormat", textLogFormat, "Set logging format: text, json")
+	showVersion := flag.Bool("version", false, "Show GoTestWAF version and exit")
 
+	// Target settings
 	urlParam := flag.String("url", "", "URL to check")
-	wsURL := flag.String("wsURL", "", "WebSocket URL to check")
 	flag.Uint16("grpcPort", 0, "gRPC port to check")
-	flag.String("proxy", "", "Proxy URL to use")
+	openapiFile := flag.String("openapiFile", "", "Path to openAPI file")
+
+	// Test cases settings
+	flag.String("testCase", "", "If set then only this test case will be run")
+	flag.String("testCasesPath", testCasesPath, "Path to a folder with test cases")
+	flag.String("testSet", "", "If set then only this test set's cases will be run")
+
+	// HTTP client settings
+	httpClient := flag.String("httpClient", "gohttp", "Which HTTP client use to send requests: chrome, gohttp")
 	flag.Bool("tlsVerify", false, "If true, the received TLS certificate will be verified")
-	flag.Int("maxIdleConns", 2, "The maximum number of keep-alive connections")
-	flag.Int("maxRedirects", 50, "The maximum number of handling redirects")
-	flag.Int("idleConnTimeout", 2, "The maximum amount of time a keep-alive connection will live")
-	flag.Bool("followCookies", false, "If true, use cookies sent by the server. May work only with --maxIdleConns=1")
-	flag.Bool("renewSession", false, "Renew cookies before each test. Should be used with --followCookies flag")
+	flag.String("proxy", "", "Proxy URL to use")
+	flag.String("addHeader", "", "An HTTP header to add to requests")
+	flag.Bool("addDebugHeader", false, "Add header with a hash of the test information in each request")
+
+	// GoHTTP client only settings
+	flag.Int("maxIdleConns", 2, "The maximum number of keep-alive connections (gohttp only)")
+	flag.Int("maxRedirects", 50, "The maximum number of handling redirects (gohttp only)")
+	flag.Int("idleConnTimeout", 2, "The maximum amount of time a keep-alive connection will live (gohttp only)")
+	flag.Bool("followCookies", false, "If true, use cookies sent by the server. May work only with --maxIdleConns=1 (gohttp only)")
+	flag.Bool("renewSession", false, "Renew cookies before each test. Should be used with --followCookies flag (gohttp only)")
+
+	// Performance settings
+	flag.Int("workers", 5, "The number of workers to scan")
+	flag.Int("sendDelay", 400, "Delay in ms between requests")
+	flag.Int("randomDelay", 400, "Random delay in ms in addition to the delay between requests")
+
+	// Analysis settings
+	flag.Bool("skipWAFBlockCheck", false, "If true, WAF detection tests will be skipped")
 	flag.Bool("skipWAFIdentification", false, "Skip WAF identification")
 	flag.IntSlice("blockStatusCodes", []int{403}, "HTTP status code that WAF uses while blocking requests")
 	flag.IntSlice("passStatusCodes", []int{200, 404}, "HTTP response status code that WAF uses while passing requests")
@@ -91,26 +113,18 @@ func parseFlags() (args []string, err error) {
 		"Regex to a detect normal (not blocked) web page with the same HTTP status code as a blocked request")
 	flag.Bool("nonBlockedAsPassed", false,
 		"If true, count requests that weren't blocked as passed. If false, requests that don't satisfy to PassStatusCodes/PassRegExp as blocked")
-	flag.Int("workers", 5, "The number of workers to scan")
-	flag.Int("sendDelay", 400, "Delay in ms between requests")
-	flag.Int("randomDelay", 400, "Random delay in ms in addition to the delay between requests")
-	flag.String("testCase", "", "If set then only this test case will be run")
-	flag.String("testSet", "", "If set then only this test set's cases will be run")
+	flag.Bool("ignoreUnresolved", false, "If true, unresolved test cases will be considered as bypassed (affect score and results)")
+	flag.Bool("blockConnReset", false, "If true, connection resets will be considered as block")
+
+	// Report settings
+	flag.String("wafName", wafName, "Name of the WAF product")
+	flag.Bool("includePayloads", false, "If true, payloads will be included in HTML/PDF report")
 	flag.String("reportPath", reportPath, "A directory to store reports")
 	reportName := flag.String("reportName", defaultReportName, "Report file name. Supports `time' package template format")
 	flag.String("reportFormat", "pdf", "Export report to one of the following formats: none, pdf, html, json")
-	flag.Bool("includePayloads", false, "If true, payloads will be included in HTML/PDF report")
 	noEmailReport := flag.Bool("noEmailReport", false, "Save report locally")
 	email := flag.String("email", "", "E-mail to which the report will be sent")
-	flag.String("testCasesPath", testCasesPath, "Path to a folder with test cases")
-	flag.String("wafName", wafName, "Name of the WAF product")
-	flag.Bool("ignoreUnresolved", false, "If true, unresolved test cases will be considered as bypassed (affect score and results)")
-	flag.Bool("blockConnReset", false, "If true, connection resets will be considered as block")
-	flag.Bool("skipWAFBlockCheck", false, "If true, WAF detection tests will be skipped")
-	flag.String("addHeader", "", "An HTTP header to add to requests")
-	flag.Bool("addDebugHeader", false, "Add header with a hash of the test information in each request")
-	flag.String("openapiFile", "", "Path to openAPI file")
-	showVersion := flag.Bool("version", false, "Show GoTestWAF version and exit")
+
 	flag.Parse()
 
 	if len(os.Args) == 1 {
@@ -165,16 +179,10 @@ func parseFlags() (args []string, err error) {
 
 	*urlParam = validURL.String()
 
-	// format WebSocket URL from given HTTP URL
-	if *wsURL == "" {
-		wsScheme := "ws"
-		if validURL.Scheme == "https" {
-			wsScheme = "wss"
-		}
-		validURL.Scheme = wsScheme
-		validURL.Path = ""
-
-		*wsURL = validURL.String()
+	// Force GoHTTP to be used as the HTTP client
+	// when scanning against the OpenAPI spec.
+	if openapiFile != nil && len(*openapiFile) > 0 {
+		*httpClient = "gohttp"
 	}
 
 	if *blockRegex != "" {