diff --git a/common/httpx/csp.go b/common/httpx/csp.go index 8950fd77..b351c6ac 100644 --- a/common/httpx/csp.go +++ b/common/httpx/csp.go @@ -2,11 +2,13 @@ package httpx import ( "bytes" + "net/url" "strings" "github.com/PuerkitoBio/goquery" mapsutil "github.com/projectdiscovery/utils/maps" stringsutil "github.com/projectdiscovery/utils/strings" + "github.com/weppos/publicsuffix-go/publicsuffix" ) // CSPHeaders is an incomplete list of most common CSP headers @@ -19,17 +21,19 @@ var CSPHeaders = []string{ // CSPData contains the Content-Security-Policy domain list type CSPData struct { + Fqdns []string `json:"fqdns,omitempty"` Domains []string `json:"domains,omitempty"` } // CSPGrab fills the CSPData func (h *HTTPX) CSPGrab(r *Response) *CSPData { domains := make(map[string]struct{}) + fqdns := make(map[string]struct{}) // extract from headers for _, cspHeader := range CSPHeaders { if cspValues, ok := r.Headers[cspHeader]; ok { for _, cspValue := range cspValues { - parsePotentialDomains(domains, cspValue) + parsePotentialDomains(fqdns, domains, cspValue) } } } @@ -41,26 +45,29 @@ func (h *HTTPX) CSPGrab(r *Response) *CSPData { doc.Find("meta").Each(func(i int, s *goquery.Selection) { if _, ok := s.Attr("http-equiv"); ok { if content, ok := s.Attr("content"); ok { - parsePotentialDomains(domains, content) + parsePotentialDomains(fqdns, domains, content) } } }) } } - if len(domains) > 0 { - return &CSPData{Domains: mapsutil.GetKeys(domains)} + if len(domains) > 0 || len(fqdns) > 0 { + return &CSPData{Domains: mapsutil.GetKeys(domains), Fqdns: mapsutil.GetKeys(fqdns)} } return nil } -func parsePotentialDomains(domains map[string]struct{}, data string) { +func parsePotentialDomains(fqdns, domains map[string]struct{}, data string) { // rule is like aa bb domain1 domain2 domain3 tokens := stringsutil.SplitAny(data, " ", ";", ",") // we extracts only potential domains for _, t := range tokens { if isPotentialDomain(t) { - domains[t] = struct{}{} + if dn, err := publicsuffix.Parse(extractDomain(t)); err == nil { + domains[dn.SLD+"."+dn.TLD] = struct{}{} + fqdns[dn.String()] = struct{}{} + } } } } @@ -68,3 +75,15 @@ func parsePotentialDomains(domains map[string]struct{}, data string) { func isPotentialDomain(s string) bool { return strings.Contains(s, ".") || strings.HasPrefix(s, "http") } + +func extractDomain(str string) string { + u := str + if !strings.Contains(str, "://") { + u = "https://" + str + } + parsedURL, err := url.Parse(u) + if err != nil { + return str + } + return parsedURL.Host +}