Skip to content

Commit f010bfe

Browse files
parthmalhotraparthmalhotra
committed
Added Awesome Search Queries intigration (-cpe) and Passive Wordpress plugin and theme detection (-wp, -wordpress) flag
1 parent c4b06f6 commit f010bfe

File tree

8 files changed

+381
-4
lines changed

8 files changed

+381
-4
lines changed

‎common/httpx/option.go

+2
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ type Options struct {
5050
CDNCheckClient *cdncheck.Client
5151
Protocol Proto
5252
Trace bool
53+
AwesomeSearchQueries bool
5354
}
5455

5556
// DefaultOptions contains the default options
@@ -71,6 +72,7 @@ var DefaultOptions = Options{
7172
VHostStripHTML: false,
7273
VHostSimilarityRatio: 85,
7374
DefaultUserAgent: "httpx - Open-source project (github.com/projectdiscovery/httpx)",
75+
AwesomeSearchQueries: false,
7476
}
7577

7678
func (options *Options) parseCustomCookies() {

‎go.mod

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
module github.com/projectdiscovery/httpx
22

3-
go 1.21
3+
go 1.21.3
4+
5+
toolchain go1.23.2
46

57
require (
68
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057
@@ -52,6 +54,7 @@ require (
5254

5355
require (
5456
github.com/go-viper/mapstructure/v2 v2.1.0
57+
github.com/projectdiscovery/awesome-search-queries v0.0.0-20241030094221-9fa3c0933578
5558
github.com/weppos/publicsuffix-go v0.30.2
5659
)
5760

‎go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF
226226
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
227227
github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kID2iwsDqI=
228228
github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60=
229+
github.com/projectdiscovery/awesome-search-queries v0.0.0-20241030094221-9fa3c0933578 h1:PSpd8NNjmDUgfbhgB/49HZqXQ+7DJRnMR1TVyEus7PM=
230+
github.com/projectdiscovery/awesome-search-queries v0.0.0-20241030094221-9fa3c0933578/go.mod h1:nSovPcipgSx/EzAefF+iCfORolkKAuodiRWL3RCGHOM=
229231
github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ=
230232
github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
231233
github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/snk0cQ3kXE=

‎runner/awesome_queries.go

+141
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
package runner
2+
3+
import (
4+
"encoding/json"
5+
"strings"
6+
7+
awesomesearchqueries "github.com/projectdiscovery/awesome-search-queries"
8+
)
9+
10+
type AwesomeQuery struct {
11+
Name string `json:"name"`
12+
Vendor string `json:"vendor"`
13+
Type string `json:"type"`
14+
Engines []Engines `json:"engines"`
15+
}
16+
17+
type Engines struct {
18+
Platform string `json:"platform"`
19+
Queries []string `json:"queries"`
20+
}
21+
22+
type AwesomeSearchMaps struct {
23+
aqTitle map[string][]ProductVendor
24+
aqBody map[string][]ProductVendor
25+
aqFavicon map[string][]ProductVendor
26+
}
27+
28+
type ProductVendor struct {
29+
Product string
30+
Vendor string
31+
}
32+
33+
func LoadAwesomeQueries() (*AwesomeSearchMaps, error) {
34+
data, err := awesomesearchqueries.GetQueries()
35+
if err != nil {
36+
return nil, err
37+
}
38+
39+
var queries []AwesomeQuery
40+
if err := json.Unmarshal(data, &queries); err != nil {
41+
return nil, err
42+
}
43+
44+
maps := &AwesomeSearchMaps{
45+
aqTitle: make(map[string][]ProductVendor),
46+
aqBody: make(map[string][]ProductVendor),
47+
aqFavicon: make(map[string][]ProductVendor),
48+
}
49+
50+
for _, query := range queries {
51+
pv := ProductVendor{
52+
Product: query.Name,
53+
Vendor: query.Vendor,
54+
}
55+
56+
for _, engine := range query.Engines {
57+
for _, q := range engine.Queries {
58+
switch engine.Platform {
59+
case "shodan":
60+
if strings.HasPrefix(q, "http.html:") {
61+
maps.aqBody[extractQuery(q, "http.html:")] = append(maps.aqBody[extractQuery(q, "http.html:")], pv)
62+
} else if strings.HasPrefix(q, "http.title:") {
63+
maps.aqTitle[extractQuery(q, "http.title:")] = append(maps.aqTitle[extractQuery(q, "http.title:")], pv)
64+
} else if strings.HasPrefix(q, "http.favicon.hash:") {
65+
maps.aqFavicon[extractQuery(q, "http.favicon.hash:")] = append(maps.aqFavicon[extractQuery(q, "http.favicon.hash:")], pv)
66+
}
67+
case "fofa":
68+
if strings.HasPrefix(q, "body=") {
69+
maps.aqBody[extractQuery(q, "body=")] = append(maps.aqBody[extractQuery(q, "body=")], pv)
70+
} else if strings.HasPrefix(q, "title=") {
71+
maps.aqTitle[extractQuery(q, "title=")] = append(maps.aqTitle[extractQuery(q, "title=")], pv)
72+
} else if strings.HasPrefix(q, "icon_hash=") {
73+
maps.aqFavicon[extractQuery(q, "icon_hash=")] = append(maps.aqFavicon[extractQuery(q, "icon_hash=")], pv)
74+
}
75+
case "google":
76+
if strings.HasPrefix(q, "intext:") {
77+
maps.aqBody[extractQuery(q, "intext:")] = append(maps.aqBody[extractQuery(q, "intext:")], pv)
78+
} else if strings.HasPrefix(q, "intitle:") {
79+
maps.aqTitle[extractQuery(q, "intitle:")] = append(maps.aqTitle[extractQuery(q, "intitle:")], pv)
80+
}
81+
}
82+
}
83+
}
84+
}
85+
86+
return maps, nil
87+
}
88+
89+
func extractQuery(query string, prefix string) string {
90+
q := strings.TrimPrefix(query, prefix)
91+
return strings.Trim(q, "\"")
92+
}
93+
94+
func (a *AwesomeSearchMaps) FindMatches(result *Result) ([]ProductVendor, bool) {
95+
var matches []ProductVendor
96+
matchMap := make(map[string]bool)
97+
98+
if result.Title != "" {
99+
for title, pvs := range a.aqTitle {
100+
if strings.Contains(strings.ToLower(result.Title), strings.ToLower(title)) {
101+
for _, pv := range pvs {
102+
key := pv.Product + pv.Vendor
103+
if !matchMap[key] {
104+
matches = append(matches, pv)
105+
matchMap[key] = true
106+
}
107+
}
108+
}
109+
}
110+
}
111+
112+
if result.ResponseBody != "" {
113+
for body, pvs := range a.aqBody {
114+
if strings.Contains(strings.ToLower(result.ResponseBody), strings.ToLower(body)) {
115+
for _, pv := range pvs {
116+
key := pv.Product + pv.Vendor
117+
if !matchMap[key] {
118+
matches = append(matches, pv)
119+
matchMap[key] = true
120+
}
121+
}
122+
}
123+
}
124+
}
125+
126+
if result.FavIconMMH3 != "" {
127+
for favicon, pvs := range a.aqFavicon {
128+
if result.FavIconMMH3 == favicon {
129+
for _, pv := range pvs {
130+
key := pv.Product + pv.Vendor
131+
if !matchMap[key] {
132+
matches = append(matches, pv)
133+
matchMap[key] = true
134+
}
135+
}
136+
}
137+
}
138+
}
139+
140+
return matches, len(matches) > 0
141+
}

‎runner/options.go

+12-3
Original file line numberDiff line numberDiff line change
@@ -335,9 +335,11 @@ type Options struct {
335335
Trace bool
336336

337337
// Optional pre-created objects to reduce allocations
338-
Wappalyzer *wappalyzer.Wappalyze
339-
Networkpolicy *networkpolicy.NetworkPolicy
340-
CDNCheckClient *cdncheck.Client
338+
Wappalyzer *wappalyzer.Wappalyze
339+
Networkpolicy *networkpolicy.NetworkPolicy
340+
CDNCheckClient *cdncheck.Client
341+
AwesomeSearchQueries bool
342+
WordPress bool
341343
}
342344

343345
// ParseOptions parses the command line options for application
@@ -369,6 +371,7 @@ func ParseOptions() *Options {
369371
flagSet.DynamicVarP(&options.ResponseBodyPreviewSize, "body-preview", "bp", 100, "display first N characters of response body"),
370372
flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "display server name"),
371373
flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "display technology in use based on wappalyzer dataset"),
374+
flagSet.BoolVar(&options.AwesomeSearchQueries, "cpe", false, "display product and vendor information based on awesome search queries"),
372375
flagSet.BoolVar(&options.OutputMethod, "method", false, "display http request method"),
373376
flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "display server using websocket"),
374377
flagSet.BoolVar(&options.OutputIP, "ip", false, "display host ip"),
@@ -377,6 +380,7 @@ func ParseOptions() *Options {
377380
flagSet.BoolVar(&options.Asn, "asn", false, "display host asn information"),
378381
flagSet.DynamicVar(&options.OutputCDN, "cdn", "true", "display cdn/waf in use"),
379382
flagSet.BoolVar(&options.Probe, "probe", false, "display probe status"),
383+
flagSet.BoolVarP(&options.WordPress, "wordpress", "wp", false, "display WordPress themes and plugins"),
380384
)
381385

382386
flagSet.CreateGroup("headless", "Headless",
@@ -623,6 +627,11 @@ func ParseOptions() *Options {
623627
gologger.Fatal().Msgf("%s\n", err)
624628
}
625629

630+
// Enable WordPress detection for JSON output
631+
if options.JSONOutput {
632+
options.WordPress = true
633+
}
634+
626635
return options
627636
}
628637

‎runner/runner.go

+95
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,8 @@ type Runner struct {
9090
pHashClusters []pHashCluster
9191
simHashes gcache.Cache[uint64, struct{}] // Include simHashes for efficient duplicate detection
9292
httpApiEndpoint *Server
93+
awesomeQueries *AwesomeSearchMaps
94+
wpData *WordPressData
9395
}
9496

9597
func (r *Runner) HTTPX() *httpx.HTTPX {
@@ -375,6 +377,26 @@ func New(options *Options) (*Runner, error) {
375377
}()
376378
}
377379

380+
if options.JSONOutput || options.AwesomeSearchQueries {
381+
aq, err := LoadAwesomeQueries()
382+
if err != nil {
383+
gologger.Warning().Msgf("Could not load awesome search queries: %s", err)
384+
} else {
385+
runner.awesomeQueries = aq
386+
}
387+
}
388+
389+
if options.WordPress {
390+
wpData, err := NewWordPressData()
391+
if err != nil {
392+
return nil, err
393+
}
394+
if err := wpData.LoadData(); err != nil {
395+
gologger.Warning().Msgf("Could not load WordPress data: %s", err)
396+
}
397+
runner.wpData = wpData
398+
}
399+
378400
return runner, nil
379401
}
380402

@@ -2225,6 +2247,74 @@ retry:
22252247
}
22262248
}
22272249

2250+
// Add awesome queries check here, before creating the result struct
2251+
var product, vendor, cpe string
2252+
if r.awesomeQueries != nil {
2253+
tempResult := Result{
2254+
Title: title,
2255+
ResponseBody: string(resp.Data),
2256+
FavIconMMH3: faviconMMH3,
2257+
}
2258+
if matches, found := r.awesomeQueries.FindMatches(&tempResult); found && len(matches) > 0 {
2259+
product = matches[0].Product
2260+
vendor = matches[0].Vendor
2261+
cpe = fmt.Sprintf("cpe:2.3:a:%s:%s:*:*:*:*:*:*:*:*", strings.ToLower(vendor), strings.ToLower(product))
2262+
2263+
// Update the builder string for CLI output
2264+
if r.options.AwesomeSearchQueries {
2265+
builder.WriteString(" [")
2266+
if !scanopts.OutputWithNoColor {
2267+
builder.WriteString(aurora.Magenta(cpe).String())
2268+
} else {
2269+
builder.WriteString(cpe)
2270+
}
2271+
builder.WriteString("] [")
2272+
if !scanopts.OutputWithNoColor {
2273+
builder.WriteString(aurora.Magenta(vendor).String())
2274+
} else {
2275+
builder.WriteString(vendor)
2276+
}
2277+
builder.WriteString("] [")
2278+
if !scanopts.OutputWithNoColor {
2279+
builder.WriteString(aurora.Magenta(product).String())
2280+
} else {
2281+
builder.WriteString(product)
2282+
}
2283+
builder.WriteString("]")
2284+
}
2285+
}
2286+
}
2287+
2288+
var wpInfo *WordPressInfo
2289+
if r.wpData != nil {
2290+
wpInfo = r.wpData.ExtractInfo(string(resp.Data))
2291+
if wpInfo != nil {
2292+
builder.WriteString(" [")
2293+
if !scanopts.OutputWithNoColor {
2294+
if len(wpInfo.Plugins) > 0 {
2295+
builder.WriteString(aurora.Magenta(fmt.Sprintf("WP Plugins: %s", strings.Join(wpInfo.Plugins, ","))).String())
2296+
}
2297+
if len(wpInfo.Themes) > 0 {
2298+
if len(wpInfo.Plugins) > 0 {
2299+
builder.WriteString("] [")
2300+
}
2301+
builder.WriteString(aurora.Magenta(fmt.Sprintf("WP Themes: %s", strings.Join(wpInfo.Themes, ","))).String())
2302+
}
2303+
} else {
2304+
if len(wpInfo.Plugins) > 0 {
2305+
builder.WriteString(fmt.Sprintf("WP Plugins: %s", strings.Join(wpInfo.Plugins, ",")))
2306+
}
2307+
if len(wpInfo.Themes) > 0 {
2308+
if len(wpInfo.Plugins) > 0 {
2309+
builder.WriteString("] [")
2310+
}
2311+
builder.WriteString(fmt.Sprintf("WP Themes: %s", strings.Join(wpInfo.Themes, ",")))
2312+
}
2313+
}
2314+
builder.WriteString("]")
2315+
}
2316+
}
2317+
22282318
result := Result{
22292319
Timestamp: time.Now(),
22302320
Request: request,
@@ -2286,6 +2376,10 @@ retry:
22862376
RequestRaw: requestDump,
22872377
Response: resp,
22882378
FaviconData: faviconData,
2379+
Product: product,
2380+
Vendor: vendor,
2381+
CPE: cpe,
2382+
WordPress: wpInfo,
22892383
}
22902384
if resp.BodyDomains != nil {
22912385
result.Fqdns = resp.BodyDomains.Fqdns
@@ -2294,6 +2388,7 @@ retry:
22942388
if r.options.Trace {
22952389
result.Trace = req.TraceInfo
22962390
}
2391+
22972392
return result
22982393
}
22992394

‎runner/types.go

+4
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,10 @@ type Result struct {
100100
Response *httpx.Response `json:"-" csv:"-" mapstructure:"-"`
101101
FaviconData []byte `json:"-" csv:"-" mapstructure:"-"`
102102
Trace *retryablehttp.TraceInfo `json:"trace,omitempty" csv:"trace" mapstructure:"trace"`
103+
Product string `json:"product,omitempty" csv:"product"`
104+
Vendor string `json:"vendor,omitempty" csv:"vendor"`
105+
WordPress *WordPressInfo `json:"wordpress,omitempty" csv:"wordpress"`
106+
CPE string `json:"cpe,omitempty" csv:"cpe"`
103107
}
104108

105109
type Trace struct {

0 commit comments

Comments
 (0)