lint + msg

projectdiscovery · Oct 16, 2024 · 5a528e3 · 5a528e3
1 parent 769c562
commit 5a528e3
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 7 deletions.
diff --git a/runner/options.go b/runner/options.go
@@ -410,7 +410,7 @@ func ParseOptions() *Options {
 	flagSet.CreateGroup("filters", "Filters",
 		flagSet.StringVarP(&options.OutputFilterStatusCode, "filter-code", "fc", "", "filter response with specified status code (-fc 403,401)"),
 		flagSet.BoolVarP(&options.OutputFilterErrorPage, "filter-error-page", "fep", false, "filter response with ML based error page detection"),
-		flagSet.BoolVarP(&options.FilterOutDuplicates, "filter-duplicates", "fd", false, "filter out near-duplicate responses"),
+		flagSet.BoolVarP(&options.FilterOutDuplicates, "filter-duplicates", "fd", false, "filter out near-duplicate responses (only first response is retained)"),
 		flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "filter response with specified content length (-fl 23,33)"),
 		flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"),
 		flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "filter response body with specified word count (-fwc 423,532)"),

diff --git a/runner/runner.go b/runner/runner.go
@@ -66,6 +66,7 @@ import (
 	"github.com/projectdiscovery/httpx/common/stringz"
 	"github.com/projectdiscovery/mapcidr"
 	"github.com/projectdiscovery/rawhttp"
+	converstionutil "github.com/projectdiscovery/utils/conversion"
 	fileutil "github.com/projectdiscovery/utils/file"
 	pdhttputil "github.com/projectdiscovery/utils/http"
 	iputil "github.com/projectdiscovery/utils/ip"
@@ -517,21 +518,21 @@ func (r *Runner) seen(k string) bool {
 	return ok
 }
 
-func (r *Runner) duplicate(resp []byte) bool {
-	respSimHash := simhash.Simhash(simhash.NewWordFeatureSet(resp))
+func (r *Runner) duplicate(result *Result) bool {
+	respSimHash := simhash.Simhash(simhash.NewWordFeatureSet(converstionutil.Bytes(result.Raw)))
 	if r.simHashes.Has(respSimHash) {
-		gologger.Warning().Msgf("Skipping duplicate response with simhash %d\n", respSimHash)
+		gologger.Debug().Msgf("Skipping duplicate response with simhash %d for URL %s\n", respSimHash, result.URL)
 		return true
 	}
 
 	for simHash := range r.simHashes.GetALL(false) {
 		// lower threshold for increased precision
 		if simhash.Compare(simHash, respSimHash) <= 3 {
-			gologger.Warning().Msgf("Skipping near-duplicate response with simhash %d\n", respSimHash)
+			gologger.Debug().Msgf("Skipping near-duplicate response with simhash %d for URL %s\n", respSimHash, result.URL)
 			return true
 		}
 	}
-	r.simHashes.Set(respSimHash, struct{}{})
+	_ = r.simHashes.Set(respSimHash, struct{}{})
 	return false
 }
 
@@ -906,7 +907,7 @@ func (r *Runner) RunEnumeration() {
 				continue
 			}
 
-			if r.options.FilterOutDuplicates && r.duplicate(resp.Response.Data) {
+			if r.options.FilterOutDuplicates && r.duplicate(&resp) {
 				continue
 			}