Add a proper --all-nameservers implementation (#485)

* add domain name to root servers default

* start the all nameservers, for a single layer

* working for google.com

* example.com working

* external lookups working

* gate --all-nameservers for all specialty CLI modules and add comments

* Clarify behavior in --help for all-nameservers and --name-servers

* handle endless loop with next layer

* add integration test

* fixed up comments

* fixed up comments

* added context to function defs

* First unit test passing

* added another unit test, 2 .com NSes and one fails

* fixed up another test

* handle the extra sibling NS case that issue 352 brings up

* remove comment

* bug fixes, query for all leaf NSes first, then followup with original query type

* lint/tests

* integration tests

* fix over-zealous replace operation

* sort for consistent unit test

* add timeout error to all-nameservers

* handle de-duplicating nameservers by name, ensureing we only query one IPv4/v6 addr for a given nameserver

* PR cleanup

* don't error for CNAME referrals, we won't implement this

* cleanup

* up timeouts on integration tests

examples/all_nameservers_lookup/main.go

@@ -0,0 +1,80 @@
+/* ZDNS Copyright 2024 Regents of the University of Michigan
+*
+* Licensed under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License. You may obtain a copy
+* of the License at http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+* implied. See the License for the specific language governing
+* permissions and limitations under the License.
+ */
+
+package main
+
+import (
+	"net"
+	"time"
+
+	"github.com/miekg/dns"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/zmap/zdns/examples/utils"
+	"github.com/zmap/zdns/src/zdns"
+)
+
+func main() {
+	// Perform the lookup
+	domain := "google.com"
+	dnsQuestion := &zdns.Question{Name: domain, Type: dns.TypeA, Class: dns.ClassINET}
+	resolver := initializeResolver()
+	// LookupAllNameserversIterative will query all root nameservers, and then all TLD nameservers, and then all authoritative nameservers for the domain.
+	result, _, status, err := resolver.LookupAllNameserversIterative(dnsQuestion, nil)
+	if err != nil {
+		log.Fatal("Error looking up domain: ", err)
+	}
+	log.Warnf("Result: %v", result)
+	log.Warnf("Status: %v", status)
+	log.Info("We can also specify which root nameservers to use by setting the argument.")
+
+	result, _, status, err = resolver.LookupAllNameserversIterative(dnsQuestion, []zdns.NameServer{{IP: net.ParseIP("198.41.0.4"), Port: 53}}) // a.root-servers.net
+	if err != nil {
+		log.Fatal("Error looking up domain: ", err)
+	}
+	log.Warnf("Result: %v", result)
+	log.Warnf("Status: %v", status)
+
+	log.Info("You can query multiple recursive resolvers as well")
+
+	externalResult, _, status, err := resolver.LookupAllNameserversExternal(dnsQuestion, []zdns.NameServer{{IP: net.ParseIP("1.1.1.1"), Port: 53}, {IP: net.ParseIP("8.8.8.8"), Port: 53}}) // Cloudflare and Google recursive resolvers, respectively
+	if err != nil {
+		log.Fatal("Error looking up domain: ", err)
+	}
+	log.Warnf("Result: %v", externalResult)
+	log.Warnf("Status: %v", status)
+	resolver.Close()
+}
+
+func initializeResolver() *zdns.Resolver {
+	localAddr, err := utils.GetLocalIPByConnecting()
+	if err != nil {
+		log.Fatal("Error getting local IP: ", err)
+	}
+	// Create a ResolverConfig object
+	resolverConfig := zdns.NewResolverConfig()
+	// Set any desired options on the ResolverConfig object
+	resolverConfig.LogLevel = log.InfoLevel
+	resolverConfig.LocalAddrsV4 = []net.IP{localAddr}
+	resolverConfig.ExternalNameServersV4 = []zdns.NameServer{{IP: net.ParseIP("1.1.1.1"), Port: 53}}
+	resolverConfig.RootNameServersV4 = zdns.RootServersV4
+	resolverConfig.IPVersionMode = zdns.IPv4Only
+	resolverConfig.Timeout = time.Minute
+	resolverConfig.IterativeTimeout = time.Minute
+	// Create a new Resolver object with the ResolverConfig object, it will retain all settings set on the ResolverConfig object
+	resolver, err := zdns.InitResolver(resolverConfig)
+	if err != nil {
+		log.Fatal("Error initializing resolver: ", err)
+	}
+	return resolver
+}

examples/multi_thread_lookup/multi_threaded.go

@@ -14,6 +14,7 @@
 package main
 
 import (
+	"context"
 	"net"
 	"sync"
 
@@ -44,15 +45,15 @@ func main() {
 	wg.Add(2)
 	go func() {
 		defer wg.Done()
-		result1, _, _, err1 := resolver1.IterativeLookup(dnsQuestion1)
+		result1, _, _, err1 := resolver1.IterativeLookup(context.Background(), dnsQuestion1)
 		if err1 != nil {
 			log.Fatal("Error looking up domain: ", err1)
 		}
 		log.Warnf("Result: %v", result1)
 	}()
 	go func() {
 		defer wg.Done()
-		result2, _, _, err2 := resolver2.IterativeLookup(dnsQuestion2)
+		result2, _, _, err2 := resolver2.IterativeLookup(context.Background(), dnsQuestion2)
 		if err2 != nil {
 			log.Fatal("Error looking up domain: ", err2)
 		}

examples/single_lookup/simple.go

@@ -14,6 +14,7 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"net"
 
@@ -30,7 +31,7 @@ func main() {
 	dnsQuestion := &zdns.Question{Name: domain, Type: dns.TypeA, Class: dns.ClassINET}
 	resolver := initializeResolver()
 
-	result, _, status, err := resolver.ExternalLookup(dnsQuestion, &zdns.NameServer{IP: net.ParseIP("1.1.1.1"), Port: 53})
+	result, _, status, err := resolver.ExternalLookup(context.Background(), dnsQuestion, &zdns.NameServer{IP: net.ParseIP("1.1.1.1"), Port: 53})
 	if err != nil {
 		log.Fatal("Error looking up domain: ", err)
 	}
@@ -44,7 +45,7 @@ func main() {
 
 	log.Warn("\n\n This lookup just used the Cloudflare recursive resolver, let's run our own recursion.")
 	// Iterative Lookups start at the root nameservers and follow the chain of referrals to the authoritative nameservers.
-	result, trace, status, err := resolver.IterativeLookup(&zdns.Question{Name: domain, Type: dns.TypeA, Class: dns.ClassINET})
+	result, trace, status, err := resolver.IterativeLookup(context.Background(), &zdns.Question{Name: domain, Type: dns.TypeA, Class: dns.ClassINET})
 	if err != nil {
 		log.Fatal("Error looking up domain: ", err)
 	}

src/cli/cli.go

@@ -44,14 +44,14 @@ type StatusHandler interface {
 // GeneralOptions core options for all ZDNS modules
 // Order here is the order they'll be printed to the user, so preserve alphabetical order
 type GeneralOptions struct {
-	LookupAllNameServers bool   `long:"all-nameservers" description:"Perform the lookup via all the nameservers for the domain."`
+	LookupAllNameServers bool   `long:"all-nameservers" description:"Behavior is dependent on --iterative. In --iterative, --all-name-servers will query all root servers, then all gtld servers, etc. recording the responses at each layer. In non-iterative mode, the query will be sent to all external resolvers specified in --name-servers."`
 	CacheSize            int    `long:"cache-size" default:"10000" description:"how many items can be stored in internal recursive cache"`
 	GoMaxProcs           int    `long:"go-processes" default:"0" description:"number of OS processes (GOMAXPROCS by default)"`
 	IterationTimeout     int    `long:"iteration-timeout" default:"8" description:"timeout for a single iterative step in an iterative query, in seconds. Only applicable with --iterative"`
 	IterativeResolution  bool   `long:"iterative" description:"Perform own iteration instead of relying on recursive resolver"`
 	MaxDepth             int    `long:"max-depth" default:"10" description:"how deep should we recurse when performing iterative lookups"`
 	NameServerMode       bool   `long:"name-server-mode" description:"Treats input as nameservers to query with a static query rather than queries to send to a static name server"`
-	NameServersString    string `long:"name-servers" description:"List of DNS servers to use. Can be passed as comma-delimited string or via @/path/to/file. If no port is specified, defaults to 53."`
+	NameServersString    string `long:"name-servers" description:"List of DNS servers to use. Can be passed as comma-delimited string or via @/path/to/file. If no port is specified, defaults to 53. If not provided, defaults to either the default root servers in --iterative or the recursive resolvers specified in /etc/resolv.conf or OS equivalent."`
 	UseNanoseconds       bool   `long:"nanoseconds" description:"Use nanosecond resolution timestamps in output"`
 	NetworkTimeout       int    `long:"network-timeout" default:"2" description:"timeout for round trip network operations, in seconds"`
 	DisableFollowCNAMEs  bool   `long:"no-follow-cnames" description:"do not follow CNAMEs/DNAMEs in the lookup process"`

src/cli/config_validation.go

@@ -128,10 +128,6 @@ func validateClientSubnetString(gc *CLIConf) error {
 }
 
 func parseNameServers(gc *CLIConf) error {
-	if gc.LookupAllNameServers && gc.NameServersString != "" {
-		log.Fatal("name servers cannot be specified in --all-nameservers mode.")
-	}
-
 	if gc.NameServersString != "" {
 		if gc.NameServerMode {
 			log.Fatal("name servers cannot be specified on command line in --name-server-mode")

src/cli/modules.go

@@ -14,6 +14,7 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/miekg/dns"
@@ -165,14 +166,23 @@ func (lm *BasicLookupModule) NewFlags() interface{} {
 	return lm
 }
 
+// Lookup performs a DNS lookup using the given resolver and lookupName.
+// The behavior with respect to the nameServers is determined by the LookupAllNameServers and IsIterative fields.
+// non-Iterative + all-Nameservers query -> we'll send a query to each of the resolver's external nameservers
+// non-Iterative query -> we'll send a query to the nameserver provided. If none provided, a random nameserver from the resolver's external nameservers will be used
+// iterative + all-Nameservers query -> we'll send a query to each root NS and query all nameservers down the chain.
+// iterative query -> we'll send a query to a random root NS and query all nameservers down the chain.
 func (lm *BasicLookupModule) Lookup(resolver *zdns.Resolver, lookupName string, nameServer *zdns.NameServer) (interface{}, zdns.Trace, zdns.Status, error) {
+	if lm.LookupAllNameServers && lm.IsIterative {
+		return resolver.LookupAllNameserversIterative(&zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass}, nil)
+	}
 	if lm.LookupAllNameServers {
-		return resolver.LookupAllNameservers(&zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass}, nameServer)
+		return resolver.LookupAllNameserversExternal(&zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass}, nil)
 	}
 	if lm.IsIterative {
-		return resolver.IterativeLookup(&zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass})
+		return resolver.IterativeLookup(context.Background(), &zdns.Question{Name: lookupName, Type: lm.DNSType, Class: lm.DNSClass})
 	}
-	return resolver.ExternalLookup(&zdns.Question{Type: lm.DNSType, Class: lm.DNSClass, Name: lookupName}, nameServer)
+	return resolver.ExternalLookup(context.Background(), &zdns.Question{Type: lm.DNSType, Class: lm.DNSClass, Name: lookupName}, nameServer)
 }
 
 func GetLookupModule(name string) (LookupModule, error) {

src/internal/util/util.go

@@ -57,9 +57,9 @@ func IsStringValidDomainName(domain string) bool {
 }
 
 // HasCtxExpired checks if the context has expired. Common function used in various places.
-func HasCtxExpired(ctx *context.Context) bool {
+func HasCtxExpired(ctx context.Context) bool {
 	select {
-	case <-(*ctx).Done():
+	case <-(ctx).Done():
 		return true
 	default:
 		return false

src/modules/alookup/a_lookup.go

@@ -34,6 +34,9 @@ func init() {
 
 // CLIInit initializes the ALookupModule with the given parameters, used to call ALOOKUP from the command line
 func (aMod *ALookupModule) CLIInit(gc *cli.CLIConf, resolverConfig *zdns.ResolverConfig) error {
+	if gc.LookupAllNameServers {
+		return errors.New("ALOOKUP module does not support --all-nameservers")
+	}
 	aMod.Init(aMod.IPv4Lookup, aMod.IPv6Lookup)
 	err := aMod.baseModule.CLIInit(gc, resolverConfig)
 	if err != nil {

src/modules/axfr/axfr.go

@@ -151,6 +151,9 @@ func (axfrMod *AxfrLookupModule) CLIInit(gc *cli.CLIConf, rc *zdns.ResolverConfi
 	if gc.IterativeResolution {
 		log.Fatal("AXFR module does not support iterative resolution")
 	}
+	if gc.LookupAllNameServers {
+		return errors.New("AXFR module does not support --all-nameservers")
+	}
 	var err error
 	if axfrMod.BlacklistPath != "" {
 		axfrMod.Blacklist = safeblacklist.New()

src/modules/bindversion/bindversion.go

@@ -15,7 +15,10 @@
 package bindversion
 
 import (
+	"context"
+
 	"github.com/miekg/dns"
+	"github.com/pkg/errors"
 
 	"github.com/zmap/zdns/src/cli"
 	"github.com/zmap/zdns/src/zdns"
@@ -42,6 +45,9 @@ func init() {
 
 // CLIInit initializes the BindVersion lookup module
 func (bindVersionMod *BindVersionLookupModule) CLIInit(gc *cli.CLIConf, rc *zdns.ResolverConfig) error {
+	if gc.LookupAllNameServers {
+		return errors.New("AXFR module does not support --all-nameservers")
+	}
 	return bindVersionMod.BasicLookupModule.CLIInit(gc, rc)
 }
 
@@ -51,9 +57,9 @@ func (bindVersionMod *BindVersionLookupModule) Lookup(r *zdns.Resolver, lookupNa
 	var status zdns.Status
 	var err error
 	if bindVersionMod.IsIterative {
-		innerRes, trace, status, err = r.IterativeLookup(&zdns.Question{Name: BindVersionQueryName, Type: dns.TypeTXT, Class: dns.ClassCHAOS})
+		innerRes, trace, status, err = r.IterativeLookup(context.Background(), &zdns.Question{Name: BindVersionQueryName, Type: dns.TypeTXT, Class: dns.ClassCHAOS})
 	} else {
-		innerRes, trace, status, err = r.ExternalLookup(&zdns.Question{Name: BindVersionQueryName, Type: dns.TypeTXT, Class: dns.ClassCHAOS}, nameServer)
+		innerRes, trace, status, err = r.ExternalLookup(context.Background(), &zdns.Question{Name: BindVersionQueryName, Type: dns.TypeTXT, Class: dns.ClassCHAOS}, nameServer)
 	}
 	resString, resStatus, err := zdns.CheckTxtRecords(innerRes, status, nil, err)
 	res := Result{BindVersion: resString}

src/modules/bindversion/bindversion_test.go

@@ -15,6 +15,7 @@
 package bindversion
 
 import (
+	"context"
 	"net"
 	"testing"
 
@@ -35,7 +36,7 @@ var queries []QueryRecord
 // DoSingleDstServerLookup(r *Resolver, q Question, nameServer string, isIterative bool) (*SingleQueryResult, Trace, Status, error)
 type MockLookup struct{}
 
-func (ml MockLookup) DoDstServersLookup(r *zdns.Resolver, question zdns.Question, nameServers []zdns.NameServer, isIterative bool) (*zdns.SingleQueryResult, zdns.Trace, zdns.Status, error) {
+func (ml MockLookup) DoDstServersLookup(ctx context.Context, r *zdns.Resolver, question zdns.Question, nameServers []zdns.NameServer, isIterative bool) (*zdns.SingleQueryResult, zdns.Trace, zdns.Status, error) {
 	queries = append(queries, QueryRecord{q: question, NameServer: &nameServers[0]})
 	if res, ok := mockResults[question.Name]; ok {
 		return res, nil, zdns.StatusNoError, nil

src/modules/dmarc/dmarc.go

@@ -42,6 +42,9 @@ type DmarcLookupModule struct {
 
 // CLIInit initializes the DMARC lookup module
 func (dmarcMod *DmarcLookupModule) CLIInit(gc *cli.CLIConf, rc *zdns.ResolverConfig) error {
+	if gc.LookupAllNameServers {
+		return errors.New("DMARC module does not support --all-nameservers")
+	}
 	dmarcMod.re = regexp.MustCompile(dmarcPrefixRegexp)
 	dmarcMod.BasicLookupModule.DNSType = dns.TypeTXT
 	dmarcMod.BasicLookupModule.DNSClass = dns.ClassINET

src/modules/dmarc/dmarc_test.go

@@ -15,6 +15,7 @@
 package dmarc
 
 import (
+	"context"
 	"net"
 	"testing"
 
@@ -35,7 +36,7 @@ var queries []QueryRecord
 
 type MockLookup struct{}
 
-func (ml MockLookup) DoDstServersLookup(r *zdns.Resolver, question zdns.Question, nameServers []zdns.NameServer, isIterative bool) (*zdns.SingleQueryResult, zdns.Trace, zdns.Status, error) {
+func (ml MockLookup) DoDstServersLookup(ctx context.Context, r *zdns.Resolver, question zdns.Question, nameServers []zdns.NameServer, isIterative bool) (*zdns.SingleQueryResult, zdns.Trace, zdns.Status, error) {
 	queries = append(queries, QueryRecord{question, &nameServers[0]})
 	if res, ok := mockResults[question.Name]; ok {
 		return res, nil, zdns.StatusNoError, nil

src/modules/mxlookup/mx_lookup.go

@@ -14,6 +14,7 @@
 package mxlookup
 
 import (
+	"context"
 	"strings"
 
 	"github.com/miekg/dns"
@@ -56,6 +57,9 @@ type MXLookupModule struct {
 
 // CLIInit initializes the MXLookupModule with the given parameters, used to call MXLookup from the command line
 func (mxMod *MXLookupModule) CLIInit(gc *cli.CLIConf, rc *zdns.ResolverConfig) error {
+	if gc.LookupAllNameServers {
+		return errors.New("MXLOOKUP module does not support --all-nameservers")
+	}
 	if !mxMod.IPv4Lookup && !mxMod.IPv6Lookup {
 		// need to use one of the two
 		mxMod.IPv4Lookup = true
@@ -92,9 +96,9 @@ func (mxMod *MXLookupModule) Lookup(r *zdns.Resolver, lookupName string, nameSer
 	var status zdns.Status
 	var err error
 	if mxMod.BasicLookupModule.IsIterative {
-		res, trace, status, err = r.IterativeLookup(&zdns.Question{Name: lookupName, Type: dns.TypeMX, Class: dns.ClassINET})
+		res, trace, status, err = r.IterativeLookup(context.Background(), &zdns.Question{Name: lookupName, Type: dns.TypeMX, Class: dns.ClassINET})
 	} else {
-		res, trace, status, err = r.ExternalLookup(&zdns.Question{Name: lookupName, Type: dns.TypeMX, Class: dns.ClassINET}, nameServer)
+		res, trace, status, err = r.ExternalLookup(context.Background(), &zdns.Question{Name: lookupName, Type: dns.TypeMX, Class: dns.ClassINET}, nameServer)
 	}
 	if status != zdns.StatusNoError || err != nil {
 		return nil, trace, status, err

src/modules/nslookup/ns_lookup.go

@@ -37,6 +37,9 @@ type NSLookupModule struct {
 
 // CLIInit initializes the NSLookupModule with the given parameters, used to call NSLookup from the command line
 func (nsMod *NSLookupModule) CLIInit(gc *cli.CLIConf, resolverConf *zdns.ResolverConfig) error {
+	if gc.LookupAllNameServers {
+		return errors.New("NSLOOKUP module does not support --all-nameservers")
+	}
 	if !nsMod.IPv4Lookup && !nsMod.IPv6Lookup {
 		log.Debug("NSModule: neither --ipv4-lookup nor --ipv6-lookup specified, will only request A records for each NS server")
 		nsMod.IPv4Lookup = true

src/modules/spf/spf.go

@@ -42,6 +42,9 @@ type SpfLookupModule struct {
 
 // CLIInit initializes the SPF lookup module
 func (spfMod *SpfLookupModule) CLIInit(gc *cli.CLIConf, rc *zdns.ResolverConfig) error {
+	if gc.LookupAllNameServers {
+		return errors.New("SPF module does not support --all-nameservers")
+	}
 	spfMod.re = regexp.MustCompile(spfPrefixRegexp)
 	spfMod.BasicLookupModule.DNSType = dns.TypeTXT
 	spfMod.BasicLookupModule.DNSClass = dns.ClassINET

src/modules/spf/spf_test.go

@@ -15,6 +15,7 @@
 package spf
 
 import (
+	"context"
 	"net"
 	"testing"
 
@@ -35,7 +36,7 @@ var queries []QueryRecord
 
 type MockLookup struct{}
 
-func (ml MockLookup) DoDstServersLookup(r *zdns.Resolver, question zdns.Question, nameServers []zdns.NameServer, isIterative bool) (*zdns.SingleQueryResult, zdns.Trace, zdns.Status, error) {
+func (ml MockLookup) DoDstServersLookup(ctx context.Context, r *zdns.Resolver, question zdns.Question, nameServers []zdns.NameServer, isIterative bool) (*zdns.SingleQueryResult, zdns.Trace, zdns.Status, error) {
 	queries = append(queries, QueryRecord{question, &nameServers[0]})
 	if res, ok := mockResults[question.Name]; ok {
 		return res, nil, zdns.StatusNoError, nil