First working version

README.md

@@ -1,4 +1,45 @@
-# Mini-Scan
+# Mini-Scan Submission
+
+A submission of the mini-scan take home assignment described below.  
+
+--
+
+The solution was implemented using an sqlite3 database. 
+
+One challenging aspect of the project was an issue with running sqlite3 in docker with a CGO_ENABLED=0.  
+
+To overcome this obstacle, a base image that supports CGO_ENABLED=1 (frolvlad/alpine-glibc) was used. 
+
+Next steps include:
+
+1) Switching to a PostgreSQL database.
+2) Handling failures with a DLQ and/or passing the timestamp in the message to make sure the most recent update is saved.
+3) Pushing to the cloud to test horizontal scaling which the emulater does not support.
+4) Actually scan the local network.
+
+---
+
+To test changes to `cmd/proessor/main.go` by examining the processor's collection of scan results run 
+
+`docker system prune --all --volumes --force`
+
+`docker compose up --build`  # this runs unit tests 
+
+`docker-compose run --entrypoint /bin/sh processor`
+
+`sqlite3 /data/processor.db`
+
+`select * from scans`
+
+
+Unit tests are available in cmd/processor and pkg/scanning for your enjoyment. 
+
+Thank You and have a great day!
+
+
+----- 
+
+# Original Assignment
 
 Hello!
 

cmd/processor/Dockerfile

@@ -0,0 +1,18 @@
+FROM golang:1.20 AS builder
+
+# Build
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download && go mod verify
+COPY . .
+CMD ["go", "test", "-v", "./..."]
+CMD ["go", "test", "-v", "pkg/scanning/..."]
+
+RUN CGO_ENABLED=1 go build -o processor ./cmd/processor
+
+FROM frolvlad/alpine-glibc
+RUN apk update && apk add --no-cache sqlite sqlite-dev
+WORKDIR app
+COPY --from=builder /src/processor .
+EXPOSE 8080
+CMD ["/app/processor"]

cmd/processor/main.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+    "context"
+    "encoding/json"
+    "flag"
+    "time"
+
+    "database/sql"
+    _ "github.com/mattn/go-sqlite3"
+
+    "cloud.google.com/go/pubsub"
+    "github.com/censys/scan-takehome/pkg/scanning"
+
+)
+
+var (
+    services = []string{"HTTP", "SSH", "DNS"} // conider sharing with scanner and performing validation here
+    dbFile = flag.String("db", "/data/processor.db", "Database File")
+)
+
+
+// Interfaces to allow for mocking
+// Consider using gomock to use *sql.DB instead of DBInterface and 
+// *pubsub.Subscription instead of SubscriptionInterface
+
+type DBInterface interface {
+	Exec(query string, args ...interface{}) (sql.Result, error)
+}
+type SubscriptionInterface interface {
+	Receive(ctx context.Context, f func(ctx context.Context, msg *pubsub.Message)) error
+}
+
+
+// main loop that recieves messsges and inserts/updates them in database
+
+func main_loop(ctx context.Context, subscription SubscriptionInterface, db DBInterface) {
+	err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) {
+        var scan scanning.Scan
+        if err := json.Unmarshal(msg.Data, &scan); err != nil {
+            panic(err)
+        }
+
+        var serviceResp string
+
+        switch scan.DataVersion {
+        case scanning.V1:
+            serviceResp = scan.ParseV1Data()
+        case scanning.V2:
+            serviceResp = scan.ParseV2Data()
+        default:
+            panic("Invalid DataVersion in scan")
+        }
+
+        // consider including timestamp in scan so retried messages don't overwrite new ones
+		_, err := db.Exec("INSERT OR REPLACE INTO scans (ip, port, service, response, scan_time) VALUES (?, ?, ?, ?, ?)", 
+							scan.Ip, scan.Port, scan.Service, serviceResp, time.Now()) 
+
+        if err != nil {
+            panic("Database insert failed") 
+        }
+
+        msg.Ack()
+    })
+
+    if err != nil {
+        panic(err)
+    }
+
+}
+
+func main() {
+    projectId := flag.String("project", "test-project", "GCP Project ID")
+    subscriptionId := flag.String("subscription", "scan-sub", "GCP PubSub Subscription ID")
+    flag.Parse()
+
+    ctx := context.Background()
+
+    client, err := pubsub.NewClient(ctx, *projectId)
+    if err != nil {
+        panic(err)
+    }
+
+    subscription := client.Subscription(*subscriptionId)
+
+    db, err := sql.Open("sqlite3", *dbFile)
+    if err != nil {
+        panic(err)
+    }
+
+    createTableSQL := `CREATE TABLE IF NOT EXISTS scans (
+        ip TEXT NOT NULL,  
+        port INTEGER NOT NULL,  
+        service TEXT NOT NULL,  
+        response TEXT NOT NULL,  
+        scan_time DATETIME NOT NULL,  
+        PRIMARY KEY (ip, port, service)
+    );`
+
+    _, err = db.Exec(createTableSQL)
+    if err != nil {
+        panic(err)
+    }
+	main_loop(ctx, subscription, db)
+}

cmd/processor/main_test.go

@@ -0,0 +1,88 @@
+package main
+
+import (
+    "context"
+    "database/sql"
+    "testing"
+    "fmt"
+
+    "github.com/golang/mock/gomock"
+    "cloud.google.com/go/pubsub"
+    "github.com/stretchr/testify/assert"
+)
+
+// SubscriptionReceiver is an interface that mocks pubsub.Subscription's methods
+type SubscriptionReceiver interface {
+    Receive(ctx context.Context, f func(ctx context.Context, msg *pubsub.Message)) error
+}
+
+// MockDB is used to mock the database interactions for testing
+type MockDB struct {
+    // Mock DB connection
+    ExecFunc func(query string, args ...interface{}) (sql.Result, error)
+}
+
+func (m *MockDB) Exec(query string, args ...interface{}) (sql.Result, error) {
+    return m.ExecFunc(query, args...)
+}
+
+// MockSubscription is used to mock pubsub.Subscription for testing
+type MockSubscription struct {
+    ReceiveFunc func(ctx context.Context, handler func(ctx context.Context, msg *pubsub.Message)) error
+}
+
+func (m *MockSubscription) Receive(ctx context.Context, handler func(ctx context.Context, msg *pubsub.Message)) error {
+    return m.ReceiveFunc(ctx, handler)
+}
+
+func TestMainLoop(t *testing.T) {
+    t.Log("Starting test of main_loop")
+    // Mock context
+    ctx := context.Background()
+
+    // Create mock controller
+    mockCtrl := gomock.NewController(t)
+    defer mockCtrl.Finish()
+
+    // Mock database interaction
+    mockDB := &MockDB{
+        ExecFunc: func(query string, args ...interface{}) (sql.Result, error) {
+            t.Logf("In MockDB insert %s, %s", query, args)
+            // Simulate a successful database insert
+            if query == "INSERT OR REPLACE INTO scans (ip, port, service, response, scan_time) VALUES (?, ?, ?, ?, ?)" &&
+				args[3] != "abdul" {
+                t.Log("Succesful Insert")
+                return nil, nil
+            }
+            t.Log("In MockDB error condition")
+            return nil, fmt.Errorf("database insert failed")
+        },
+    }
+
+    // Mock Pub/Sub subscription
+    mockSub := &MockSubscription{
+        ReceiveFunc: func(ctx context.Context, handler func(ctx context.Context, msg *pubsub.Message)) error {
+            // Create a mock Pub/Sub message
+            mockMsg1 := &pubsub.Message{
+                Data: []byte(`{"ip":"192.168.1.1","port":80,"service":"HTTP","data_version":1,"data":{"response_bytes_utf8":[97,98,99,100]}}`),
+            }
+
+            // Create second mock Pub/Sub message (error case)
+            mockMsg2 := &pubsub.Message{
+                Data: []byte(`{"ip":"192.168.1.1","port":80,"service":"HTTP","data_version":2,"data":{"response_str":"abdul"}}`),
+            }
+
+            t.Log("Sending first message")
+            handler(ctx, mockMsg1) // Simulate receiving message 1
+            t.Log("Sending second message")
+            handler(ctx, mockMsg2) // Simulate receiving message 2
+
+            return nil
+        },
+    }
+
+    // Test that the first message was processed correctly and inserted into the database and the second message panics
+
+    assert.Panics(t, func() { main_loop(ctx, mockSub, mockDB) }, "Function should panic")
+
+}

cmd/scanner/main.go

@@ -19,6 +19,7 @@ var (
 func main() {
 	projectId := flag.String("project", "test-project", "GCP Project ID")
 	topicId := flag.String("topic", "scan-topic", "GCP PubSub Topic ID")
+	flag.Parse()
 
 	ctx := context.Background()
 

docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3'
 services:
 
   # Starts the P/S emulator
@@ -40,3 +39,22 @@ services:
     build:
       context: .
       dockerfile: ./cmd/scanner/Dockerfile
+
+  # Runs the "processor"
+  processor:
+    depends_on:
+      mk-subscription:
+        condition: service_completed_successfully
+    environment:
+      PUBSUB_EMULATOR_HOST: pubsub:8085
+      PUBSUB_PROJECT_ID: test-project
+      DB_PATH: /data/processor.db
+    build:
+      context: .
+      dockerfile: ./cmd/processor/Dockerfile
+    volumes:
+      - processor-data:/data
+
+volumes:
+  processor-data:
+

go.mod

@@ -2,7 +2,18 @@ module github.com/censys/scan-takehome
 
 go 1.20
 
-require cloud.google.com/go/pubsub v1.33.0
+require (
+	cloud.google.com/go/pubsub v1.33.0
+	github.com/golang/mock v1.6.0
+	github.com/matryer/is v1.4.1
+	github.com/stretchr/testify v1.10.0
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
 
 require (
 	cloud.google.com/go v0.110.2 // indirect
@@ -15,6 +26,7 @@ require (
 	github.com/google/s2a-go v0.1.4 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
 	github.com/googleapis/gax-go/v2 v2.11.0 // indirect
+	github.com/mattn/go-sqlite3 v1.14.20
 	go.opencensus.io v0.24.0 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/net v0.17.0 // indirect