Recipe: Splunk

main.tf

@@ -31,3 +31,9 @@ output "bastion-public_dns" {
 output "bastion-public_ip" {
   value = "${module.openshift.bastion-public_ip}"
 }
+output "splunk-private_ip" {
+  value = "${module.openshift.splunk-private_ip}"
+}
+output "splunk-console-url" {
+  value = "http://${module.openshift.splunk-public_dns}:8000"
+}

makefile

@@ -25,6 +25,8 @@ openshift:
 # Open the console.
 browse-openshift:
 	open $$(terraform output master-url)
+browse-splunk:
+	open $$(terraform output splunk-console-url)
 
 # SSH onto the master.
 ssh-bastion:
@@ -42,4 +44,9 @@ sample:
 	oc new-project sample
 	oc process -f ./sample/counter-service.yml | oc create -f - 
 
+# Setup splunk.
+splunk:
+	cat ./recipes/splunk/setup-cluster.sh | ssh -A ec2-user@$$(terraform output bastion-public_dns) ssh master.openshift.local
+	sed "s/\$${SPLUNK_FORWARD_SERVER}/$$(terraform output splunk-private_ip)/" ./recipes/splunk/splunk-forwarder.template.yml | ssh -A ec2-user@$$(terraform output bastion-public_dns) ssh master.openshift.local oc create -f -
+
 .PHONY: sample

modules/openshift/99-outputs.tf

@@ -50,3 +50,11 @@ output "bastion-private_dns" {
 output "bastion-private_ip" {
   value = "${aws_instance.bastion.private_ip}"
 }
+
+# Output some information about the Splunk host.
+output "splunk-private_ip" {
+  value = "${aws_instance.splunk.private_ip}"
+}
+output "splunk-public_dns" {
+  value = "${aws_instance.splunk.public_dns}"
+}

modules/openshift/files/setup-splunk.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# This script template is expected to be populated during the setup of a
+# OpenShift  node. It runs on host startup.
+
+# Log everything we do.
+set -x
+exec > /var/log/user-data.log 2>&1
+
+# Create initial logs config.
+cat > ./awslogs.conf << EOF
+[general]
+state_file = /var/awslogs/state/agent-state
+
+[/var/log/messages]
+log_stream_name = openshift-splunk-{instance_id}
+log_group_name = /var/log/messages
+file = /var/log/messages
+datetime_format = %b %d %H:%M:%S
+buffer_duration = 5000
+initial_position = start_of_file
+
+[/var/log/user-data.log]
+log_stream_name = splunk-{instance_id}
+log_group_name = /var/log/user-data.log
+file = /var/log/user-data.log
+EOF
+
+# Download and run the AWS logs agent.
+curl https://s3.amazonaws.com/aws-cloudwatch/downloads/latest/awslogs-agent-setup.py -O
+python ./awslogs-agent-setup.py --non-interactive --region us-east-1 -c ./awslogs.conf
+
+# Start the awslogs service, also start on reboot.
+# Note: Errors go to /var/log/awslogs.log
+service awslogs start
+chkconfig awslogs on
+
+# Download splunk.
+aws s3 cp s3://dwmkerr-public/splunk-7.0.0-c8a78efdd40f-Linux-x86_64.tgz ./splunk.tgz
+tar xvzf splunk.tgz -C /opt
+
+# Everything else we do now is with the splunk binary.
+cd /opt/splunk/bin
+
+# Start splunk on reboot, then start splunk. Set the admin password.
+./splunk enable boot-start --accept-license
+./splunk start --accept-license
+./splunk edit user admin -password 123 -role admin -auth admin:changeme
+
+# Enable receiving of events on 9997.
+./splunk enable listen 9997 -auth admin:123

modules/openshift/recipe-splunk.tf

@@ -0,0 +1,53 @@
+# This security group allows public ingress on port 8000, which is Splunk's
+# management console port.
+resource "aws_security_group" "splunk-public-management-ingress" {
+  name        = "openshift-public-management-"
+  description = "Security group that allows ingress on port 8000"
+  vpc_id      = "${aws_vpc.openshift.id}"
+
+  ingress {
+    from_port   = 8000
+    to_port     = 8000
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags {
+    Name    = "OpenShift Splunk Management Public Access"
+    Project = "openshift"
+  }
+}
+
+//  Create the userdata script.
+data "template_file" "setup-splunk" {
+  template = "${file("${path.module}/files/setup-splunk.sh")}"
+
+  //  Currently, no vars needed.
+}
+
+resource "aws_instance" "splunk" {
+  ami                  = "${data.aws_ami.amazonlinux.id}"
+  instance_type        = "t2.medium"
+  subnet_id            = "${aws_subnet.public-subnet.id}"
+  # The profile below provides access to cloudwatch.
+  iam_instance_profile = "${aws_iam_instance_profile.openshift-instance-profile.id}"
+  user_data            = "${data.template_file.setup-splunk.rendered}"
+
+  security_groups = [
+    "${aws_security_group.openshift-vpc.id}",
+    "${aws_security_group.openshift-public-egress.id}",
+    "${aws_security_group.splunk-public-management-ingress.id}",
+  ]
+
+  # Give ourselves a bit more space...
+  root_block_device {
+    volume_size = 50
+  }
+
+  key_name = "${aws_key_pair.keypair.key_name}"
+
+  tags {
+    Name    = "OpenShift Splunk"
+    Project = "openshift"
+  }
+}

recipes/splunk/setup-cluster.sh

@@ -0,0 +1,5 @@
+# Create the forwarder service account.
+# Add priviledges to mount volumes and run as root.
+oc create sa splunk-forwarder
+oadm policy add-scc-to-user anyuid system:serviceaccount:default:splunk-forwarder
+oadm policy add-scc-to-user privileged system:serviceaccount:default:splunk-forwarder

recipes/splunk/splunk-forwarder.template.yml

@@ -0,0 +1,69 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: splunk-forwarder
+spec:
+  selector:
+      matchLabels:
+        name: splunk-forwarder
+  template:
+    metadata:
+      labels:
+        name: splunk-forwarder
+    spec:
+      serviceAccountName: splunk-forwarder
+      restartPolicy: Always
+      nodeSelector:
+        region: primary
+      containers:
+      - name: splunk-forwarder
+        image: splunk/universalforwarder:6.5.2-monitor
+        securityContext:
+          privileged: true
+        env:
+        - name: SPLUNK_START_ARGS
+          value: "--accept-license --answer-yes"
+        - name: SPLUNK_USER
+          value: root
+        - name: SPLUNK_FORWARD_SERVER
+          value: "${SPLUNK_FORWARD_SERVER}:9997"
+        # Monitor the containers logs.
+        - name: SPLUNK_ADD_1
+          value: "monitor /var/log/containers"
+        volumeMounts:
+        - mountPath: /var/run/docker.sock
+          readOnly: true
+          name: docker-socket
+        - mountPath: /var/lib/docker/containers
+          readOnly: true
+          name: var-lib-docker-containers
+        - mountPath: /var/log/containers
+          readOnly: true
+          name: var-log-containers
+        - mountPath: /var/log/pods
+          readOnly: true
+          name: var-log-pods
+        # Having some weird issues with chown permissions here, skipping for now...
+        #- mountPath: /opt/splunk/etc
+        #  name: opt-splunk-etc
+        # - mountPath: /opt/splunk/var
+        #  name: opt-splunk-var
+      volumes:
+        - name: docker-socket
+          hostPath:
+            path: /var/run/docker.sock
+        - name: var-lib-docker-containers
+          hostPath:
+            path: /var/lib/docker/containers
+        - name: var-log-containers
+          hostPath:
+            path: /var/log/containers
+        - name: var-log-pods
+          hostPath:
+            path: /var/log/pods
+        - name: opt-splunk-etc
+          hostPath:
+            path: /opt/splunk/etc
+        - name: opt-splunk-var
+          hostPath:
+            path: /opt/splunk/var