update spark submit test script (#21)

examples/extender.yml

@@ -1,18 +1,23 @@
 apiVersion: v1
+kind: Namespace
+metadata:
+  name: spark
+---
+apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: spark-scheduler
-  namespace: kube-system
+  namespace: spark
 ---
   apiVersion: rbac.authorization.k8s.io/v1
   kind: ClusterRoleBinding
   metadata:
     name: spark-scheduler-admin-binding
-    namespace: kube-system
+    namespace: spark
   subjects:
     - kind: ServiceAccount
       name: spark-scheduler
-      namespace: kube-system
+      namespace: default
   roleRef:
     apiGroup: rbac.authorization.k8s.io
     kind: ClusterRole
@@ -22,7 +27,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: spark-scheduler
-  namespace: kube-system
+  namespace: spark
 data:
   policy.cfg : |
     {
@@ -55,7 +60,7 @@ data:
       policy:
         configMap:
           name: spark-scheduler
-          namespace: kube-system
+          namespace: spark
     leaderElection:
       lockObjectName: spark-scheduler
   spark-scheduler-extender-config: |
@@ -72,7 +77,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: spark-scheduler
-  namespace: kube-system
+  namespace: spark
   labels:
     app: spark-scheduler
 spec:
@@ -102,7 +107,7 @@ spec:
         - name: kube-scheduler-config-volume
           mountPath: /etc/config
       - name: spark-scheduler-extender
-        image: palantir/spark-scheduler:latest
+        image: palantirtechnologies/spark-scheduler:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
           httpGet:

examples/submit-test-spark-app.sh

@@ -34,6 +34,10 @@ spec:
     image: nginx
     ports:
     - containerPort: 80
+    resources:
+      requests:
+        cpu: "$DRIVER_CPU"
+        memory: "$DRIVER_MEM"
 EOF)
 
 # wait for driver to be running
@@ -64,6 +68,10 @@ spec:
     image: nginx
     ports:
     - containerPort: 80
+    resources:
+      requests:
+        cpu: "$EXECUTOR_CPU"
+        memory: "$EXECUTOR_MEM"
 EOF)
 done
 

hack/dev/spark-submit-test.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+EXECUTOR_COUNT=$1
+DRIVER_CPU=$2
+DRIVER_MEM=$3 # in mb
+EXECUTOR_CPU=$4
+EXECUTOR_MEM=$5 # in mb
+
+TEMPLATE=$(cat <<EOF
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    spark-app-id: $RANDOM
+spec:
+  schedulerName: spark-scheduler
+EOF
+)
+
+
+$SPARK_HOME/bin/spark-submit \
+  --master k8s://https://localhost:6443 \
+  --deploy-mode cluster \
+  --name spark-test \
+  --class "org.apache.spark.examples.DriverAndExecutorsSubmissionTest" \
+  --conf "spark.kubernetes.container.image=spark:test" \
+  --conf "spark.kubernetes.driver.podTemplateFile="<(echo "$TEMPLATE") \
+  --conf "spark.kubernetes.executor.podTemplateFile="<(echo "$TEMPLATE") \
+  --conf "spark.executor.instances=$EXECUTOR_COUNT" \
+  --conf "spark.driver.cores=$DRIVER_CPU" \
+  --conf "spark.driver.memory=$DRIVER_MEM"m \
+  --conf "spark.executor.cores=$EXECUTOR_CPU" \
+  --conf "spark.executor.memory=$EXECUTOR_MEM"m \
+  --conf "spark.kubernetes.driver.annotation.spark-executor-count=$EXECUTOR_COUNT" \
+  --conf "spark.kubernetes.driver.annotation.spark-driver-cpu=$DRIVER_CPU" \
+  --conf "spark.kubernetes.driver.annotation.spark-driver-mem=$DRIVER_MEM"M \
+  --conf "spark.kubernetes.driver.annotation.spark-executor-cpu=$EXECUTOR_CPU" \
+  --conf "spark.kubernetes.driver.annotation.spark-executor-mem=$EXECUTOR_MEM"M \
+  local:///opt/spark/examples/jars/spark-examples_2.12-3.0.0-SNAPSHOT.jar \
+  60