new 3.0 container with sec patch

new_images.yml

@@ -1,7 +1,7 @@
 ---
 new_images:
-  - spark: "3.0.0"
+  - spark: "2.4.6"
     use-case: "processing"
     processors: ["cpu"]
     python: ["py37"]
-    sm_version: "1.1"
+    sm_version: "1.3"

requirements.txt

@@ -6,5 +6,5 @@ waitress==1.4.4
 requests==2.24.0
 rsa==4.3
 pyasn1==0.4.8
-boto3==1.14.58
+boto3==1.17.14
 cryptography==3.3

test/integration/local/test_multinode_container.py

@@ -75,12 +75,12 @@ def test_pyspark_multinode(input_data: str, output_data: str, verbose_opt: str)
 def test_scala_spark_multinode(input_data: str, output_data: str, verbose_opt: str) -> None:
     input = "--input {}".format(input_data)
     output = "--output {}".format(output_data)
-    host_jars_dir = "./test/resources/code/scala/hello-scala-spark/lib_managed/jars/org.json4s/json4s-native_2.12"
+    host_jars_dir = "./test/resources/code/scala/hello-scala-spark/lib_managed/jars/org.json4s/json4s-native_2.11"
     container_jars_dir = "/opt/ml/processing/input/jars"
     jars_mount = f"{host_jars_dir}:{container_jars_dir}"
     jars_arg = f"--jars {container_jars_dir}"
     class_arg = "--class com.amazonaws.sagemaker.spark.test.HelloScalaSparkApp"
-    app_jar = "/opt/ml/processing/input/code/scala/hello-scala-spark/target/scala-2.12/hello-scala-spark_2.12-1.0.jar"
+    app_jar = "/opt/ml/processing/input/code/scala/hello-scala-spark/target/scala-2.11/hello-scala-spark_2.11-1.0.jar"
     docker_compose_cmd = (
         f"JARS_MOUNT={jars_mount} "
         f"CMD='{jars_arg} {class_arg} {verbose_opt} {app_jar} {input} {output}' "

test/integration/sagemaker/test_spark.py

@@ -298,10 +298,10 @@ def test_sagemaker_scala_jar_multinode(role, image_uri, configuration, sagemaker
 
     scala_project_dir = "test/resources/code/scala/hello-scala-spark"
     spark.run(
-        submit_app="{}/target/scala-2.12/hello-scala-spark_2.12-1.0.jar".format(scala_project_dir),
+        submit_app="{}/target/scala-2.11/hello-scala-spark_2.11-1.0.jar".format(scala_project_dir),
         submit_class="com.amazonaws.sagemaker.spark.test.HelloScalaSparkApp",
         submit_jars=[
-            "{}/lib_managed/jars/org.json4s/json4s-native_2.12/json4s-native_2.12-3.6.9.jar".format(scala_project_dir)
+            "{}/lib_managed/jars/org.json4s/json4s-native_2.11/json4s-native_2.11-3.6.9.jar".format(scala_project_dir)
         ],
         arguments=["--input", input_data_uri, "--output", output_data_uri],
         configuration=configuration,

test/resources/code/scala/hello-scala-spark/hello-scala-spark.sbt

@@ -1,9 +1,9 @@
 name := "hello-scala-spark"
 version := "1.0"
-scalaVersion := "2.12.12"
+scalaVersion := "2.11.12"
 useCoursier := false
 retrieveManaged := true
-libraryDependencies += "org.apache.spark" %% "spark-sql" % "3.0.0"
+libraryDependencies += "org.apache.spark" %% "spark-sql" % "2.4.4"
 libraryDependencies += "org.json4s" %% "json4s-native" % "3.6.9"
 mainClass in (Compile, packageBin) := Some("HelloScalaSparkApp")
 mainClass in (Compile, run) := Some("HelloScalaSparkApp")