diff --git a/new_images.yml b/new_images.yml index 595a0ff..f1f47b9 100644 --- a/new_images.yml +++ b/new_images.yml @@ -1,7 +1,7 @@ --- new_images: - - spark: "3.0.0" + - spark: "2.4.6" use-case: "processing" processors: ["cpu"] python: ["py37"] - sm_version: "1.1" + sm_version: "1.3" diff --git a/requirements.txt b/requirements.txt index 76fc69d..9834fff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,5 @@ waitress==1.4.4 requests==2.24.0 rsa==4.3 pyasn1==0.4.8 -boto3==1.14.58 +boto3==1.17.14 cryptography==3.3 \ No newline at end of file diff --git a/test/integration/local/test_multinode_container.py b/test/integration/local/test_multinode_container.py index d70143a..4c63857 100644 --- a/test/integration/local/test_multinode_container.py +++ b/test/integration/local/test_multinode_container.py @@ -75,12 +75,12 @@ def test_pyspark_multinode(input_data: str, output_data: str, verbose_opt: str) def test_scala_spark_multinode(input_data: str, output_data: str, verbose_opt: str) -> None: input = "--input {}".format(input_data) output = "--output {}".format(output_data) - host_jars_dir = "./test/resources/code/scala/hello-scala-spark/lib_managed/jars/org.json4s/json4s-native_2.12" + host_jars_dir = "./test/resources/code/scala/hello-scala-spark/lib_managed/jars/org.json4s/json4s-native_2.11" container_jars_dir = "/opt/ml/processing/input/jars" jars_mount = f"{host_jars_dir}:{container_jars_dir}" jars_arg = f"--jars {container_jars_dir}" class_arg = "--class com.amazonaws.sagemaker.spark.test.HelloScalaSparkApp" - app_jar = "/opt/ml/processing/input/code/scala/hello-scala-spark/target/scala-2.12/hello-scala-spark_2.12-1.0.jar" + app_jar = "/opt/ml/processing/input/code/scala/hello-scala-spark/target/scala-2.11/hello-scala-spark_2.11-1.0.jar" docker_compose_cmd = ( f"JARS_MOUNT={jars_mount} " f"CMD='{jars_arg} {class_arg} {verbose_opt} {app_jar} {input} {output}' " diff --git a/test/integration/sagemaker/test_spark.py b/test/integration/sagemaker/test_spark.py index abfc176..fa938a7 100644 --- a/test/integration/sagemaker/test_spark.py +++ b/test/integration/sagemaker/test_spark.py @@ -298,10 +298,10 @@ def test_sagemaker_scala_jar_multinode(role, image_uri, configuration, sagemaker scala_project_dir = "test/resources/code/scala/hello-scala-spark" spark.run( - submit_app="{}/target/scala-2.12/hello-scala-spark_2.12-1.0.jar".format(scala_project_dir), + submit_app="{}/target/scala-2.11/hello-scala-spark_2.11-1.0.jar".format(scala_project_dir), submit_class="com.amazonaws.sagemaker.spark.test.HelloScalaSparkApp", submit_jars=[ - "{}/lib_managed/jars/org.json4s/json4s-native_2.12/json4s-native_2.12-3.6.9.jar".format(scala_project_dir) + "{}/lib_managed/jars/org.json4s/json4s-native_2.11/json4s-native_2.11-3.6.9.jar".format(scala_project_dir) ], arguments=["--input", input_data_uri, "--output", output_data_uri], configuration=configuration, diff --git a/test/resources/code/scala/hello-scala-spark/hello-scala-spark.sbt b/test/resources/code/scala/hello-scala-spark/hello-scala-spark.sbt index 1ba7d17..7b44959 100644 --- a/test/resources/code/scala/hello-scala-spark/hello-scala-spark.sbt +++ b/test/resources/code/scala/hello-scala-spark/hello-scala-spark.sbt @@ -1,9 +1,9 @@ name := "hello-scala-spark" version := "1.0" -scalaVersion := "2.12.12" +scalaVersion := "2.11.12" useCoursier := false retrieveManaged := true -libraryDependencies += "org.apache.spark" %% "spark-sql" % "3.0.0" +libraryDependencies += "org.apache.spark" %% "spark-sql" % "2.4.4" libraryDependencies += "org.json4s" %% "json4s-native" % "3.6.9" mainClass in (Compile, packageBin) := Some("HelloScalaSparkApp") mainClass in (Compile, run) := Some("HelloScalaSparkApp")