From 80628ea88dda195d35373fde27970266a9a8233c Mon Sep 17 00:00:00 2001 From: Enrico Deusebio Date: Fri, 6 Dec 2024 08:47:40 +0100 Subject: [PATCH 1/6] [DPE-6154] Update rockcraft.yaml recipes using newer Rockcraft features --- Makefile | 21 ++++----------------- images/charmed-spark-gpu/Dockerfile | 4 ---- images/charmed-spark-gpu/rockcraft.yaml | 1 + images/charmed-spark/Dockerfile | 4 ---- images/charmed-spark/bin/sparkd.sh | 9 --------- images/charmed-spark/rockcraft.yaml | 1 + 6 files changed, 6 insertions(+), 34 deletions(-) delete mode 100644 images/charmed-spark-gpu/Dockerfile delete mode 100644 images/charmed-spark/Dockerfile diff --git a/Makefile b/Makefile index 481683d7..5a3982d0 100644 --- a/Makefile +++ b/Makefile @@ -131,7 +131,6 @@ AZURE_MARKER=$(_MAKE_DIR)/azure.tag # The names of different flavours of the image in the docker container registry -STAGED_IMAGE_DOCKER_ALIAS=staged-charmed-spark:latest SPARK_DOCKER_ALIAS=charmed-spark:$(SPARK_VERSION) SPARK_GPU_DOCKER_ALIAS=charmed-spark-gpu:$(SPARK_VERSION) JUPYTER_DOCKER_ALIAS=charmed-spark-jupyter:$(SPARK_VERSION)-$(JUPYTER_VERSION) @@ -186,17 +185,11 @@ rock: $(ROCK_FILE) # Recipe that builds Spark image and exports it to a tarfile in the current directory -$(SPARK_MARKER): $(ROCK_FILE) images/charmed-spark/Dockerfile +$(SPARK_MARKER): $(ROCK_FILE) rockcraft.skopeo --insecure-policy \ copy \ oci-archive:"$(ROCK_FILE)" \ - docker-daemon:"$(STAGED_IMAGE_DOCKER_ALIAS)" - - docker build -t $(SPARK_DOCKER_ALIAS) \ - --build-arg BASE_IMAGE="$(STAGED_IMAGE_DOCKER_ALIAS)" \ - images/charmed-spark-gpu - - docker save $(SPARK_DOCKER_ALIAS) -o $(SPARK_ARTIFACT) + docker-daemon:"$(SPARK_DOCKER_ALIAS)" touch $(SPARK_MARKER) @@ -244,17 +237,11 @@ rock-gpu: $(ROCK_FILE_GPU) # Recipe that builds Spark GPU image and exports it to a tarfile in the current directory -$(SPARK_GPU_MARKER): $(ROCK_FILE_GPU) images/charmed-spark-gpu/Dockerfile +$(SPARK_GPU_MARKER): $(ROCK_FILE_GPU) rockcraft.skopeo --insecure-policy \ copy \ oci-archive:"$(ROCK_FILE_GPU)" \ - docker-daemon:"$(STAGED_IMAGE_DOCKER_ALIAS)" - - docker build -t $(SPARK_GPU_DOCKER_ALIAS) \ - --build-arg BASE_IMAGE="$(STAGED_IMAGE_DOCKER_ALIAS)" \ - images/charmed-spark-gpu - - docker save $(SPARK_GPU_DOCKER_ALIAS) -o $(SPARK_GPU_ARTIFACT) + docker-daemon:"$(SPARK_GPU_DOCKER_ALIAS)" touch $(SPARK_GPU_MARKER) diff --git a/images/charmed-spark-gpu/Dockerfile b/images/charmed-spark-gpu/Dockerfile deleted file mode 100644 index 5373a047..00000000 --- a/images/charmed-spark-gpu/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -ARG BASE_IMAGE=staged-charmed-spark:latest -FROM $BASE_IMAGE -# Provide Default Entrypoint for Pebble -ENTRYPOINT [ "/bin/pebble", "enter", "--verbose", "--args", "sparkd" ] \ No newline at end of file diff --git a/images/charmed-spark-gpu/rockcraft.yaml b/images/charmed-spark-gpu/rockcraft.yaml index 084f3cb1..6d67869a 100644 --- a/images/charmed-spark-gpu/rockcraft.yaml +++ b/images/charmed-spark-gpu/rockcraft.yaml @@ -19,6 +19,7 @@ platforms: amd64: run_user: _daemon_ +entrypoint-service: sparkd environment: SPARK_HOME: /opt/spark diff --git a/images/charmed-spark/Dockerfile b/images/charmed-spark/Dockerfile deleted file mode 100644 index 5373a047..00000000 --- a/images/charmed-spark/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -ARG BASE_IMAGE=staged-charmed-spark:latest -FROM $BASE_IMAGE -# Provide Default Entrypoint for Pebble -ENTRYPOINT [ "/bin/pebble", "enter", "--verbose", "--args", "sparkd" ] \ No newline at end of file diff --git a/images/charmed-spark/bin/sparkd.sh b/images/charmed-spark/bin/sparkd.sh index d143a517..de9c0617 100644 --- a/images/charmed-spark/bin/sparkd.sh +++ b/images/charmed-spark/bin/sparkd.sh @@ -24,15 +24,6 @@ function log_forwarding { fi } -function finish { - if [ $? -ne 0 ] - then - kill -1 1 - sleep 1 - fi -} -trap finish EXIT - FLAVOUR=$1 echo "Running script with ${FLAVOUR} flavour" diff --git a/images/charmed-spark/rockcraft.yaml b/images/charmed-spark/rockcraft.yaml index dd75ad8d..4edc02ec 100644 --- a/images/charmed-spark/rockcraft.yaml +++ b/images/charmed-spark/rockcraft.yaml @@ -18,6 +18,7 @@ platforms: amd64: run_user: _daemon_ +entrypoint-service: sparkd environment: SPARK_HOME: /opt/spark From b19f8e144fffbd1705d81b014c2f9e917737406c Mon Sep 17 00:00:00 2001 From: Enrico Deusebio Date: Fri, 6 Dec 2024 13:50:42 +0100 Subject: [PATCH 2/6] fix --- images/charmed-spark/bin/sparkd.sh | 2 +- images/charmed-spark/rockcraft.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/images/charmed-spark/bin/sparkd.sh b/images/charmed-spark/bin/sparkd.sh index de9c0617..21a1df9f 100644 --- a/images/charmed-spark/bin/sparkd.sh +++ b/images/charmed-spark/bin/sparkd.sh @@ -34,7 +34,7 @@ case "${FLAVOUR}" in pushd /opt/spark ./entrypoint.sh "$@" ;; - "") + ""|"sleep") # Infinite sleep to allow pebble to be running indefinitely sleep inf ;; diff --git a/images/charmed-spark/rockcraft.yaml b/images/charmed-spark/rockcraft.yaml index 4edc02ec..257dabb8 100644 --- a/images/charmed-spark/rockcraft.yaml +++ b/images/charmed-spark/rockcraft.yaml @@ -33,7 +33,7 @@ environment: services: sparkd: - command: "/bin/bash /opt/pebble/sparkd.sh" + command: "/bin/bash /opt/pebble/sparkd.sh [ sleep ]" summary: "This is the service to startup Spark processes using the Spark entrypoint" override: replace startup: enabled From 1373bb97d580cf2efd89779c26a114b09850d887 Mon Sep 17 00:00:00 2001 From: Enrico Deusebio Date: Fri, 6 Dec 2024 13:52:41 +0100 Subject: [PATCH 3/6] fix on gpu image --- images/charmed-spark-gpu/rockcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/charmed-spark-gpu/rockcraft.yaml b/images/charmed-spark-gpu/rockcraft.yaml index 6d67869a..f77e14cd 100644 --- a/images/charmed-spark-gpu/rockcraft.yaml +++ b/images/charmed-spark-gpu/rockcraft.yaml @@ -34,7 +34,7 @@ environment: services: sparkd: - command: "/bin/bash /opt/pebble/sparkd.sh" + command: "/bin/bash /opt/pebble/sparkd.sh [ sleep ]" summary: "This is the service to startup Spark processes using the Spark entrypoint" override: replace startup: enabled From 05fd19cfceee5f16d200bd5c7cac5e6c1f255e2a Mon Sep 17 00:00:00 2001 From: Enrico Deusebio Date: Fri, 6 Dec 2024 14:50:56 +0100 Subject: [PATCH 4/6] create tar file --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 5a3982d0..2afb0a1d 100644 --- a/Makefile +++ b/Makefile @@ -191,6 +191,8 @@ $(SPARK_MARKER): $(ROCK_FILE) oci-archive:"$(ROCK_FILE)" \ docker-daemon:"$(SPARK_DOCKER_ALIAS)" + docker save $(SPARK_DOCKER_ALIAS) -o $(SPARK_ARTIFACT) + touch $(SPARK_MARKER) @@ -243,6 +245,8 @@ $(SPARK_GPU_MARKER): $(ROCK_FILE_GPU) oci-archive:"$(ROCK_FILE_GPU)" \ docker-daemon:"$(SPARK_GPU_DOCKER_ALIAS)" + docker save $(SPARK_GPU_DOCKER_ALIAS) -o $(SPARK_ARTIFACT) + touch $(SPARK_GPU_MARKER) # Shorthand recipe for building Spark-gpu image From 4d7d1769b7752f3696d02f8aa81e62e98b9b3b9d Mon Sep 17 00:00:00 2001 From: Enrico Deusebio Date: Fri, 6 Dec 2024 17:51:07 +0100 Subject: [PATCH 5/6] fix reference --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2afb0a1d..cb2c0a80 100644 --- a/Makefile +++ b/Makefile @@ -245,7 +245,7 @@ $(SPARK_GPU_MARKER): $(ROCK_FILE_GPU) oci-archive:"$(ROCK_FILE_GPU)" \ docker-daemon:"$(SPARK_GPU_DOCKER_ALIAS)" - docker save $(SPARK_GPU_DOCKER_ALIAS) -o $(SPARK_ARTIFACT) + docker save $(SPARK_GPU_DOCKER_ALIAS) -o $(SPARK_GPU_ARTIFACT) touch $(SPARK_GPU_MARKER) From 788680bd5403b16bab9174ecf48033df3d125ae2 Mon Sep 17 00:00:00 2001 From: Enrico Deusebio Date: Fri, 6 Dec 2024 20:51:26 +0100 Subject: [PATCH 6/6] re-insert trap --- images/charmed-spark/bin/sparkd.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/images/charmed-spark/bin/sparkd.sh b/images/charmed-spark/bin/sparkd.sh index 21a1df9f..3e58f872 100644 --- a/images/charmed-spark/bin/sparkd.sh +++ b/images/charmed-spark/bin/sparkd.sh @@ -24,6 +24,15 @@ function log_forwarding { fi } +function finish { + if [ $? -ne 0 ] + then + kill -1 1 + sleep 1 + fi +} +trap finish EXIT + FLAVOUR=$1 echo "Running script with ${FLAVOUR} flavour"