diff --git a/README.md b/README.md index ac073d3..41a8af2 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ More details on my motivation are mentioned in the presentation [here](docs/Topi 2. Helm v3 (Tested version v3.11.3) 3. Kubernetes (Tested version v1.25.5 - Runtime: Docker v20.10.24 - Provisioned by Minikube v1.26.1) +Noted: At the moment, chart supports PersistenceVolume on local HostPath only. Other types of PV will be added later. + ## Install the chart ```shell @@ -38,6 +40,20 @@ Built chart is located under target/helm/repo/scalable-selenium-grid-x.x.x.tgz ## Change Log +### :heavy_check_mark: 23.10.24 +**Removed** +- Scripts, configs in video-recording are initialized when build Docker image. Refer to [source repo](../../../docker-selenium/tree/trunk/Video). + +**Updated** +- Node directory `/home/seluser/Downloads` is mounted to `/data/$POD_NAME`. +- For traceability, a session is run in which node `$POD_NAME` is logged in file `/data/$SESSION_ID.txt`. +- New image for video-recording `ndviet/video:ffmpeg-6.0-alpine-20231024`. + +**Added** +- [WebDAV](../../../test-webdav-docker) service is deployed to access and browse data and videos of distributed nodes remotely. + - By default, WebDAV is exposed via ingress TCP host port `8080`. Change another port via `ingress-nginx.tcp` in scalable-selenium-grid/values.yaml. + - Videos and node data can be browsed via WebDAV client. + ### :heavy_check_mark: 23.10.19 **Updated** - Chart dependencies: diff --git a/patch-selenium-grid/src/main/resources/patch-selenium-grid/patch/configurations/Video/supervisord.conf b/patch-selenium-grid/src/main/resources/patch-selenium-grid/patch/configurations/Video/supervisord.conf deleted file mode 100644 index 924a397..0000000 --- a/patch-selenium-grid/src/main/resources/patch-selenium-grid/patch/configurations/Video/supervisord.conf +++ /dev/null @@ -1,37 +0,0 @@ -; Documentation of this file format -> http://supervisord.org/configuration.html - -[supervisord] -childlogdir=/var/log/supervisor ; ('AUTO' child log dir, default $TEMP) -logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log) -logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) -logfile_backups=10 ; (num of main logfile rotation backups;default 10) -loglevel=info ; (log level;default info; others: debug,warn,trace) -pidfile=/var/run/supervisor/supervisord.pid ; (supervisord pidfile;default supervisord.pid) -nodaemon=true ; (start in foreground if true;default false) -minfds=1024 ; (min. avail startup file descriptors;default 1024) -minprocs=200 ; (min. avail process descriptors;default 200) -user=root - -[program:video-recording] -priority=0 -command=bash -c "/opt/bin/video.sh; EXIT_CODE=$?; kill -s SIGTERM `cat /var/run/supervisor/supervisord.pid`; exit $EXIT_CODE" -autostart=true -autorestart=unexpected -stopsignal=TERM - -;Logs (all activity redirected to stdout so it can be seen through "docker logs" -redirect_stderr=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 - -[program:video-ready] -priority=5 -command=bash -c "python3 -u /opt/bin/video_ready.py; EXIT_CODE=$?; kill -s SIGTERM `cat /var/run/supervisor/supervisord.pid`; exit $EXIT_CODE" -autostart=true -autorestart=unexpected -stopsignal=TERM - -;Logs (all activity redirected to stdout so it can be seen through "docker logs" -redirect_stderr=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 \ No newline at end of file diff --git a/patch-selenium-grid/src/main/resources/patch-selenium-grid/patch/configurations/Video/video.sh b/patch-selenium-grid/src/main/resources/patch-selenium-grid/patch/configurations/Video/video.sh deleted file mode 100644 index d0d2844..0000000 --- a/patch-selenium-grid/src/main/resources/patch-selenium-grid/patch/configurations/Video/video.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash - -VIDEO_SIZE="${SE_SCREEN_WIDTH}""x""${SE_SCREEN_HEIGHT}" -DISPLAY_CONTAINER_NAME=${DISPLAY_CONTAINER_NAME} -DISPLAY_NUM=${DISPLAY_NUM} -FILE_NAME=${FILE_NAME} -FRAME_RATE=${FRAME_RATE:-$SE_FRAME_RATE} -CODEC=${CODEC:-$SE_CODEC} -PRESET=${PRESET:-$SE_PRESET} - -function get_session_id { - session_id=$(curl -s --request GET 'http://'${DISPLAY_CONTAINER_NAME}':'${SE_NODE_PORT}'/status' | jq -r '.[]?.node?.slots | .[0]?.session?.sessionId') -} - -function star_recording { - video_file_name="${SE_VIDEO_FOLDER}/$session_id.mp4" - ffmpeg -nostdin -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM}.0 -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p $video_file_name & - FFMPEG_PID=$! - recording_started="true" -} - -function stop_recording { - kill -s SIGINT ${FFMPEG_PID} - wait ${FFMPEG_PID} - recording_started="false" - recorded_count=$((recorded_count+1)) -} - -function terminate_gracefully { - echo "Trapped SIGTERM/SIGINT/x so shutting down video-recording..." - while true; - do - get_session_id - if [ "$session_id" != "null" -a "$session_id" != "" ] && [ $recording_started = "true" ]; - then - echo "Session: $session_id is active, waiting for it to finish" - sleep 1 - else - pkill --signal TERM -f "ffmpeg" - break - fi - done - pkill --signal TERM -f "video_ready.py" - echo "Shutdown completed!" -} - -if [ "${SE_VIDEO_RECORD}" = "true" ]; -then - max_recorded_count=$((${DRAIN_AFTER_SESSION_COUNT:-SE_DRAIN_AFTER_SESSION_COUNT})) - recorded_count=0 - return_code=1 - max_attempts=50 - attempts=0 - recording_started="false" - video_file_name="" - - trap terminate_gracefully SIGTERM SIGINT - - echo 'Checking if the display is open...' - until [ $return_code -eq 0 -o $attempts -eq $max_attempts ]; do - xset -display ${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM} b off > /dev/null 2>&1 - return_code=$? - if [ $return_code -ne 0 ]; then - echo 'Waiting before next display check...' - sleep 0.5 - fi - attempts=$((attempts+1)) - done - - while true; - do - get_session_id - if [ "$session_id" != "null" -a "$session_id" != "" ] && [ $recording_started = "false" ]; - then - echo "Reached session: $session_id" - echo "Starting to record video" - star_recording - echo "Video recording started" - elif [ "$session_id" = "null" -o "$session_id" = "" ] && [ $recording_started = "true" ]; - then - echo "Stopping to record video" - stop_recording - echo "Video recording stopped" - if [ $max_recorded_count -gt 0 ] && [ $recorded_count -ge $max_recorded_count ]; - then - echo "Max recorded count reached, exiting" - exit 0 - fi - elif [ $recording_started = "true" ]; - then - echo "Video recording in progress" - fi - sleep 1 - done -fi diff --git a/patch-selenium-grid/src/main/resources/patch-selenium-grid/patch/configurations/Video/video_ready.py b/patch-selenium-grid/src/main/resources/patch-selenium-grid/patch/configurations/Video/video_ready.py deleted file mode 100644 index b5e4d44..0000000 --- a/patch-selenium-grid/src/main/resources/patch-selenium-grid/patch/configurations/Video/video_ready.py +++ /dev/null @@ -1,79 +0,0 @@ -from http.server import BaseHTTPRequestHandler,HTTPServer -from os import environ -import json -import psutil -import time -import signal -import sys - -video_ready_port = int(environ.get('VIDEO_READY_PORT', 9000)) - -class Handler(BaseHTTPRequestHandler): - - def do_GET(self): - if self.path == '/recording': - self.video_recording() - elif self.path == '/status': - self.video_status() - else: - self.video_status() - - def do_POST(self): - if self.path == '/drain': - self.video_drain() - - def video_recording(self): - self.send_response(200) - self.end_headers() - self.wfile.write(json.dumps(Handler.is_recording()).encode('utf-8')) - - def video_status(self): - video_status = Handler.is_alive() - response_code = 200 if video_status else 404 - self.send_response(response_code) - self.end_headers() - self.wfile.write(json.dumps(video_status).encode('utf-8')) - - def video_drain(self): - recording = Handler.is_recording() - response_text = 'waiting for recording to be completed' if recording else 'terminating now' - self.send_response(200) - self.end_headers() - self.wfile.write(json.dumps({'status': response_text}).encode('utf-8')) - quit() - - @staticmethod - def terminate_gracefully(signum, stack_frame): - print("Trapped SIGTERM/SIGINT/x so shutting down video-ready...") - while Handler.is_recording(): - time.sleep(1) - quit() - - @staticmethod - def is_alive(): - return Handler.is_proc_running("video.sh") - - @staticmethod - def is_recording(): - return Handler.is_proc_running("ffmpeg") - - @staticmethod - def is_proc_running(process_name): - for proc in psutil.process_iter(): - if process_name in "".join(proc.cmdline()): - return True - return False - - @staticmethod - def stop_proc(process_name): - for proc in psutil.process_iter(): - if process_name in "".join(proc.cmdline()): - proc.terminate() - print("Terminated process: " + process_name) - -# register SIGTERM handlers to enable graceful shutdown of recording -signal.signal(signal.SIGINT, Handler.terminate_gracefully) -signal.signal(signal.SIGTERM, Handler.terminate_gracefully) - -httpd = HTTPServer( ('0.0.0.0', video_ready_port), Handler ) -httpd.serve_forever() diff --git a/patch-selenium-grid/src/main/resources/patch-selenium-grid/patch/templates/tracing-configmap.yaml b/patch-selenium-grid/src/main/resources/patch-selenium-grid/patch/templates/tracing-configmap.yaml new file mode 100644 index 0000000..e1a9e64 --- /dev/null +++ b/patch-selenium-grid/src/main/resources/patch-selenium-grid/patch/templates/tracing-configmap.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: selenium-tracing-config + namespace: {{ .Release.Namespace }} +{{- with .Values.nodeConfigMap.annotations }} + annotations: {{- toYaml . | nindent 4 }} +{{- end }} + labels: + {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- with .Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +data: + SE_ENABLE_TRACING: {{ (.Values.global.seleniumGrid.tracing).enabled | default "false" | quote }} + {{- $jaegerEndpoint := (.Values.global.seleniumGrid.tracing).jaegerEndpoint | default (printf "http://%s-jaeger-all-in-one-headless:14250" .Release.Name) }} + {{- $jaegerEndpoint := tpl $jaegerEndpoint . }} + {{- $JAVA_OPTS := printf "-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=%s -Dotel.resource.attributes=service.name=selenium -Dotel.java.global-autoconfigure.enabled=true" $jaegerEndpoint }} + JAVA_OPTS: {{ $JAVA_OPTS | quote }} diff --git a/pom.xml b/pom.xml index 9b389cd..6999ca8 100644 --- a/pom.xml +++ b/pom.xml @@ -18,14 +18,14 @@ - 23.10.19 + 23.10.24 bash -c ${project.version} selenium-grid https://www.selenium.dev/docker-selenium 0.22.0 - ndviet/video:ffmpeg-6.0-alpine-20231019 + ndviet/video:ffmpeg-6.0-alpine-20231024 jaeger-all-in-one https://raw.githubusercontent.com/hansehe/jaeger-all-in-one/master/helm/charts 0.1.11 @@ -35,6 +35,9 @@ keda https://kedacore.github.io/charts 2.12.0 + webdav + https://www.ndviet.org/charts + 4.3.0 11 11 UTF-8 diff --git a/scalable-selenium-grid/pom.xml b/scalable-selenium-grid/pom.xml index 05b472b..a097ddc 100644 --- a/scalable-selenium-grid/pom.xml +++ b/scalable-selenium-grid/pom.xml @@ -51,6 +51,11 @@ ${chart.keda.repository} true + + ${chart.webdav.name} + ${chart.webdav.repository} + true + diff --git a/scalable-selenium-grid/src/main/resources/scalable-selenium-grid/Chart.yaml b/scalable-selenium-grid/src/main/resources/scalable-selenium-grid/Chart.yaml index 2ef1b2a..96de297 100644 --- a/scalable-selenium-grid/src/main/resources/scalable-selenium-grid/Chart.yaml +++ b/scalable-selenium-grid/src/main/resources/scalable-selenium-grid/Chart.yaml @@ -51,4 +51,12 @@ dependencies: version: ${chart.ingress.version} tags: - full - - ingress-nginx \ No newline at end of file + - ingress-nginx + + - condition: global.webdav.enabled, webdav.enabled + name: ${chart.webdav.name} + repository: ${chart.webdav.repository} + version: ${chart.webdav.version} + tags: + - full + - webdav \ No newline at end of file diff --git a/scalable-selenium-grid/src/main/resources/scalable-selenium-grid/templates/NOTES.txt b/scalable-selenium-grid/src/main/resources/scalable-selenium-grid/templates/NOTES.txt new file mode 100644 index 0000000..86b324a --- /dev/null +++ b/scalable-selenium-grid/src/main/resources/scalable-selenium-grid/templates/NOTES.txt @@ -0,0 +1,12 @@ +TestOps Autoscaling Selenium Grid ecosystem deployed successfully. + +{{- $pvEnabled := index .Values "grid-pvc" "selenium-node" "persistence" "pv" "enabled" }} +{{- if and .Values.global.seleniumGrid.persistence.enabled $pvEnabled }} +Noted: Please ensure local hostPath directory is created and set permissions correctly. If not created yet, kindly execute below commands once! +{{- $hostPathDir := index .Values "grid-pvc" "selenium-node" "persistence" "pv" "hostPath" "path" }} +--- +sudo mkdir -p {{ $hostPathDir }} +sudo chown 999 {{ $hostPathDir }} +sudo chmod 755 {{ $hostPathDir }} +--- +{{- end }} \ No newline at end of file diff --git a/scalable-selenium-grid/src/main/resources/scalable-selenium-grid/templates/README.txt b/scalable-selenium-grid/src/main/resources/scalable-selenium-grid/templates/README.txt deleted file mode 100644 index e69de29..0000000 diff --git a/scalable-selenium-grid/src/main/resources/scalable-selenium-grid/values.yaml b/scalable-selenium-grid/src/main/resources/scalable-selenium-grid/values.yaml index 7db8824..1932d77 100644 --- a/scalable-selenium-grid/src/main/resources/scalable-selenium-grid/values.yaml +++ b/scalable-selenium-grid/src/main/resources/scalable-selenium-grid/values.yaml @@ -4,6 +4,7 @@ tags: jaeger-all-in-one: false ingress-nginx: false keda: false + webdav: false global: seleniumGrid: @@ -11,6 +12,9 @@ global: enabled: &persistence true autoscaling: enableWithExistingKEDA: &enableWithExistingKEDA true + tracing: + enabled: true + jaegerEndpoint: http://{{ .Release.Name }}-jaeger-all-in-one-headless:14250 videoImage: &videoImage "${chart.images.video.tag}" selenium-grid: #enabled: true @@ -43,7 +47,7 @@ selenium-grid: helm.sh/hook: post-install,post-upgrade,post-rollback scaledJobOptions: minReplicaCount: &minReplicaCount 0 - successfulJobsHistoryLimit: 0 # clean up pods with status Completed + successfulJobsHistoryLimit: 5 # clean up pods with status Completed failedJobsHistoryLimit: 5 pollingInterval: 20 scalingStrategy: @@ -59,22 +63,39 @@ selenium-grid: curl -X POST 127.0.0.1:9000/drain --header 'X-REGISTRATION-SECRET;' && \ while curl 127.0.0.1:5555/status; do sleep 1; done; components: - extraEnvironmentVariables: &extraEnvironmentVariables - # Strategy: Node serve only 1 session and to be terminated + extraEnvironmentVariables: + extraEnvFrom: &extraEnvFrom + - configMapRef: + name: selenium-tracing-config + + chromeNode: + maxReplicaCount: &maxReplicaCount 8 + minReplicaCount: *minReplicaCount + resources: &resources + requests: + memory: "1Gi" + cpu: "1" + limits: + memory: "1Gi" + cpu: "1" + extraEnvFrom: *extraEnvFrom + extraEnvironmentVariables: &extraEnvironmentVariablesNodes # - name: SE_NODE_SESSION_TIMEOUT # value: "300" # - name: SE_NODE_OVERRIDE_MAX_SESSIONS # value: "true" - - name: SE_NODE_MAX_SESSIONS - value: "1" - name: DRAIN_AFTER_SESSION_COUNT #must add this to drain video together with session value: "1" - - name: SE_ENABLE_TRACING - value: "true" - - name: JAVA_OPTS - value: "-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://{{ .Release.Name }}-jaeger-all-in-one-headless:14250 -Dotel.resource.attributes=service.name=selenium -Dotel.java.global-autoconfigure.enabled=true" + - name: SE_NODE_MAX_SESSIONS #ensure the mapping is 1 session - 1 recording + value: "1" - name: SE_VIDEO_RECORD #must add this to enable video recording value: "true" + - name: SE_VIDEO_FOLDER + value: &SE_VIDEO_FOLDER "/videos" + - name: SE_SESSION_FOLDER + value: &SE_SESSION_FOLDER "/sessions" + - name: SE_OPTS + value: "--enable-managed-downloads true" - name: POD_NAME valueFrom: fieldRef: @@ -85,20 +106,8 @@ selenium-grid: fieldPath: status.podIP - name: DISPLAY_CONTAINER_NAME value: $(POD_IP) -# - name: FILE_NAME -# value: $(POD_IP)_$(POD_NAME).mp4 - - chromeNode: - maxReplicaCount: &maxReplicaCount 8 - minReplicaCount: *minReplicaCount - resources: &resources - requests: - memory: "1Gi" - cpu: "1" - limits: - memory: "1Gi" - cpu: "1" - extraEnvironmentVariables: *extraEnvironmentVariables + - name: SE_DOWNLOAD_MOUNT_PATH + value: data/$(POD_NAME) ports: &nodePorts - 99 - 5555 @@ -110,30 +119,28 @@ selenium-grid: imagePullPolicy: IfNotPresent ports: - containerPort: &videoContainerPort 9000 - env: *extraEnvironmentVariables + env: *extraEnvironmentVariablesNodes volumeMounts: - - name: videos - mountPath: /videos + - name: node-data + mountPath: *SE_SESSION_FOLDER + subPath: data + - name: node-data + mountPath: *SE_VIDEO_FOLDER subPath: videos - - name: configurations - mountPath: /opt/bin/video.sh - subPath: video.sh - - name: configurations - mountPath: /opt/bin/video_ready.py - subPath: video_ready.py - - name: configurations - mountPath: /etc/supervisord.conf - subPath: supervisord.conf startupProbe: httpGet: path: /status port: *videoContainerPort failureThreshold: 120 periodSeconds: 1 + extraVolumeMounts: + - name: node-data + mountPath: /home/seluser/Downloads + subPathExpr: $(SE_DOWNLOAD_MOUNT_PATH) extraVolumes: &nodeVolumes - - name: videos + - name: node-data persistentVolumeClaim: - claimName: "selenium-node" + claimName: &seleniumClaim "selenium-node" - name: configurations configMap: name: "selenium-configurations" @@ -149,7 +156,8 @@ selenium-grid: maxReplicaCount: *maxReplicaCount minReplicaCount: *minReplicaCount resources: *resources - extraEnvironmentVariables: *extraEnvironmentVariables + extraEnvFrom: *extraEnvFrom + extraEnvironmentVariables: *extraEnvironmentVariablesNodes ports: *nodePorts sidecars: *nodeVideoRecord extraVolumes: *nodeVolumes @@ -159,7 +167,8 @@ selenium-grid: maxReplicaCount: *maxReplicaCount minReplicaCount: *minReplicaCount resources: *resources - extraEnvironmentVariables: *extraEnvironmentVariables + extraEnvFrom: *extraEnvFrom + extraEnvironmentVariables: *extraEnvironmentVariablesNodes ports: *nodePorts sidecars: *nodeVideoRecord extraVolumes: *nodeVolumes @@ -175,6 +184,12 @@ keda: http: timeout: 15000 +webdav: + #enabled: true + persistence: + enabled: *persistence + existingClaim: *seleniumClaim + ingress-nginx: #enabled: true controller: @@ -187,3 +202,4 @@ ingress-nginx: tcp: 4444: "{{ .Release.Namespace }}/selenium-router:4444" 16686: "{{ .Release.Namespace }}/{{ .Release.Name }}-jaeger-all-in-one-headless:16686" + 8080: "{{ .Release.Namespace }}/{{ .Release.Name }}-webdav:8080"