Skip to content

Commit 2cae664

Browse files
streamer45streamer45
authored
[MM-62065] e2e HA support (#913)
* Refactor e2e scripts * HA Support * Fix signaling after WS reconnect in non-RTCD HA (#920)
1 parent fef920d commit 2cae664

File tree

9 files changed

+234
-151
lines changed

9 files changed

+234
-151
lines changed

‎.github/workflows/e2e.yml

+7-44
Original file line numberDiff line numberDiff line change
@@ -73,13 +73,15 @@ jobs:
7373
COMPOSE_PROJECT_NAME: playwright_tests
7474
DOCKER_NETWORK: playwright_tests
7575
CONTAINER_SERVER: playwright_tests_server
76+
CONTAINER_PROXY: playwright_tests_proxy
7677
IMAGE_CALLS_OFFLOADER: mattermost/calls-offloader:v0.8.0
7778
IMAGE_CALLS_RECORDER: mattermost/calls-recorder:v0.7.7
7879
IMAGE_CALLS_TRANSCRIBER: mattermost/calls-transcriber:v0.5.0
7980
IMAGE_SERVER: mattermostdevelopment/mattermost-enterprise-edition:master
8081
IMAGE_CURL: curlimages/curl:8.7.1
8182
CI_NODE_INDEX: ${{ matrix.run_id }}
8283
CI_NODE_TOTAL: ${{ needs.generate-matrix.outputs.parallelism }}
84+
WORKSPACE: ${{github.workspace}}
8385
strategy:
8486
matrix: ${{ fromJson(needs.generate-matrix.outputs.parallelism-matrix) }}
8587
steps:
@@ -108,14 +110,6 @@ jobs:
108110
mkdir -p ${{ github.workspace }}/config
109111
go run main.go
110112
111-
- name: e2e/patch-generated-configuration
112-
env:
113-
OUTPUT_CONFIG: ${{ github.workspace }}/config/config.json
114-
run: |
115-
cp ${OUTPUT_CONFIG} ${OUTPUT_CONFIG}.orig
116-
jq -f ${{ github.workspace }}/e2e/overlay-config.jq ${OUTPUT_CONFIG}.orig > ${OUTPUT_CONFIG}
117-
diff ${OUTPUT_CONFIG}.orig ${OUTPUT_CONFIG} || true
118-
119113
- name: e2e/download-mattermost-plugin-calls-package
120114
uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
121115
with:
@@ -130,41 +124,14 @@ jobs:
130124
username: ${{ secrets.DOCKERHUB_DEV_USERNAME }}
131125
password: ${{ secrets.DOCKERHUB_DEV_TOKEN }}
132126

133-
- name: e2e/prepare-server
127+
- name: e2e/run-server
134128
env:
135129
DOCKER_CLIENT_TIMEOUT: 120
136130
COMPOSE_HTTP_TIMEOUT: 120
137131
DOCKER_COMPOSE_FILE: ${{ github.workspace }}/e2e/docker/docker-compose.yaml
132+
MM_PLUGIN_CALLS_TEST_LICENSE: ${{ secrets.MM_PLUGIN_CALLS_TEST_LICENSE }}
138133
run: |
139-
mkdir -p ${{ github.workspace }}/logs
140-
mkdir -p ${{ github.workspace }}/config
141-
mkdir -p ${{ github.workspace }}/dotenv
142-
143-
## Navigate to mattermost server build folder to spawn the environment
144-
cd mattermost/server/build/
145134
${{ github.workspace }}/e2e/scripts/prepare-server.sh
146-
cd -
147-
148-
## Add extra environment variables for mattermost server
149-
echo "MM_LICENSE=${{ secrets.MM_PLUGIN_CALLS_TEST_LICENSE }}" >> dotenv/app.private.env
150-
echo "MM_FEATUREFLAGS_BoardsProduct=true" >> dotenv/app.private.env
151-
echo "MM_SERVICEENVIRONMENT=test" >> dotenv/app.private.env
152-
echo "MM_CALLS_JOB_SERVICE_URL=http://calls-offloader:4545" >> dotenv/app.private.env
153-
154-
sudo chown -R 2000:2000 ${{ github.workspace }}/logs
155-
sudo chown -R 2000:2000 ${{ github.workspace }}/config
156-
157-
# Spawn mattermost server
158-
echo "Spawning mattermost server ... "
159-
docker run -d --quiet --name ${CONTAINER_SERVER} \
160-
--net ${DOCKER_NETWORK} \
161-
--net-alias mm-server \
162-
--user mattermost:mattermost \
163-
--env-file="${{ github.workspace }}/dotenv/app.private.env" \
164-
-v ${{ github.workspace }}/config:/mattermost/config:rw \
165-
-v ${{ github.workspace }}/logs:/mattermost/logs:rw \
166-
${IMAGE_SERVER} \
167-
sh -c "/mattermost/bin/mattermost server"
168135
169136
- name: e2e/build-e2e-image
170137
working-directory: e2e
@@ -174,17 +141,13 @@ jobs:
174141
docker build --quiet -t mm-playwright -f Playwright.Dockerfile .
175142
176143
- name: e2e/test
144+
env:
145+
WORKSPACE: ${{github.workspace}}
146+
RUN_ID: ${{matrix.run_id}}
177147
id: test
178148
run: |
179-
mkdir -p ${{ github.workspace }}/results
180149
${{ github.workspace }}/e2e/scripts/run.sh
181150
182-
## Check if we have an early failures in order to upload logs
183-
NUM_FAILURES=0
184-
NUM_FAILURES=$((NUM_FAILURES + $(jq '.suites[].suites[].specs[].tests[] | last(.results[]) | select(.status != "passed").status' < ${{ github.workspace }}/results/pw-results-${{ matrix.run_id }}.json | wc -l)))
185-
echo "FAILURES=${NUM_FAILURES}" >> ${GITHUB_OUTPUT}
186-
sudo chown -R 1001:1001 ${{ github.workspace }}/logs
187-
188151
- name: e2e/persist-report-results
189152
uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
190153
with:

‎e2e/docker/docker-compose.yaml

+14
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,24 @@ services:
2121
timeout: 10s
2222
retries: 3
2323

24+
haproxy:
25+
image: nginx
26+
container_name: ${CONTAINER_PROXY}
27+
volumes:
28+
- ./nginx.conf:/etc/nginx/conf.d/default.conf
29+
restart: on-failure
30+
networks:
31+
default:
32+
aliases:
33+
- mm-server
34+
ports:
35+
- "8065:8065"
36+
2437
start_dependencies:
2538
image: mattermost/mattermost-wait-for-dep:latest
2639
depends_on:
2740
- postgres
41+
- haproxy
2842
command: postgres:5432
2943
networks:
3044
default:

‎e2e/docker/nginx.conf

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
upstream app_cluster {
2+
server mm-server1:8065 fail_timeout=10s max_fails=10;
3+
server mm-server2:8065 fail_timeout=10s max_fails=10;
4+
}
5+
6+
server {
7+
listen 8065;
8+
9+
location ~ /api/v[0-9]+/(users/)?websocket$ {
10+
proxy_set_header Upgrade $http_upgrade;
11+
proxy_set_header Connection "upgrade";
12+
proxy_http_version 1.1;
13+
client_max_body_size 50M;
14+
proxy_set_header Host $http_host;
15+
proxy_set_header X-Real-IP $remote_addr;
16+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
17+
proxy_set_header X-Forwarded-Proto $scheme;
18+
proxy_set_header X-Frame-Options SAMEORIGIN;
19+
proxy_buffers 256 16k;
20+
proxy_buffer_size 16k;
21+
proxy_read_timeout 600s;
22+
proxy_pass http://app_cluster;
23+
}
24+
25+
location / {
26+
client_max_body_size 100M;
27+
proxy_set_header Upgrade $http_upgrade;
28+
proxy_set_header Connection "upgrade";
29+
proxy_http_version 1.1;
30+
proxy_set_header Host $http_host;
31+
proxy_set_header X-Real-IP $remote_addr;
32+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
33+
proxy_set_header X-Forwarded-Proto $scheme;
34+
proxy_set_header X-Frame-Options SAMEORIGIN;
35+
proxy_pass http://app_cluster;
36+
}
37+
}

‎e2e/overlay-config.jq

-48
This file was deleted.

‎e2e/scripts/prepare-server.sh

+94-10
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,23 @@
22
set -eu
33
set -o pipefail
44

5+
function print_logs {
6+
docker logs ${CONTAINER_SERVER}1
7+
docker logs ${CONTAINER_SERVER}2
8+
docker logs ${CONTAINER_PROXY}
9+
}
10+
11+
trap print_logs EXIT
12+
13+
mkdir -p ${WORKSPACE}/logs
14+
mkdir -p ${WORKSPACE}/config
15+
mkdir -p ${WORKSPACE}/dotenv
16+
517
docker network create ${DOCKER_NETWORK}
618

719
# Start server dependencies
820
echo "Starting server dependencies ... "
9-
DOCKER_NETWORK=${DOCKER_NETWORK} docker compose -f ${DOCKER_COMPOSE_FILE} run -d --rm start_dependencies
21+
DOCKER_NETWORK=${DOCKER_NETWORK} CONTAINER_PROXY=${CONTAINER_PROXY} docker compose -f ${DOCKER_COMPOSE_FILE} run -d --rm start_dependencies
1022
timeout --foreground 90s bash -c "until docker compose -f ${DOCKER_COMPOSE_FILE} exec -T postgres pg_isready ; do sleep 5 ; done"
1123

1224
echo "Pulling ${IMAGE_CALLS_RECORDER} in order to be quickly accessible ... "
@@ -29,15 +41,87 @@ docker images
2941
echo "Spawning calls-offloader service with docker host access ..."
3042
# Spawn calls offloader image as root to access local docker socket
3143
docker run -d --quiet --user root --name "${COMPOSE_PROJECT_NAME}_callsoffloader" \
32-
-v /var/run/docker.sock:/var/run/docker.sock:rw \
33-
--net ${DOCKER_NETWORK} \
34-
--env "API_SECURITY_ALLOWSELFREGISTRATION=true" \
35-
--env "JOBS_MAXCONCURRENTJOBS=20" \
36-
--env "LOGGER_ENABLEFILE=false" \
37-
--env "LOGGER_CONSOLELEVEL=DEBUG" \
38-
--env "DEV_MODE=true" \
39-
--env "DOCKER_NETWORK=${DOCKER_NETWORK}" \
40-
--network-alias=calls-offloader ${IMAGE_CALLS_OFFLOADER}
44+
-v /var/run/docker.sock:/var/run/docker.sock:rw \
45+
--net ${DOCKER_NETWORK} \
46+
--env "API_SECURITY_ALLOWSELFREGISTRATION=true" \
47+
--env "JOBS_MAXCONCURRENTJOBS=20" \
48+
--env "LOGGER_ENABLEFILE=false" \
49+
--env "LOGGER_CONSOLELEVEL=DEBUG" \
50+
--env "DEV_MODE=true" \
51+
--env "DOCKER_NETWORK=${DOCKER_NETWORK}" \
52+
--network-alias=calls-offloader ${IMAGE_CALLS_OFFLOADER}
4153

4254
# Check that calls-offloader is up and ready
4355
docker run --rm --quiet --name "${COMPOSE_PROJECT_NAME}_curl_callsoffloader" --net ${DOCKER_NETWORK} ${IMAGE_CURL} sh -c "until curl -fs http://calls-offloader:4545/version; do echo Waiting for calls-offloader; sleep 5; done; echo calls-offloader is up"
56+
57+
## Add extra environment variables for mattermost server. This is needed to override configuration in HA since
58+
## the config is stored in DB.
59+
echo "MM_LICENSE=${MM_PLUGIN_CALLS_TEST_LICENSE}" >>${WORKSPACE}/dotenv/app.private.env
60+
echo "MM_FEATUREFLAGS_BoardsProduct=true" >>${WORKSPACE}/dotenv/app.private.env
61+
echo "MM_SERVICEENVIRONMENT=test" >>${WORKSPACE}/dotenv/app.private.env
62+
echo "MM_CALLS_JOB_SERVICE_URL=http://calls-offloader:4545" >>${WORKSPACE}/dotenv/app.private.env
63+
echo "MM_CONFIG=postgres://mmuser:mostest@postgres/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes" >>${WORKSPACE}/dotenv/app.private.env
64+
echo "MM_SERVICESETTINGS_SITEURL=http://mm-server:8065" >>${WORKSPACE}/dotenv/app.private.env
65+
echo "MM_SERVICESETTINGS_ENABLELOCALMODE=true" >>${WORKSPACE}/dotenv/app.private.env
66+
echo "MM_SERVICESETTINGS_ENABLEDEVELOPER=true" >>${WORKSPACE}/dotenv/app.private.env
67+
echo "MM_SERVICESETTINGS_ENABLETESTING=true" >>${WORKSPACE}/dotenv/app.private.env
68+
echo "MM_SERVICESETTINGS_ALLOWCORSFROM=http://localhost:8065" >>${WORKSPACE}/dotenv/app.private.env
69+
echo "MM_SERVICESETTINGS_ENABLEONBOARDINGFLOW=false" >>${WORKSPACE}/dotenv/app.private.env
70+
echo "MM_PLUGINSETTINGS_ENABLE=true" >>${WORKSPACE}/dotenv/app.private.env
71+
echo "MM_PLUGINSETTINGS_ENABLEUPLOADS=true" >>${WORKSPACE}/dotenv/app.private.env
72+
echo "MM_PLUGINSETTINGS_AUTOMATICPREPACKAGEDPLUGINS=false" >>${WORKSPACE}/dotenv/app.private.env
73+
echo "MM_CLUSTERSETTINGS_ENABLE=true" >>${WORKSPACE}/dotenv/app.private.env
74+
echo "MM_CLUSTERSETTINGS_CLUSTERNAME=mm_server_e2e" >>${WORKSPACE}/dotenv/app.private.env
75+
echo "MM_LOGSETTINGS_CONSOLELEVEL=DEBUG" >>${WORKSPACE}/dotenv/app.private.env
76+
echo "MM_LOGSETTINGS_FILELEVEL=DEBUG" >>${WORKSPACE}/dotenv/app.private.env
77+
echo "MM_SQLSETTINGS_DATASOURCE=postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes" >>${WORKSPACE}/dotenv/app.private.env
78+
echo "MM_EXPERIMENTALSETTINGS_DISABLEAPPBAR=false" >>${WORKSPACE}/dotenv/app.private.env
79+
echo "MM_ANNOUNCEMENTSETTINGS_USERNOTICESENABLED=false" >>${WORKSPACE}/dotenv/app.private.env
80+
echo "MM_ANNOUNCEMENTSETTINGS_ADMINNOTICESENABLED=false" >>${WORKSPACE}/dotenv/app.private.env
81+
82+
sudo cp -r ${WORKSPACE}/logs ${WORKSPACE}/logs1
83+
sudo cp -r ${WORKSPACE}/config ${WORKSPACE}/config1
84+
sudo chown -R 2000:2000 ${WORKSPACE}/logs1
85+
sudo chown -R 2000:2000 ${WORKSPACE}/config1
86+
87+
sudo cp -r ${WORKSPACE}/logs ${WORKSPACE}/logs2
88+
sudo cp -r ${WORKSPACE}/config ${WORKSPACE}/config2
89+
sudo chown -R 2000:2000 ${WORKSPACE}/logs2
90+
sudo chown -R 2000:2000 ${WORKSPACE}/config2
91+
92+
mkdir -p ${WORKSPACE}/mmdata
93+
sudo chown -R 2000:2000 ${WORKSPACE}/mmdata
94+
95+
# Spawn mattermost server
96+
echo "Spawning mattermost server 1 ... "
97+
docker run -d --quiet --name ${CONTAINER_SERVER}1 \
98+
--net ${DOCKER_NETWORK} \
99+
--net-alias mm-server1 \
100+
--user mattermost:mattermost \
101+
--env-file="${WORKSPACE}/dotenv/app.private.env" \
102+
-v ${WORKSPACE}/config1:/mattermost/config:rw \
103+
-v ${WORKSPACE}/logs1:/mattermost/logs:rw \
104+
-v ${WORKSPACE}/mmdata:/mattermost/data:rw \
105+
${IMAGE_SERVER} \
106+
sh -c "/mattermost/bin/mattermost server"
107+
108+
echo "Checking node 1 is up and running"
109+
timeout --foreground 90s bash -c "until docker run --rm --quiet --name ${COMPOSE_PROJECT_NAME}_curl_mm1 --net ${DOCKER_NETWORK} ${IMAGE_CURL} curl -fs http://mm-server1:8065/api/v4/system/ping; do echo Waiting for mm-server1; sleep 2; done; echo mm-server1 is up"
110+
111+
echo "Spawning mattermost server 2 ... "
112+
docker run -d --quiet --name ${CONTAINER_SERVER}2 \
113+
--net ${DOCKER_NETWORK} \
114+
--net-alias mm-server2 \
115+
--user mattermost:mattermost \
116+
--env-file="${WORKSPACE}/dotenv/app.private.env" \
117+
-v ${WORKSPACE}/config2:/mattermost/config:rw \
118+
-v ${WORKSPACE}/logs2:/mattermost/logs:rw \
119+
-v ${WORKSPACE}/mmdata:/mattermost/data:rw \
120+
${IMAGE_SERVER} \
121+
sh -c "/mattermost/bin/mattermost server"
122+
123+
echo "Checking node 2 is up and running"
124+
timeout --foreground 90s bash -c "until docker run --rm --quiet --name ${COMPOSE_PROJECT_NAME}_curl_mm2 --net ${DOCKER_NETWORK} ${IMAGE_CURL} curl -fs http://mm-server2:8065/api/v4/system/ping; do echo Waiting for mm-server2; sleep 2; done; echo mm-server2 is up"
125+
126+
echo "Checking proxy is up and running"
127+
timeout --foreground 90s bash -c "until docker run --rm --quiet --name ${COMPOSE_PROJECT_NAME}_curl_proxy --net ${DOCKER_NETWORK} ${IMAGE_CURL} curl -fs http://mm-server:8065/api/v4/system/ping; do echo Waiting for proxy; sleep 2; done; echo proxy is up"

0 commit comments

Comments
 (0)