-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathdocker-compose.yml
237 lines (230 loc) · 11.2 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
version: '3.5'
services:
prometheus:
image: prom/prometheus:v2.28.1
deploy:
resources:
limits:
cpus: '1'
memory: 512m
reservations:
cpus: '1'
memory: 512m
ports:
- "9000:9090"
user: "1000:1000"
volumes:
- ./kolibri-fleet-zio/prometheus:/etc/prometheus
# /prometheus is the default data dir
- ./kolibri-fleet-zio/prometheus/data:/prometheus/data
command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml
grafana:
image: grafana/grafana:8.0.5
deploy:
resources:
limits:
cpus: '1'
memory: 512m
reservations:
cpus: '1'
memory: 512m
volumes:
- ./kolibri-fleet-zio/grafana:/var/lib/grafana
ports:
- "3000:3000"
# user setting needed to be set to owner/group of the volume (above grafana folder on host machine)
user: "1000:1000"
kolibri-zio-1:
image: awagen/kolibri-fleet-zio:0.2.4
deploy:
# NOTE: you will need to adjust those limits to your setup
resources:
limits:
cpus: '12'
memory: 4gb
reservations:
cpus: '12'
memory: 4gb
ports:
- "8001:8001"
user: "1000:1000"
environment:
JVM_OPTS: >
-XX:+UseG1GC
-Xms3072m
-Xmx3072m
PROFILE: prod
NODE_HASH: "abc1"
HTTP_SERVER_PORT: 8001
PERSISTENCE_MODE: 'CLASS'
PERSISTENCE_MODULE_CLASS: 'de.awagen.kolibri.fleet.zio.config.di.modules.persistence.LocalPersistenceModule'
# AWS_PROFILE: 'developer'
# AWS_S3_BUCKET: 'kolibri-dev'
# AWS_S3_PATH: 'kolibri_fleet_zio_test'
# AWS_S3_REGION: 'EU_CENTRAL_1'
# GCP settings
# PERSISTENCE_MODE: 'GCP'
# GOOGLE_APPLICATION_CREDENTIALS: '/home/kolibri/gcp/kolibri-credentials.json'
# GCP_GS_BUCKET: '[your-bucket-name]'
# GCP_GS_PATH: '[your-bucket-base-path]'
# GCP_GS_PROJECT_ID: '[your-project-id]'
# the file path in the job definitions are to be given relative to the path (or bucket path) defined
# for the respective configuration of persistence
LOCAL_STORAGE_WRITE_BASE_PATH: '/app/test-files'
LOCAL_STORAGE_READ_BASE_PATH: '/app/test-files'
# JOB_TEMPLATES_PATH must be relative to the base path or bucket path, depending on the persistence selected
JOB_TEMPLATES_PATH: 'templates/jobs'
OUTPUT_RESULTS_PATH: 'test-results'
JUDGEMENT_FILE_SOURCE_TYPE: 'CSV'
# if judgement file format set to 'JSON_LINES', need to set 'DOUBLE' in case judgements are numeric in the json,
# if the numeric value is represented as string, use 'STRING'. This purely refers to how the json value is interpreted,
# later this will be cast to double either way
JUDGEMENT_FILE_JSON_LINES_JUDGEMENT_VALUE_TYPE_CAST: 'STRING'
MAX_RESOURCE_DIRECTIVES_LOAD_TIME_IN_MINUTES: 10
MAX_PARALLEL_ITEMS_PER_BATCH: 100
CONNECTION_POOL_SIZE_MIN: 400
CONNECTION_POOL_SIZE_MAX: 400
CONNECTION_TTL_IN_SECONDS: 1200
CONNECTION_TIMEOUT_IN_SECONDS: 10
CONNECTION_POOL_TYPE: 'FIXED'
# How many batches are to be processed at max at the same time?
MAX_NR_JOBS_PROCESSING: 5
# How many batches are at most claimed for processing at any given time? If this is higher than
# MAX_NR_JOBS_PROCESSING, will result in a range of queued batches
MAX_NR_JOBS_CLAIMED: 8
# Optional thread settings. For each: if set to -1, defaults are applied
NETTY_HTTP_CLIENT_THREADS_MAX: -1
# we need a fraction of threads for the blocking thread pool for stuff like file system IO
# yet the majority of computation is async, thus we assign majority of available threads
# (assuming 1 thread, 1 core).
BLOCKING_POOL_THREADS: 4
NON_BLOCKING_POOL_THREADS: 8
# The processing produces elements in a result queue that are then consumed into an aggregator.
# This can either be set to unlimited capacity (-1) or any value > 0 to limit the capacity, thus
# back-pressuring on the producer
RESULT_QUEUE_SIZE: -1
# Allows definition of number of single aggregators per batch. The end result is the aggregation of
# all results of all aggregators. This setting allows to increase throughput by speeding up the
# processed elements consumer.
NUM_AGGREGATORS_PER_BATCH: 100
# allows limiting the maximal throughput of elements for the batch by limiting the emission of elements
# of the input element stream. Default: -1 (unlimited).
MAX_BATCH_THROUGHPUT_PER_SECOND: -1
volumes:
- ./kolibri-fleet-zio/tmp_data:/app/test-files
# mount only needed in case you want to use PERSISTENCE_MODE 'AWS' (adjust the mounted folder from local
# to folder where credentials are located)
- ${HOME}/.aws/credentials:/home/kolibri/.aws/credentials:ro
# mount only needed in case you want to use PERSISTENCE_MODE 'GCP' (adjust the mounted folder from local
# to folder where credentials are located)
- ${HOME}/Clouds/gcp:/home/kolibri/gcp:ro
# NOTE: starting response-juggler requires cloning https://github.com/awagen/response-juggler and building
# the image locally via 'docker build -t response-juggler:0.1.0 .'. It will respond to each request with a random
# sampling of the comma-separated PID_SAMPLEs, each result containing between MIN_PRODUCTS and MAX_PRODUCTS.
# If another response structure needed, this can easily be adjusted within the response-juggler
search-service-1:
image: awagen/response-juggler:0.1.2
deploy:
resources:
limits:
cpus: '4'
memory: 512m
reservations:
cpus: '4'
memory: 512m
user: "1000:1000"
environment:
PORT: 80
NUM_CONNECTIONS: 400
RESPONSE_MAIN_TEMPLATE: "searchresponse.json"
RESPONSE_PARTIAL_IDENT_DOCS: "{{DOCS}}"
RESPONSE_PARTIAL_CONTENT_DOCS: "doc_w_image_and_desc.json"
RESPONSE_PARTIAL_SAMPLER_TYPE_DOCS: "LIST"
RESPONSE_PARTIAL_SAMPLER_MIN_NUM_DOCS: 5
RESPONSE_PARTIAL_SAMPLER_MAX_NUM_DOCS: 20
RESPONSE_FIELD_IDENT_DESCRIPTION: "{{DESCRIPTION}}"
RESPONSE_FIELD_SAMPLER_TYPE_DESCRIPTION: "SINGLE"
RESPONSE_FIELD_SAMPLER_ELEMENT_CAST_DESCRIPTION: "STRING"
RESPONSE_FIELD_SAMPLER_SELECTION_DESCRIPTION: "Awesome looking jeans,What a great TV that is,Great Great Product,Hot as the Summer,Cold as the Winter"
RESPONSE_FIELD_IDENT_IMAGE: "{{IMAGE}}"
RESPONSE_FIELD_SAMPLER_TYPE_IMAGE: "SINGLE"
RESPONSE_FIELD_SAMPLER_ELEMENT_CAST_IMAGE: "STRING"
RESPONSE_FIELD_SAMPLER_SELECTION_IMAGE: "https://i.otto.de/i/otto/3cae2f65-543c-5ec1-be6c-3f88f4341ffe?$$tile_portrait$$&fmt=webp,https://i.otto.de/i/otto/f03260dc-4735-500e-b2f4-8ecef542f012?$$tile_portrait$$&fmt=webp,https://i.otto.de/i/otto/9b2465fe-a566-5705-bcfb-3ee250d9a1e1?$$tile_portrait$$&fmt=webp,https://i.otto.de/i/otto/6afeaec1-815c-530d-ab72-460bdc7e384e?$$tile_portrait$$&fmt=webp,https://i.otto.de/i/otto/a4bc5211-42a4-5e73-8a7e-e2ee391b37ab?$$tile_portrait$$&fmt=webp,https://i.otto.de/i/otto/cba8352a-4302-56f1-aeac-f290367103cc?$$tile_portrait$$&fmt=webp"
RESPONSE_FIELD_IDENT_PID: "{{PID}}"
RESPONSE_FIELD_SAMPLER_TYPE_PID: "SINGLE"
RESPONSE_FIELD_SAMPLER_ELEMENT_CAST_PID: "STRING"
RESPONSE_FIELD_SAMPLER_SELECTION_PID: "p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19"
RESPONSE_FIELD_IDENT_BOOL: "{{BOOL}}"
RESPONSE_FIELD_SAMPLER_TYPE_BOOL: "SINGLE"
RESPONSE_FIELD_SAMPLER_ELEMENT_CAST_BOOL: "BOOL"
RESPONSE_FIELD_SAMPLER_SELECTION_BOOL: "True,False"
RESPONSE_FIELD_IDENT_NUM_FOUND: "{{NUM_FOUND}}"
RESPONSE_FIELD_SAMPLER_TYPE_NUM_FOUND: "SINGLE"
RESPONSE_FIELD_SAMPLER_ELEMENT_CAST_NUM_FOUND: "INT"
RESPONSE_FIELD_SAMPLER_SELECTION_NUM_FOUND: "1,5,6,8,10,12,100,50,200"
RESPONSE_FIELD_IDENT_STRING_VAL1: "{{STRING_VAL1}}"
RESPONSE_FIELD_SAMPLER_TYPE_STRING_VAL1: "SINGLE"
RESPONSE_FIELD_SAMPLER_ELEMENT_CAST_STRING_VAL1: "STRING"
RESPONSE_FIELD_SAMPLER_SELECTION_STRING_VAL1: "p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19"
ports:
- "80:80"
search-service-2:
image: awagen/response-juggler:0.1.2
deploy:
resources:
limits:
cpus: '4'
memory: 512m
reservations:
cpus: '4'
memory: 512m
user: "1000:1000"
environment:
PORT: 81
NUM_CONNECTIONS: 400
# base template
RESPONSE_MAIN_TEMPLATE: "searchresponse.json"
# config for doc sampling
RESPONSE_PARTIAL_IDENT_DOCS: "{{DOCS}}"
RESPONSE_PARTIAL_CONTENT_DOCS: "doc_w_image_and_desc.json"
RESPONSE_PARTIAL_SAMPLER_TYPE_DOCS: "LIST"
RESPONSE_PARTIAL_SAMPLER_MIN_NUM_DOCS: 5
RESPONSE_PARTIAL_SAMPLER_MAX_NUM_DOCS: 20
# config for description sampling
RESPONSE_FIELD_IDENT_DESCRIPTION: "{{DESCRIPTION}}"
RESPONSE_FIELD_SAMPLER_TYPE_DESCRIPTION: "SINGLE"
RESPONSE_FIELD_SAMPLER_ELEMENT_CAST_DESCRIPTION: "STRING"
RESPONSE_FIELD_SAMPLER_SELECTION_DESCRIPTION: "Awesome looking jeans,What a great TV that is,Great Great Product,Hot as the Summer,Cold as the Winter"
# config for image sampling
RESPONSE_FIELD_IDENT_IMAGE: "{{IMAGE}}"
RESPONSE_FIELD_SAMPLER_TYPE_IMAGE: "SINGLE"
RESPONSE_FIELD_SAMPLER_ELEMENT_CAST_IMAGE: "STRING"
RESPONSE_FIELD_SAMPLER_SELECTION_IMAGE: "https://i.otto.de/i/otto/3cae2f65-543c-5ec1-be6c-3f88f4341ffe?$$tile_portrait$$&fmt=webp,https://i.otto.de/i/otto/f03260dc-4735-500e-b2f4-8ecef542f012?$$tile_portrait$$&fmt=webp,https://i.otto.de/i/otto/9b2465fe-a566-5705-bcfb-3ee250d9a1e1?$$tile_portrait$$&fmt=webp,https://i.otto.de/i/otto/6afeaec1-815c-530d-ab72-460bdc7e384e?$$tile_portrait$$&fmt=webp,https://i.otto.de/i/otto/a4bc5211-42a4-5e73-8a7e-e2ee391b37ab?$$tile_portrait$$&fmt=webp,https://i.otto.de/i/otto/cba8352a-4302-56f1-aeac-f290367103cc?$$tile_portrait$$&fmt=webp"
# config for product id sampling
RESPONSE_FIELD_IDENT_PID: "{{PID}}"
RESPONSE_FIELD_SAMPLER_TYPE_PID: "SINGLE"
RESPONSE_FIELD_SAMPLER_ELEMENT_CAST_PID: "STRING"
RESPONSE_FIELD_SAMPLER_SELECTION_PID: "p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19"
# config for bool attribute sampling
RESPONSE_FIELD_IDENT_BOOL: "{{BOOL}}"
RESPONSE_FIELD_SAMPLER_TYPE_BOOL: "SINGLE"
RESPONSE_FIELD_SAMPLER_ELEMENT_CAST_BOOL: "BOOL"
RESPONSE_FIELD_SAMPLER_SELECTION_BOOL: "True,False"
# config for numFound sampling
RESPONSE_FIELD_IDENT_NUM_FOUND: "{{NUM_FOUND}}"
RESPONSE_FIELD_SAMPLER_TYPE_NUM_FOUND: "SINGLE"
RESPONSE_FIELD_SAMPLER_ELEMENT_CAST_NUM_FOUND: "INT"
RESPONSE_FIELD_SAMPLER_SELECTION_NUM_FOUND: "1,5,6,8,10,12,100,50,200"
# config for STRING_VAL1 sampling
RESPONSE_FIELD_IDENT_STRING_VAL1: "{{STRING_VAL1}}"
RESPONSE_FIELD_SAMPLER_TYPE_STRING_VAL1: "SINGLE"
RESPONSE_FIELD_SAMPLER_ELEMENT_CAST_STRING_VAL1: "STRING"
RESPONSE_FIELD_SAMPLER_SELECTION_STRING_VAL1: "p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19"
ports:
- "81:81"
kolibri-watch:
image: awagen/kolibri-watch:0.2.4
environment:
KOLIBRI_BASE_URL: "http://localhost:8001"
ports:
- "82:80"