Flex uploader

README.md

@@ -18,28 +18,54 @@ Set the following environment variable in
 * `public.env`:
 
 ```bash
-VCF_FILE=...
+...
+HG_ASSEMBLY=GRCh37 # or GRCh38
+
+# Set if local path or remote "s3://_bucket_/_folder_/_file_.vcf.gz"
+VCF_FILE=schema/small_demo.vcf.gz
 ```
 
-Where `VCF_FILE` can be either a local file (e.g. `path/file.vcf.gz`) or a remote `S3` file (e.g. `s3://any_remote/file.vcf.gz` )
+Where `VCF_FILE` can be either a local file (e.g. `path/file.vcf.gz`) or a remote `S3` file (e.g. `s3://any_remote/file.vcf.gz`)
 
 It's critical that the `VCF_FILE` has along its `tbi` file as well.
 
 * Create `private.env` and add:
 
 ```bash
-AWS_SECRET_ACCESS_KEY=....
-AWS_ACCESS_KEY_ID=....
+APP_ENV=prod # or debug
+
+# Set bucket name and region
+BUCKET=_your_bucket_
+REGION=_region_
+
+# Set below if using remote S3 (AWS, Wasabi etc.) or local (MinIO)
+S3_ACCESS_KEY_ID=...
+S3_SECRET_ACCESS_KEY=...
+
+# Wasabi example
+# S3_ACCESS_KEY_ID=...
+# S3_SECRET_ACCESS_KEY=...
+# ENDPOINT=https://s3.eu-central-1.wasabisys.com
+# REGION=eu-central-1
+
+# MinIO example
+# S3_ACCESS_KEY_ID=minio # change it for your own safety
+# S3_SECRET_ACCESS_KEY=minio123 # change it for your own safety
+# ENDPOINT=http://minio-server:9000
+
+# Set accordingly
+MAIL_SUPPRESS_SEND=false # or true
+MAIL_PASSWORD=...
 ```
 
 Note: do not add single or double quotes around the value as they are preserved.
 
 ### Build and launch the services
 
-This will set up the database and load the demo database.
+This will set up the database and load the demo database running local S3 file service (by [MinIO](https://min.io/)).
 
 ```bash
-docker compose up
+docker-compose -f docker-compose.yml -f docker-compose.minio1.yml up
 ```
 
 If one wants to rebuild fresh images, one can do:

config/nginx1.conf

@@ -0,0 +1,102 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+    }
+
+    upstream console {
+        ip_hash;
+        server minio1:9001;
+    }
+
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+            # proxy_pass http://minio-server:9000;
+            # proxy_set_header Origin http://minio-server:9000;
+            # proxy_hide_header Access-Control-Allow-Origin;
+            # add_header Access-Control-Allow-Origin "$http_origin" always;
+        }
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-NginX-Proxy true;
+
+            # This is necessary to pass the correct IP to be hashed
+            real_ip_header X-Real-IP;
+
+            proxy_connect_timeout 300;
+
+            # To support websocket
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+
+            chunked_transfer_encoding off;
+
+            proxy_pass http://console;
+        }
+    }
+}

config/nginx4.conf

@@ -0,0 +1,104 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+        server minio2:9000;
+        server minio3:9000;
+        server minio4:9000;
+    }
+
+    upstream console {
+        ip_hash;
+        server minio1:9001;
+        server minio2:9001;
+        server minio3:9001;
+        server minio4:9001;
+    }
+
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-NginX-Proxy true;
+
+            # This is necessary to pass the correct IP to be hashed
+            real_ip_header X-Real-IP;
+
+            proxy_connect_timeout 300;
+
+            # To support websocket
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+
+            chunked_transfer_encoding off;
+
+            proxy_pass http://console;
+        }
+    }
+}

docker-compose.minio.debug.yml → docker-compose.minio1.yml

@@ -1,16 +1,16 @@
-# docker-compose -f docker-compose.yml -f docker-compose.minio1.yml up
+# source private.env; export S3_ACCESS_KEY_ID; export S3_SECRET_ACCESS_KEY; docker-compose -f docker-compose.yml -f docker-compose.minio1.yml up
 version: '3.8'
 
 x-minio-common:
    &minio-common
    env_file:
       - private.env
       - public.env
+   environment:
+      MINIO_ROOT_USER: ${S3_ACCESS_KEY_ID}
+      MINIO_ROOT_PASSWORD: ${S3_SECRET_ACCESS_KEY}
    image: quay.io/minio/minio:latest
    command: server --console-address ":9001" /data
-   expose:
-      - "9000"
-      - "9001"
    healthcheck:
       test:
          [
@@ -25,34 +25,23 @@ x-minio-common:
 
 services:
    app:
-      command:
-         [
-            "sh",
-            "-c",
-            "pip3 install -U debugpy -t /tmp && python3 /tmp/debugpy --wait-for-client --listen 0.0.0.0:5678 -m flask run --no-debugger --no-reload --host 0.0.0.0 --port 5000"
-         ]
-      ports:
-         - 5000:5000
-         - 5678:5678
-      environment:
-         - FLASK_APP=application
-
-   db:
-      ports:
-         - 5432:5432
+      depends_on:
+         - db
+         - minio-server
 
    minio1:
       <<: *minio-common
       hostname: minio1
       volumes:
          - data:/data
 
-   nginx:
+   minio-server:
       image: nginx:1.19.2-alpine
-      hostname: nginx
+      hostname: minio-server
       volumes:
-         - ./nginx1.conf:/etc/nginx/nginx.conf:ro
+         - ./config/nginx1.conf:/etc/nginx/nginx.conf:ro
       ports:
+         # needed to access console http://127.0.0.1:9001/login
          - "9000:9000"
          - "9001:9001"
       depends_on: