diff --git a/README.md b/README.md index 605bf5e1..386f0458 100644 --- a/README.md +++ b/README.md @@ -18,28 +18,54 @@ Set the following environment variable in * `public.env`: ```bash -VCF_FILE=... +... +HG_ASSEMBLY=GRCh37 # or GRCh38 + +# Set if local path or remote "s3://_bucket_/_folder_/_file_.vcf.gz" +VCF_FILE=schema/small_demo.vcf.gz ``` -Where `VCF_FILE` can be either a local file (e.g. `path/file.vcf.gz`) or a remote `S3` file (e.g. `s3://any_remote/file.vcf.gz` ) +Where `VCF_FILE` can be either a local file (e.g. `path/file.vcf.gz`) or a remote `S3` file (e.g. `s3://any_remote/file.vcf.gz`) It's critical that the `VCF_FILE` has along its `tbi` file as well. * Create `private.env` and add: ```bash -AWS_SECRET_ACCESS_KEY=.... -AWS_ACCESS_KEY_ID=.... +APP_ENV=prod # or debug + +# Set bucket name and region +BUCKET=_your_bucket_ +REGION=_region_ + +# Set below if using remote S3 (AWS, Wasabi etc.) or local (MinIO) +S3_ACCESS_KEY_ID=... +S3_SECRET_ACCESS_KEY=... + +# Wasabi example +# S3_ACCESS_KEY_ID=... +# S3_SECRET_ACCESS_KEY=... +# ENDPOINT=https://s3.eu-central-1.wasabisys.com +# REGION=eu-central-1 + +# MinIO example +# S3_ACCESS_KEY_ID=minio # change it for your own safety +# S3_SECRET_ACCESS_KEY=minio123 # change it for your own safety +# ENDPOINT=http://minio-server:9000 + +# Set accordingly +MAIL_SUPPRESS_SEND=false # or true +MAIL_PASSWORD=... ``` Note: do not add single or double quotes around the value as they are preserved. ### Build and launch the services -This will set up the database and load the demo database. +This will set up the database and load the demo database running local S3 file service (by [MinIO](https://min.io/)). ```bash -docker compose up +docker-compose -f docker-compose.yml -f docker-compose.minio1.yml up ``` If one wants to rebuild fresh images, one can do: diff --git a/config/nginx1.conf b/config/nginx1.conf new file mode 100644 index 00000000..88c43202 --- /dev/null +++ b/config/nginx1.conf @@ -0,0 +1,102 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + # include /etc/nginx/conf.d/*.conf; + + upstream minio { + server minio1:9000; + } + + upstream console { + ip_hash; + server minio1:9001; + } + + server { + listen 9000; + listen [::]:9000; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + # proxy_pass http://minio-server:9000; + # proxy_set_header Origin http://minio-server:9000; + # proxy_hide_header Access-Control-Allow-Origin; + # add_header Access-Control-Allow-Origin "$http_origin" always; + } + } + + server { + listen 9001; + listen [::]:9001; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-NginX-Proxy true; + + # This is necessary to pass the correct IP to be hashed + real_ip_header X-Real-IP; + + proxy_connect_timeout 300; + + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + chunked_transfer_encoding off; + + proxy_pass http://console; + } + } +} diff --git a/config/nginx4.conf b/config/nginx4.conf new file mode 100644 index 00000000..faa4681f --- /dev/null +++ b/config/nginx4.conf @@ -0,0 +1,104 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + # include /etc/nginx/conf.d/*.conf; + + upstream minio { + server minio1:9000; + server minio2:9000; + server minio3:9000; + server minio4:9000; + } + + upstream console { + ip_hash; + server minio1:9001; + server minio2:9001; + server minio3:9001; + server minio4:9001; + } + + server { + listen 9000; + listen [::]:9000; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + } + + server { + listen 9001; + listen [::]:9001; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-NginX-Proxy true; + + # This is necessary to pass the correct IP to be hashed + real_ip_header X-Real-IP; + + proxy_connect_timeout 300; + + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + chunked_transfer_encoding off; + + proxy_pass http://console; + } + } +} diff --git a/docker-compose.minio.debug.yml b/docker-compose.minio1.yml similarity index 53% rename from docker-compose.minio.debug.yml rename to docker-compose.minio1.yml index bec67898..6f845a61 100644 --- a/docker-compose.minio.debug.yml +++ b/docker-compose.minio1.yml @@ -1,4 +1,4 @@ -# docker-compose -f docker-compose.yml -f docker-compose.minio1.yml up +# source private.env; export S3_ACCESS_KEY_ID; export S3_SECRET_ACCESS_KEY; docker-compose -f docker-compose.yml -f docker-compose.minio1.yml up version: '3.8' x-minio-common: @@ -6,11 +6,11 @@ x-minio-common: env_file: - private.env - public.env + environment: + MINIO_ROOT_USER: ${S3_ACCESS_KEY_ID} + MINIO_ROOT_PASSWORD: ${S3_SECRET_ACCESS_KEY} image: quay.io/minio/minio:latest command: server --console-address ":9001" /data - expose: - - "9000" - - "9001" healthcheck: test: [ @@ -25,21 +25,9 @@ x-minio-common: services: app: - command: - [ - "sh", - "-c", - "pip3 install -U debugpy -t /tmp && python3 /tmp/debugpy --wait-for-client --listen 0.0.0.0:5678 -m flask run --no-debugger --no-reload --host 0.0.0.0 --port 5000" - ] - ports: - - 5000:5000 - - 5678:5678 - environment: - - FLASK_APP=application - - db: - ports: - - 5432:5432 + depends_on: + - db + - minio-server minio1: <<: *minio-common @@ -47,12 +35,13 @@ services: volumes: - data:/data - nginx: + minio-server: image: nginx:1.19.2-alpine - hostname: nginx + hostname: minio-server volumes: - - ./nginx1.conf:/etc/nginx/nginx.conf:ro + - ./config/nginx1.conf:/etc/nginx/nginx.conf:ro ports: + # needed to access console http://127.0.0.1:9001/login - "9000:9000" - "9001:9001" depends_on: diff --git a/docker-compose.minio4.yml b/docker-compose.minio4.yml new file mode 100644 index 00000000..9c420a48 --- /dev/null +++ b/docker-compose.minio4.yml @@ -0,0 +1,86 @@ +# source private.env; export S3_ACCESS_KEY_ID; export S3_SECRET_ACCESS_KEY; docker-compose -f docker-compose.yml -f docker-compose.minio4.yml up +version: '3.8' + +x-minio-common: + &minio-common + env_file: + - private.env + - public.env + environment: + MINIO_ROOT_USER: ${S3_ACCESS_KEY_ID} + MINIO_ROOT_PASSWORD: ${S3_SECRET_ACCESS_KEY} + image: quay.io/minio/minio:latest + command: server --console-address ":9001" http://minio{1...4}/data{1...2} + healthcheck: + test: + [ + "CMD", + "curl", + "-f", + "http://localhost:9000/minio/health/live" + ] + interval: 30s + timeout: 20s + retries: 3 + +services: + app: + depends_on: + - db + - minio-server + + # starts 4 docker containers running minio server instances. + # using nginx reverse proxy, load balancing, you can access + # it through port 9000. + minio1: + <<: *minio-common + hostname: minio1 + volumes: + - data1-1:/data1 # data1-1 can a physical HD e.g. /mnt/data1 + - data1-2:/data2 + + minio2: + <<: *minio-common + hostname: minio2 + volumes: + - data2-1:/data1 + - data2-2:/data2 + + minio3: + <<: *minio-common + hostname: minio3 + volumes: + - data3-1:/data1 + - data3-2:/data2 + + minio4: + <<: *minio-common + hostname: minio4 + volumes: + - data4-1:/data1 + - data4-2:/data2 + + minio-server: + image: nginx:1.19.2-alpine + hostname: minio-server + volumes: + - ./config/nginx4.conf:/etc/nginx/nginx.conf:ro + ports: + # needed to access console http://127.0.0.1:9001/login + - "9000:9000" + - "9001:9001" + depends_on: + - minio1 + - minio2 + - minio3 + - minio4 + +volumes: + data1-1: + data1-2: + data2-1: + data2-2: + data3-1: + data3-2: + data4-1: + data4-2: diff --git a/docker-compose.yml b/docker-compose.yml index 9d0ab580..0094ec4d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,42 +1,47 @@ version: '3.8' -x-common: &common + +x-env-common: + &env-common env_file: - - public.env - - private.env + - private.env + - public.env + services: app: - <<: *common + <<: *env-common build: context: . image: phenopolis_api command: | sh -c "gunicorn -b 0.0.0.0:5000 --reload --workers=1 --threads=15 application:application" ports: - - 5000:5000 + - 5000:5000 volumes: - - ./:/app + - ./:/app depends_on: - - db + - db db: - <<: *common + <<: *env-common image: postgres:12-alpine volumes: - - db:/var/lib/postgresql/data - - ./:/app - - ./schema/initdb.d/:/docker-entrypoint-initdb.d + - db:/var/lib/postgresql/data + - ./:/app + - ./schema/initdb.d/:/docker-entrypoint-initdb.d + frontend: - <<: *common + <<: *env-common tty: true # give extra colours in term log build: context: frontend image: phenopolis_frontend command: ./entrypoint.sh _redirects_docker ports: - - 8888:8888 + - 8888:8888 # export 8889 to avoid jupyter on 8888 depends_on: - - app + - app volumes: - - ./frontend/:/app + - ./frontend/:/app stdin_open: true + volumes: - db: null + db: diff --git a/frontend/.pnp.cjs b/frontend/.pnp.cjs index 606436db..2dc2989d 100755 --- a/frontend/.pnp.cjs +++ b/frontend/.pnp.cjs @@ -7652,7 +7652,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "packageDependencies": [ ["autoprefixer", "npm:9.8.6"], ["browserslist", "npm:4.16.6"], - ["caniuse-lite", "npm:1.0.30001235"], + ["caniuse-lite", "npm:1.0.30001311"], ["colorette", "npm:1.2.2"], ["normalize-range", "npm:0.1.2"], ["num2fraction", "npm:1.2.2"], @@ -8482,7 +8482,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "packageLocation": "./.yarn/cache/browserslist-npm-4.14.2-7fd5fe4d30-44b5d7a444.zip/node_modules/browserslist/", "packageDependencies": [ ["browserslist", "npm:4.14.2"], - ["caniuse-lite", "npm:1.0.30001235"], + ["caniuse-lite", "npm:1.0.30001311"], ["electron-to-chromium", "npm:1.3.749"], ["escalade", "npm:3.1.1"], ["node-releases", "npm:1.1.72"] @@ -8493,7 +8493,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "packageLocation": "./.yarn/cache/browserslist-npm-4.16.6-a20cef1ca7-3dffc86892.zip/node_modules/browserslist/", "packageDependencies": [ ["browserslist", "npm:4.16.6"], - ["caniuse-lite", "npm:1.0.30001235"], + ["caniuse-lite", "npm:1.0.30001311"], ["colorette", "npm:1.2.2"], ["electron-to-chromium", "npm:1.3.749"], ["escalade", "npm:3.1.1"], @@ -8772,7 +8772,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "packageDependencies": [ ["caniuse-api", "npm:3.0.0"], ["browserslist", "npm:4.16.6"], - ["caniuse-lite", "npm:1.0.30001235"], + ["caniuse-lite", "npm:1.0.30001311"], ["lodash.memoize", "npm:4.1.2"], ["lodash.uniq", "npm:4.5.0"] ], @@ -8780,10 +8780,10 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { }] ]], ["caniuse-lite", [ - ["npm:1.0.30001235", { - "packageLocation": "./.yarn/cache/caniuse-lite-npm-1.0.30001235-56c9d2c91e-4703293ada.zip/node_modules/caniuse-lite/", + ["npm:1.0.30001311", { + "packageLocation": "./.yarn/cache/caniuse-lite-npm-1.0.30001311-84162d801c-a96a2724db.zip/node_modules/caniuse-lite/", "packageDependencies": [ - ["caniuse-lite", "npm:1.0.30001235"] + ["caniuse-lite", "npm:1.0.30001311"] ], "linkType": "HARD", }] @@ -18347,7 +18347,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["postcss-preset-env", "npm:6.7.0"], ["autoprefixer", "npm:9.8.6"], ["browserslist", "npm:4.16.6"], - ["caniuse-lite", "npm:1.0.30001235"], + ["caniuse-lite", "npm:1.0.30001311"], ["css-blank-pseudo", "npm:0.1.4"], ["css-has-pseudo", "npm:0.10.0"], ["css-prefers-color-scheme", "npm:3.1.1"], diff --git a/frontend/.yarn/cache/caniuse-lite-npm-1.0.30001235-56c9d2c91e-4703293ada.zip b/frontend/.yarn/cache/caniuse-lite-npm-1.0.30001235-56c9d2c91e-4703293ada.zip deleted file mode 100644 index 712077ca..00000000 Binary files a/frontend/.yarn/cache/caniuse-lite-npm-1.0.30001235-56c9d2c91e-4703293ada.zip and /dev/null differ diff --git a/frontend/.yarn/cache/caniuse-lite-npm-1.0.30001311-84162d801c-a96a2724db.zip b/frontend/.yarn/cache/caniuse-lite-npm-1.0.30001311-84162d801c-a96a2724db.zip new file mode 100644 index 00000000..134ac80d Binary files /dev/null and b/frontend/.yarn/cache/caniuse-lite-npm-1.0.30001311-84162d801c-a96a2724db.zip differ diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 768ff31b..8a788132 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -5134,9 +5134,9 @@ __metadata: linkType: hard "caniuse-lite@npm:^1.0.0, caniuse-lite@npm:^1.0.30000981, caniuse-lite@npm:^1.0.30001109, caniuse-lite@npm:^1.0.30001125, caniuse-lite@npm:^1.0.30001219": - version: 1.0.30001235 - resolution: "caniuse-lite@npm:1.0.30001235" - checksum: 4703293ada758a8e73e5dae0a988a3247b7a9d80e6d4cad38ca9347f54767fcee4e9e627f05926dde50e9de0fec305c603959ee78a4f8f92bfe0dce309df2b01 + version: 1.0.30001311 + resolution: "caniuse-lite@npm:1.0.30001311" + checksum: a96a2724db91f7d820d7c83d98d6d80f71f4ef5542fd9af49478e6f5057a92d61fbb6c2a2c151e13323c760c7d9f046f422a9b1ac77ab08dc2cacc62bd22f941 languageName: node linkType: hard diff --git a/public.env b/public.env index a1f97ae0..dfb12c32 100644 --- a/public.env +++ b/public.env @@ -6,8 +6,9 @@ PH_DB_USER=phenopolis_api PH_DB_PASSWORD=phenopolis_api PH_DB_PORT=5432 -VCF_FILE=schema/small_demo.vcf.gz - MAIL_USERNAME=no-reply@phenopolis.com HG_ASSEMBLY=GRCh37 + +# Set if local path or remote "s3://_bucket_/_folder_/_file_.vcf.gz" +VCF_FILE=schema/small_demo.vcf.gz diff --git a/tests/test_variants.py b/tests/test_variants.py index 52a824f3..d1b110ec 100644 --- a/tests/test_variants.py +++ b/tests/test_variants.py @@ -42,6 +42,7 @@ def test_variant_genotype_vcf(_admin_client): def test_cyvcf2_S3(_admin_client): from cyvcf2 import VCF + # a valid AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY is needed here vcf_S3 = VCF("s3://3kricegenome/test/test.vcf.gz") # public VCF file assert len(vcf_S3.raw_header) == 559362, "Critical, S3 access not working" diff --git a/views/__init__.py b/views/__init__.py index ffd01805..8b28bbb3 100644 --- a/views/__init__.py +++ b/views/__init__.py @@ -1,5 +1,5 @@ """ -Package to init views +Package views """ import datetime import logging @@ -34,7 +34,7 @@ if APP_ENV in ["prod"]: ENV_LOG_FLAG = False -variant_file = VCF(os.getenv("VCF_FILE")) +variant_file = VCF(os.getenv("VCF_FILE", "schema/small_demo.vcf.gz")) def _configure_logs(): @@ -65,7 +65,7 @@ def _configure_logs(): "root": {"level": log_level, "handlers": ["wsgi"]}, } ) - # add SQLalchemy logs + # add SQLAlchemy logs logging.getLogger("sqlalchemy").addHandler(default_handler) @@ -133,7 +133,6 @@ def _init_sqlalchemy(): # NOTE: These imports must be placed at the end of this file # flake8: noqa E402 - import views.auth import views.autocomplete import views.gene diff --git a/views/auth.py b/views/auth.py index 246e3a0b..36f2d486 100644 --- a/views/auth.py +++ b/views/auth.py @@ -82,6 +82,15 @@ def decorated(*args, **kwargs): return decorated +def requires_user(f): + @wraps(f) + def decorated(*args, **kwargs): + if session.get(USER) != DEMO_USER: + return f(*args, **kwargs) + + return decorated + + @application.route("//login", methods=["POST"]) @application.route("/login", methods=["POST"]) def login(): diff --git a/views/general.py b/views/general.py index a5e0c5fc..55e18834 100644 --- a/views/general.py +++ b/views/general.py @@ -169,7 +169,7 @@ def wrapped_f(*args, **kwargs): def _get_pagination_parameters(): try: offset = int(request.args.get("offset", 0)) - limit = int(request.args.get("limit", 10)) + limit = int(request.args.get("limit", 1500)) except ValueError as e: raise PhenopolisException(str(e), 500) return limit, offset diff --git a/views/individual.py b/views/individual.py index eeac9a19..ffd2f8c1 100644 --- a/views/individual.py +++ b/views/individual.py @@ -40,7 +40,7 @@ def get_all_individuals(): for ind in individuals: a1, a2 = zip(*[x.split("@") for x in sorted(ind["ancestor_observed_features"])]) o1, o2 = zip(*[x.split("@") for x in sorted(ind["observed_features"])]) - # NOTE: casting list in strings just for frotend, but list is better, I guess (Alan) + # NOTE: casting list in strings just for frontend, but list is better, I guess (Alan) ind["ancestor_observed_features"] = ",".join(a1) ind["ancestor_observed_features_names"] = ",".join(a2) ind["observed_features"] = ",".join(o1) @@ -147,12 +147,12 @@ def create_individual(): message = "Individuals were created" ids_new_individuals = [] try: - # generate a new unique id for the individual + # generate a new unique ID for the individual for trio in new_individuals: i, g, f = trio # insert individual db_session.add(i) - # to refresh i and with new id and phenopolis_id, both lines below needed (black magic) + # to refresh i and with new ID and phenopolis_id, both lines below needed (black magic) db_session.query(Individual).count() db_session.refresh(i) # add entry to user_individual @@ -249,7 +249,7 @@ def _individual_complete_view(db_session: Session, config, individual: Individua variants = _get_variants(individual.phenopolis_id) hom_vars = [x for x in variants if "HOM" in x["zigosity"]] het_vars = [x for x in variants if "HET" in x["zigosity"]] - # hom variants + # HOM variants config[0]["rare_homs"]["data"] = hom_vars # rare variants config[0]["rare_variants"]["data"] = het_vars diff --git a/views/upload.py b/views/upload.py index 07c2d661..36bb3a6c 100644 --- a/views/upload.py +++ b/views/upload.py @@ -8,34 +8,54 @@ from flask import jsonify, request from views import application -from views.auth import requires_admin +from views.auth import requires_user from views.exceptions import PhenopolisException -UPLOAD_FOLDER = "upload" +BUCKET = os.getenv("BUCKET", "phenopolis-website-uploads") +REGION = os.getenv("REGION", "eu-west-2") -S3_KEY = os.getenv("VCF_S3_KEY") -SECRET_ACCESS_KEY = os.environ.get("VCF_S3_SECRET") -DOWNLOAD_SIGNED_URL_TIME = 300 +S3_USER = os.getenv("S3_ACCESS_KEY_ID") +S3_PASS = os.getenv("S3_SECRET_ACCESS_KEY") +ENDPOINT = os.getenv("ENDPOINT") -s3_client = boto3.client( +s3_client1 = boto3.client( # for listing and delete "s3", - aws_access_key_id=S3_KEY, - aws_secret_access_key=SECRET_ACCESS_KEY, - config=Config(signature_version="s3v4", region_name="eu-west-2"), + endpoint_url=ENDPOINT, # http://minio-server:9000 works + # endpoint_url="http://localhost:9000", # did not work + # endpoint_url="http://host.docker.internal:9000", # does work + aws_access_key_id=S3_USER, + aws_secret_access_key=S3_PASS, + config=Config(signature_version="s3v4", region_name=REGION), ) +s3_client2 = s3_client1 +if ENDPOINT: + if "minio-server" in ENDPOINT: + s3_client2 = boto3.client( # needs localhost, because of uppy? to presign (upload) and download + "s3", + endpoint_url="http://localhost:9000", # worked + # see https://docs.min.io/docs/how-to-use-aws-sdk-for-python-with-minio-server.html + # endpoint_url="http://minio-server:9000", # does not work + aws_access_key_id=S3_USER, + aws_secret_access_key=S3_PASS, + config=Config(signature_version="s3v4", region_name=REGION), + ) + +try: + s3_client1.create_bucket(Bucket=BUCKET, CreateBucketConfiguration={"LocationConstraint": REGION}) +except Exception: + pass + @application.route("/preSignS3URL", methods=["GET", "POST"]) -@requires_admin +@requires_user def presign_S3(): data = request.get_json() filename = data.get("filename") prefix = data.get("prefix") try: - response = s3_client.generate_presigned_post( - Bucket="phenopolis-website-uploads", Key=prefix + "/" + filename, ExpiresIn=3600 - ) + response = s3_client2.generate_presigned_post(Bucket=BUCKET, Key=prefix + "/" + filename, ExpiresIn=3600) except PhenopolisException as e: application.logger.error(str(e)) return None @@ -44,11 +64,10 @@ def presign_S3(): @application.route("/files/", methods=["GET", "POST"]) -@requires_admin +@requires_user def getUploadedFile(individual_id): try: - response = s3_client.list_objects_v2(Bucket="phenopolis-website-uploads", Prefix=individual_id, MaxKeys=100) - + response = s3_client1.list_objects_v2(Bucket=BUCKET, Prefix=individual_id, MaxKeys=100) if response["KeyCount"] == 0: return jsonify(response), 404 except PhenopolisException as e: @@ -59,23 +78,21 @@ def getUploadedFile(individual_id): @application.route("/files", methods=["DELETE"]) -@requires_admin +@requires_user def delete_file(): data = request.get_json() fileKey = data.get("fileKey") - response = s3_client.delete_object(Bucket="phenopolis-website-uploads", Key=fileKey) + response = s3_client1.delete_object(Bucket=BUCKET, Key=fileKey) return jsonify(message="Delete File Success", response=response), 200 @application.route("/file_download", methods=["POST"]) -@requires_admin +@requires_user def download_file(): data = request.get_json() fileKey = data.get("fileKey") - response = s3_client.generate_presigned_url( - "get_object", - Params={"Bucket": "phenopolis-website-uploads", "Key": fileKey}, - ExpiresIn=DOWNLOAD_SIGNED_URL_TIME, + response = s3_client2.generate_presigned_url( + "get_object", Params={"Bucket": BUCKET, "Key": fileKey}, ExpiresIn=300, ) return jsonify(filename=fileKey, response=response), 200 diff --git a/views/variant.py b/views/variant.py index 767ce2fc..feb2f1d9 100644 --- a/views/variant.py +++ b/views/variant.py @@ -81,6 +81,14 @@ def _get_variants(target: str): Returns: List[dict variant]: empty ([]), one or more variants depending on input target """ + limit, offset = _get_pagination_parameters() + if limit > MAX_PAGE_SIZE: + return ( + jsonify(message=f"The maximum page size for variants is {MAX_PAGE_SIZE}"), + 400, + ) + sql_page = sql.SQL(f"limit {limit} offset {offset}") + if CHROMOSOME_POS_REF_ALT_REGEX.match(target): c, p, r, a = target.split("-") filter = sql.SQL(f"""where v.chrom = '{c}' and v.pos = {p} and v."ref" = '{r}' and v.alt = '{a}'""") @@ -148,7 +156,7 @@ def _get_variants(target: str): """ ) - sqlq = sqlq_main + filter + sqlq_end + sqlq = sqlq_main + filter + sqlq_end + sql_page with get_db() as conn: with conn.cursor() as cur: cur.execute(sqlq, {"hga": HG_ASSEMBLY})