Merge pull request #37 from Ensembl/vep_support_path

Dynamically generate VEP support paths.

app/.env.sample

@@ -5,3 +5,5 @@ NF_PIPELINE_URL=https://github.com/jonk-ebi/platform-test-pipeline
 NF_WORK_DIR=/path/to/work/dir
 SEQERA_API=https://api.cloud.seqera.io
 UPLOAD_DIRECTORY=/tmpdir
+WEB_METADATA_API=https://beta.ensembl.org/api/metadata/
+VEP_SUPPORT_PATH=/nfs/public/ro/enswbsites/dev/vep/support/

app/core/config.py

@@ -40,21 +40,19 @@
 with open("/data/blast_config.json") as f:
     BLAST_CONFIG = json.load(f)
 
+
 # logging configuration
+
 LOGGING_LEVEL = logging.DEBUG if DEBUG else logging.INFO
 LOGGERS = ("uvicorn.asgi", "uvicorn.access")
 
-log = logging.getLogger("gunicorn.access")
-
 logging.getLogger().handlers = [InterceptHandler()]
 for logger_name in LOGGERS:
     logging_logger = logging.getLogger(logger_name)
     logging_logger.handlers = [InterceptHandler(level=LOGGING_LEVEL)]
 
 logger.configure(handlers=[{"sink": sys.stderr, "level": LOGGING_LEVEL}])
 
-logging.info(BLAST_CONFIG)
-
 # Nextflow Configurations
 NF_TOKEN = config("NF_TOKEN")
 NF_COMPUTE_ENV_ID = config("NF_COMPUTE_ENV_ID")
@@ -64,3 +62,7 @@
 NF_WORKSPACE_ID = config("NF_WORKSPACE_ID")
 
 UPLOAD_DIRECTORY = config("UPLOAD_DIRECTORY", default="/tmpdir")
+WEB_METADATA_API = config(
+    "WEB_METADATA_API", default="https://beta.ensembl.org/api/metadata/"
+)
+VEP_SUPPORT_PATH = config("VEP_SUPPORT_PATH", default="/tmpdir")

app/vep/models/pipeline_model.py

@@ -15,6 +15,8 @@
 from core.logging import InterceptHandler
 import json, logging, os
 
+from vep.utils.web_metadata import get_vep_support_location
+
 logging.getLogger().handlers = [InterceptHandler()]
 
 
@@ -33,9 +35,7 @@ def vep_config_serialiser(self):
         bin_str = f'"bin_size": {self.bin_size}'
         sort_str = f'"sort": {"true" if self.sort else "false"}'
         json_str = (
-            "{" + ", ".join(
-                [vcf_str, config_str, outdir_str, bin_str, sort_str]
-            ) + "}"
+            "{" + ", ".join([vcf_str, config_str, outdir_str, bin_str, sort_str]) + "}"
         )
         return json_str
 
@@ -55,15 +55,19 @@ class PipelineParams(BaseModel):
 
 
 class ConfigIniParams(BaseModel):
+    genome_id: str
     force_overwrite: int = 1
     symbol: bool = False
     biotype: bool = False
     transcript_version: int = 1
     canonical: int = 1
-    gff: str = "/nfs/public/rw/enswbsites/dev/vep/test/genes.gff3.bgz"
-    fasta: str = "/nfs/public/rw/enswbsites/dev/vep/test/unmasked.fa.bgz"
+    gff: str = "genes.gff3.bgz"
+    fasta: str = "unmasked.fa.bgz"
 
     def create_config_ini_file(self, dir):
+        vep_support_location = get_vep_support_location(self.genome_id)
+        self.gff = vep_support_location + self.gff
+        self.fasta = vep_support_location + self.fasta
         symbol = 1 if self.symbol else 0
         biotype = 1 if self.biotype else 0
 
@@ -76,7 +80,6 @@ def create_config_ini_file(self, dir):
 gff {self.gff}
 fasta {self.fasta}
 """
-
         try:
             with open(os.path.join(dir, "config.ini"), "w") as ini_file:
                 ini_file.write(config_ini)

app/vep/utils/web_metadata.py

@@ -0,0 +1,26 @@
+import requests
+
+import logging
+from requests import HTTPError
+
+from core.config import WEB_METADATA_API, VEP_SUPPORT_PATH
+from core.logging import InterceptHandler
+
+logging.getLogger().handlers = [InterceptHandler()]
+
+
+def get_vep_support_location(genome_id):
+    try:
+        response = requests.get(
+            WEB_METADATA_API
+            + "genome/"
+            + genome_id
+            + "/dataset/genebuild/attributes?attribute_names=vep.bgz_location"
+        )
+        response.raise_for_status()
+        return VEP_SUPPORT_PATH + response.json()["attributes"].pop()["value"]
+    except HTTPError as http_error:
+        logging.debug(http_error)
+        raise HTTPError
+    except Exception as e:
+        logging.debug(e)

app/vep/vep_resources.py

@@ -58,10 +58,10 @@ async def submit_vep(request: Request):
         stream_result = await request_streamer.stream()
         vep_job_parameters = request_streamer.parameters.value.decode()
         genome_id = request_streamer.genome_id.value.decode()
-
         vep_job_parameters_dict = json.loads(vep_job_parameters)
-        ini_parameters = ConfigIniParams(**vep_job_parameters_dict)
+        ini_parameters = ConfigIniParams(**vep_job_parameters_dict, genome_id=genome_id)
         ini_file = ini_parameters.create_config_ini_file(request_streamer.temp_dir)
+
         vep_job_config_parameters = VEPConfigParams(
             vcf=request_streamer.filepath,
             vep_config=ini_file.name,

docker-compose.yaml

@@ -1,6 +1,8 @@
 version: '2'
 services:
   tools_api:
+    env_file:
+      - ./app/.env
     build:
       context: .
       dockerfile: Dockerfile