Allow separate config for worker Part 1 (#376)

* allow separate config for worker

* test tests

* fix test

* create download cache if not exists

* create download cache if not exists

* create resource storage if not exists

* create log folder

* further work on tests

* add TODOs in tests

docker/actinia-core-tests/Dockerfile

@@ -9,6 +9,7 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0
 ENV ACTINIA_CUSTOM_TEST_CFG /etc/default/actinia_test
 # TODO do not set DEFAULT_CONFIG_PATH if this is fixed
 ENV DEFAULT_CONFIG_PATH /etc/default/actinia_test
+ENV GRASS_DATABASE=/tmp/actinia_core/grassdb
 
 # install things only for tests
 RUN apk add redis
@@ -18,16 +19,27 @@ RUN pip3 install iniconfig colorlog
 RUN pip3 uninstall actinia-core -y
 
 # add data for tests
+WORKDIR ${GRASS_DATABASE}
 RUN wget --quiet https://grass.osgeo.org/sampledata/north_carolina/nc_spm_08_micro.zip && \
   unzip nc_spm_08_micro.zip && \
   rm -f nc_spm_08_micro.zip && \
-  mv nc_spm_08_micro /actinia_core/grassdb/nc_spm_08
-RUN grass -e -c 'EPSG:4326' /actinia_core/grassdb/latlong_wgs84
+  mv nc_spm_08_micro nc_spm_08
+RUN grass -e -c 'EPSG:4326' latlong_wgs84
 RUN wget --quiet https://grass.osgeo.org/sampledata/north_carolina/nc_spm_mapset_modis2015_2016_lst_grass8.zip && \
   unzip nc_spm_mapset_modis2015_2016_lst_grass8.zip && \
   rm -f nc_spm_mapset_modis2015_2016_lst_grass8.zip && \
-  mv  modis_lst /actinia_core/grassdb/nc_spm_08/modis_lst
-RUN chown -R 1001:1001 /actinia_core/grassdb/nc_spm_08/modis_lst && chmod -R g+w /actinia_core/grassdb/nc_spm_08/modis_lst
+  mv  modis_lst nc_spm_08/modis_lst
+RUN chown -R 1001:1001 nc_spm_08/modis_lst && chmod -R g+w nc_spm_08/modis_lst
+
+RUN rmdir /actinia_core/grassdb
+RUN rmdir /actinia_core/userdata
+RUN rmdir /actinia_core/resources
+RUN rmdir /actinia_core/workspace/actinia
+RUN rmdir /actinia_core/workspace/download_cache
+RUN rmdir /actinia_core/workspace/temp_db
+RUN rmdir /actinia_core/workspace/tmp
+RUN rmdir /actinia_core/workspace
+RUN rmdir /actinia_core
 
 # install actinia-api
 RUN pip3 install actinia-api@https://github.com/actinia-org/actinia-api/releases/download/${ACTINIA_API_VERSION}/actinia_api-${ACTINIA_API_VERSION}-py3-none-any.whl

docker/actinia-core-tests/actinia-test-noauth.cfg

@@ -1,8 +1,8 @@
 [GRASS]
-grass_database = /actinia_core/grassdb
-grass_user_database = /actinia_core/userdata
-grass_tmp_database = /actinia_core/workspace/temp_db
-grass_resource_dir = /actinia_core/resources
+grass_database = /tmp/actinia_core/grassdb
+grass_user_database = /tmp/actinia_core/userdata
+grass_tmp_database = /tmp/actinia_core/workspace/temp_db
+grass_resource_dir = /tmp/actinia_core/resources
 
 [API]
 force_https_urls = True
@@ -14,13 +14,13 @@ process_time_limt = 60
 process_num_limit = 20
 
 [REDIS]
-worker_logfile = /actinia_core/workspace/tmp/actinia_worker_test.log
+worker_logfile = /tmp/actinia_core/workspace/actinia_worker_test.log
 redis_server_url = localhost
 redis_server_port = 6379
 
 [LOGGING]
 log_level = 1
 
 [MISC]
-tmp_workdir = /actinia_core/workspace/tmp
-download_cache = /actinia_core/workspace/download_cache
+tmp_workdir = /tmp/actinia_core/workspace
+download_cache = /tmp/actinia_core/workspace/download_cache

docker/actinia-core-tests/actinia-test.cfg

@@ -1,8 +1,8 @@
 [GRASS]
-grass_database = /actinia_core/grassdb
-grass_user_database = /actinia_core/userdata
-grass_tmp_database = /actinia_core/workspace/temp_db
-grass_resource_dir = /actinia_core/resources
+grass_database = /tmp/actinia_core/grassdb
+grass_user_database = /tmp/actinia_core/userdata
+grass_tmp_database = /tmp/actinia_core/workspace/temp_db
+grass_resource_dir = /tmp/actinia_core/resources
 
 [API]
 force_https_urls = True
@@ -13,13 +13,13 @@ process_time_limt = 60
 process_num_limit = 20
 
 [REDIS]
-worker_logfile = /actinia_core/workspace/tmp/actinia_worker_test.log
+worker_logfile = /tmp/actinia_core/workspace/actinia_worker_test.log
 redis_server_url = localhost
 redis_server_port = 6379
 
 [LOGGING]
 log_level = 1
 
 [MISC]
-tmp_workdir = /actinia_core/workspace/tmp
-download_cache = /actinia_core/workspace/download_cache
+tmp_workdir = /tmp/actinia_core/workspace
+download_cache = /tmp/actinia_core/workspace/download_cache

src/actinia_core/core/logging_interface.py

@@ -27,6 +27,7 @@
 
 import logging
 import logging.handlers
+import os
 import sys
 import platform
 from datetime import datetime
@@ -118,6 +119,10 @@ def setLogHandler(self, logger, type, format):
         if type == "stdout" and global_config.LOG_INTERFACE == "stdout":
             handler = logging.StreamHandler()
         elif type == "file":
+            logfilepath = global_config.WORKER_LOGFILE.strip(
+                global_config.WORKER_LOGFILE.split("/")[-1]
+            )
+            os.makedirs(logfilepath, exist_ok=True)
             handler = logging.handlers.RotatingFileHandler(
                 global_config.WORKER_LOGFILE, maxBytes=2000000, backupCount=5
             )

src/actinia_core/processing/actinia_processing/ephemeral/download_cache_management.py

@@ -32,7 +32,6 @@
     EphemeralProcessing,
 )
 from actinia_core.core.common.process_object import Process
-from actinia_core.core.common.exceptions import AsyncProcessError
 from actinia_core.models.response_models import (
     StorageResponseModel,
     StorageModel,
@@ -59,37 +58,30 @@ def __init__(self, *args):
     def _execute(self):
         self._setup()
 
-        if os.path.exists(self.user_download_cache_path) and os.path.isdir(
-            self.user_download_cache_path
-        ):
-            executable = "/usr/bin/du"
-            args = ["-sb", self.user_download_cache_path]
+        if not os.path.exists(self.user_download_cache_path):
+            os.mkdir(self.user_download_cache_path)
 
-            self._run_process(
-                Process(
-                    exec_type="exec",
-                    executable=executable,
-                    id="compute_download_cache_size",
-                    executable_params=args,
-                )
-            )
-            print("Disk usage ", self.module_output_log[0]["stdout"])
-            dc_size = int(self.module_output_log[0]["stdout"].split("\t")[0])
-            quota_size = int(
-                self.config.DOWNLOAD_CACHE_QUOTA * 1024 * 1024 * 1024
-            )
+        executable = "/usr/bin/du"
+        args = ["-sb", self.user_download_cache_path]
 
-            model = StorageModel(
-                used=dc_size,
-                free=quota_size - dc_size,
-                quota=quota_size,
-                free_percent=int(100 * (quota_size - dc_size) / quota_size),
+        self._run_process(
+            Process(
+                exec_type="exec",
+                executable=executable,
+                id="compute_download_cache_size",
+                executable_params=args,
             )
-            self.module_results = model
+        )
+        print("Disk usage ", self.module_output_log[0]["stdout"])
+        dc_size = int(self.module_output_log[0]["stdout"].split("\t")[0])
+        quota_size = int(self.config.DOWNLOAD_CACHE_QUOTA * 1024 * 1024 * 1024)
 
-            self.finish_message = "Download cache size successfully computed"
-        else:
-            raise AsyncProcessError(
-                "Download cache directory <%s> does not exist."
-                % self.user_download_cache_path
-            )
+        model = StorageModel(
+            used=dc_size,
+            free=quota_size - dc_size,
+            quota=quota_size,
+            free_percent=int(100 * (quota_size - dc_size) / quota_size),
+        )
+        self.module_results = model
+
+        self.finish_message = "Download cache size successfully computed"

src/actinia_core/processing/actinia_processing/ephemeral/resource_storage_management.py

@@ -32,7 +32,6 @@
     EphemeralProcessing,
 )
 from actinia_core.core.common.process_object import Process
-from actinia_core.core.common.exceptions import AsyncProcessError
 from actinia_core.models.response_models import (
     StorageResponseModel,
     StorageModel,
@@ -59,37 +58,30 @@ def __init__(self, *args):
     def _execute(self):
         self._setup()
 
-        if os.path.exists(self.user_resource_storage_path) and os.path.isdir(
-            self.user_resource_storage_path
-        ):
-            executable = "/usr/bin/du"
-            args = ["-sb", self.user_resource_storage_path]
+        if not os.path.exists(self.user_resource_storage_path):
+            os.mkdir(self.user_resource_storage_path)
 
-            self._run_process(
-                Process(
-                    exec_type="exec",
-                    executable=executable,
-                    id="compute_download_cache_size",
-                    executable_params=args,
-                )
-            )
+        executable = "/usr/bin/du"
+        args = ["-sb", self.user_resource_storage_path]
 
-            dc_size = int(self.module_output_log[0]["stdout"].split("\t")[0])
-            quota_size = int(
-                self.config.GRASS_RESOURCE_QUOTA * 1024 * 1024 * 1024
+        self._run_process(
+            Process(
+                exec_type="exec",
+                executable=executable,
+                id="compute_download_cache_size",
+                executable_params=args,
             )
+        )
 
-            model = StorageModel(
-                used=dc_size,
-                free=quota_size - dc_size,
-                quota=quota_size,
-                free_percent=int(100 * (quota_size - dc_size) / quota_size),
-            )
-            self.module_results = model
+        dc_size = int(self.module_output_log[0]["stdout"].split("\t")[0])
+        quota_size = int(self.config.GRASS_RESOURCE_QUOTA * 1024 * 1024 * 1024)
 
-            self.finish_message = "Resource storage size successfully computed"
-        else:
-            raise AsyncProcessError(
-                "Resource storage directory <%s> does not exist."
-                % self.user_resource_storage_path
-            )
+        model = StorageModel(
+            used=dc_size,
+            free=quota_size - dc_size,
+            quota=quota_size,
+            free_percent=int(100 * (quota_size - dc_size) / quota_size),
+        )
+        self.module_results = model
+
+        self.finish_message = "Resource storage size successfully computed"

src/actinia_core/processing/actinia_processing/ephemeral_processing.py

@@ -40,6 +40,7 @@
 from flask import json
 from requests.auth import HTTPBasicAuth
 
+from actinia_core.core.common.config import global_config, DEFAULT_CONFIG_PATH
 from actinia_core.core.common.process_object import Process
 from actinia_core.core.grass_init import GrassInitializer
 from actinia_core.core.messages_logger import MessageLogger
@@ -141,7 +142,14 @@ def __init__(self, rdc):
         # rdc = ResourceDataContainer()
 
         self.rdc = rdc
-        self.config = self.rdc.config
+        if os.path.exists(DEFAULT_CONFIG_PATH) is True and os.path.isfile(
+            DEFAULT_CONFIG_PATH
+        ):
+            self.config = global_config
+            self.rdc.config = self.config
+        else:
+            self.config = self.rdc.config
+
         self.data = self.rdc.user_data
         self.grass_temp_database = self.config.GRASS_TMP_DATABASE
 

src/actinia_core/testsuite.py

@@ -172,6 +172,13 @@ def setUpClass(cls):
             #     global_config.NUMBER_OF_WORKERS
             # )
 
+        # create configured folders
+        os.makedirs(global_config.GRASS_USER_DATABASE, exist_ok=True)
+        os.makedirs(global_config.GRASS_TMP_DATABASE, exist_ok=True)
+        os.makedirs(global_config.GRASS_RESOURCE_DIR, exist_ok=True)
+        os.makedirs(global_config.TMP_WORKDIR, exist_ok=True)
+        os.makedirs(global_config.DOWNLOAD_CACHE, exist_ok=True)
+
         # Start the redis interface
         redis_args = (
             global_config.REDIS_SERVER_URL,

tests/test_download_cache.py

@@ -52,7 +52,7 @@
 
 class DownloadCacheTestCase(ActiniaResourceTestCaseBase):
     def test_download_cache(self):
-        global_config.DOWNLOAD_CACHE = "/tmp/dcache_tmp"
+        # global_config.DOWNLOAD_CACHE = "/tmp/dcache_tmp"
         global_config.DOWNLOAD_CACHE_QUOTA = 1
         try:
             os.mkdir(global_config.DOWNLOAD_CACHE)
@@ -117,45 +117,49 @@ def test_download_cache(self):
             "free_percent" in json_load(rv.data)["process_results"]
         )
 
-    def test_download_cache_error_1(self):
-        if os.path.isdir("/tmp/dcache_tmp_nope") is True:
-            os.rmdir("/tmp/dcache_tmp_nope")
-
-        global_config.DOWNLOAD_CACHE = "/tmp/dcache_tmp_nope"
-        global_config.DOWNLOAD_CACHE_QUOTA = 1
-
-        rv = self.server.get(
-            URL_PREFIX + "/download_cache", headers=self.admin_auth_header
-        )
-        print(rv.data.decode())
-        self.assertEqual(
-            rv.status_code,
-            400,
-            "HTML status code is wrong %i" % rv.status_code,
-        )
-        self.assertEqual(
-            rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype
-        )
-
-    def test_download_cache_error_2(self):
-        if os.path.isdir("/tmp/dcache_tmp_nope") is True:
-            os.rmdir("/tmp/dcache_tmp_nope")
-
-        global_config.DOWNLOAD_CACHE = "/tmp/dcache_tmp_nope"
-        global_config.DOWNLOAD_CACHE_QUOTA = 1
-
-        rv = self.server.delete(
-            URL_PREFIX + "/download_cache", headers=self.admin_auth_header
-        )
-        print(rv.data.decode())
-        self.assertEqual(
-            rv.status_code,
-            400,
-            "HTML status code is wrong %i" % rv.status_code,
-        )
-        self.assertEqual(
-            rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype
-        )
+    # TODO: configure wrong download cache path differently (via config file)
+    # see https://github.com/actinia-org/actinia-core/issues/422
+    # def test_download_cache_error_1(self):
+    #     if os.path.isdir("/tmp/dcache_tmp_nope") is True:
+    #         os.rmdir("/tmp/dcache_tmp_nope")
+
+    #     global_config.DOWNLOAD_CACHE = "/tmp/dcache_tmp_nope"
+    #     global_config.DOWNLOAD_CACHE_QUOTA = 1
+
+    #     rv = self.server.get(
+    #         URL_PREFIX + "/download_cache", headers=self.admin_auth_header
+    #     )
+    #     print(rv.data.decode())
+    #     self.assertEqual(
+    #         rv.status_code,
+    #         400,
+    #         "HTML status code is wrong %i" % rv.status_code,
+    #     )
+    #     self.assertEqual(
+    #         rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype
+    #     )
+
+    # TODO: configure wrong download cache path differently (via config file)
+    # see https://github.com/actinia-org/actinia-core/issues/422
+    # def test_download_cache_error_2(self):
+    #     if os.path.isdir("/tmp/dcache_tmp_nope") is True:
+    #         os.rmdir("/tmp/dcache_tmp_nope")
+
+    #     global_config.DOWNLOAD_CACHE = "/tmp/dcache_tmp_nope"
+    #     global_config.DOWNLOAD_CACHE_QUOTA = 1
+
+    #     rv = self.server.delete(
+    #         URL_PREFIX + "/download_cache", headers=self.admin_auth_header
+    #     )
+    #     print(rv.data.decode())
+    #     self.assertEqual(
+    #         rv.status_code,
+    #         400,
+    #         "HTML status code is wrong %i" % rv.status_code,
+    #     )
+    #     self.assertEqual(
+    #         rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype
+    #     )
 
 
 if __name__ == "__main__":