From 093952cb88c435ae27251759180e33a91e7e0499 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 27 Apr 2020 15:23:52 -0400 Subject: [PATCH 1/9] C4-137 some utils scaffolding for indexer deployment --- dcicutils/deployment_utils.py | 26 +++++++++++++++++++----- dcicutils/env_utils.py | 5 +++++ test/test_deployment_utils.py | 37 +++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/dcicutils/deployment_utils.py b/dcicutils/deployment_utils.py index 7bc9504fd..624341ff3 100644 --- a/dcicutils/deployment_utils.py +++ b/dcicutils/deployment_utils.py @@ -30,7 +30,7 @@ def main(): import argparse from dcicutils.env_utils import ( - is_stg_or_prd_env, prod_bucket_env, get_standard_mirror_env, data_set_for_env, + is_stg_or_prd_env, prod_bucket_env, get_standard_mirror_env, data_set_for_env, INDEXER_ENVS ) from dcicutils.misc_utils import PRINT @@ -39,12 +39,13 @@ class Deployer: TEMPLATE_DIR = None INI_FILE_NAME = "production.ini" + INDEXER_ENTRY = 'ENCODED.INDEXER = "true"\n' PYPROJECT_FILE_NAME = None @classmethod def build_ini_file_from_template(cls, template_file_name, init_file_name, bs_env=None, bs_mirror_env=None, s3_bucket_env=None, - data_set=None, es_server=None, es_namespace=None): + data_set=None, es_server=None, es_namespace=None, indexer=False): """ Builds a .ini file from a given template file. @@ -57,6 +58,7 @@ def build_ini_file_from_template(cls, template_file_name, init_file_name, data_set (str): An identifier for data to load (either 'prod' for prd/stg envs, or 'test' for others) es_server (str): The server name (or server:port) for the ElasticSearch server. es_namespace (str): The ElasticSearch namespace to use (probably but not necessarily same as bs_env). + indexer (bool): Whether or not we are building an ini file for an indexer. """ with io.open(init_file_name, 'w') as init_file_fp: cls.build_ini_stream_from_template(template_file_name=template_file_name, @@ -66,7 +68,8 @@ def build_ini_file_from_template(cls, template_file_name, init_file_name, s3_bucket_env=s3_bucket_env, data_set=data_set, es_server=es_server, - es_namespace=es_namespace) + es_namespace=es_namespace, + indexer=indexer) # Ref: https://stackoverflow.com/questions/19911123/how-can-you-get-the-elastic-beanstalk-application-version-in-your-application # noqa: E501 EB_MANIFEST_FILENAME = "/opt/elasticbeanstalk/deploy/manifest" @@ -104,7 +107,7 @@ def get_app_version(cls): # This logic (perhaps most or all of this file) shoul @classmethod def build_ini_stream_from_template(cls, template_file_name, init_file_stream, bs_env=None, bs_mirror_env=None, s3_bucket_env=None, data_set=None, - es_server=None, es_namespace=None): + es_server=None, es_namespace=None, indexer=False): """ Sends output to init_file_stream corresponding to the data noe would want in an ini file for the given template_file_name and available environment variables. @@ -118,13 +121,13 @@ def build_ini_stream_from_template(cls, template_file_name, init_file_stream, data_set: 'test' or 'prod'. Default is 'test' unless bs_env is a staging or production environment. es_server: The name of an es server to use. es_namespace: The namespace to use on the es server. If None, this uses the bs_env. + indexer: Whether or not we are building an ini file for an indexer. Returns: None """ # print("data_set given = ", data_set) - es_server = es_server or os.environ.get('ENCODED_ES_SERVER', "MISSING_ENCODED_ES_SERVER") bs_env = bs_env or os.environ.get("ENCODED_BS_ENV", "MISSING_ENCODED_BS_ENV") bs_mirror_env = bs_mirror_env or os.environ.get("ENCODED_BS_MIRROR_ENV", get_standard_mirror_env(bs_env)) or "" @@ -135,6 +138,7 @@ def build_ini_stream_from_template(cls, template_file_name, init_file_stream, data_set = data_set or os.environ.get("ENCODED_DATA_SET", data_set_for_env(bs_env) or "MISSING_ENCODED_DATA_SET") es_namespace = es_namespace or os.environ.get("ENCODED_ES_NAMESPACE", bs_env) + indexer = indexer or 'ENCODED.INDEXER' in os.environ # set this env variable to deploy an indexer # print("data_set computed = ", data_set) @@ -149,6 +153,12 @@ def build_ini_stream_from_template(cls, template_file_name, init_file_stream, 'ES_NAMESPACE': es_namespace, } + # if we specify an indexer name for bs_env, we did the deployment wrong and should bail + if bs_env in INDEXER_ENVS: + raise RuntimeError("Deployed with bs_env %s, which is an indexer env." + "Re-deploy with the env you want to index and set the 'ENCODED.INDEXER'" + "environment variable." % bs_env) + # We assume these variables are not set, but best to check first. Confusion might result otherwise. for extra_var in extra_vars: if extra_var in os.environ: @@ -172,6 +182,10 @@ def build_ini_stream_from_template(cls, template_file_name, init_file_stream, if not cls.EMPTY_ASSIGNMENT.match(expanded_line): init_file_stream.write(expanded_line) + # if we are an indexer, set the application.indexer option + if indexer: + init_file_stream.write(cls.INDEXER_ENTRY) + finally: for key in extra_vars.keys(): @@ -239,6 +253,8 @@ def main(cls): parser.add_argument("--es_namespace", help="an ElasticSearch namespace", default=None) + parser.add_argument("--indexer", + help="") args = parser.parse_args() template_file_name = cls.environment_template_filename(args.env) ini_file_name = args.target diff --git a/dcicutils/env_utils.py b/dcicutils/env_utils.py index cf4037f4f..2af8aa0e0 100644 --- a/dcicutils/env_utils.py +++ b/dcicutils/env_utils.py @@ -11,6 +11,7 @@ FF_ENV_WEBPROD = 'fourfront-webprod' FF_ENV_WEBPROD2 = 'fourfront-webprod2' FF_ENV_WOLF = 'fourfront-wolf' +FF_ENV_INDEXER = 'fourfront-indexer' # to be used by ELB Indexer CGAP_ENV_DEV = 'fourfront-cgapdev' CGAP_ENV_HOTSEAT = 'fourfront-cgaphotseat' # Maybe not used @@ -22,6 +23,7 @@ CGAP_ENV_WEBPROD = 'fourfront-cgap' # CGAP_ENV_WEBPROD2 is meaningless here. See CGAP_ENV_STAGING. CGAP_ENV_WOLF = 'fourfront-cgapwolf' # Maybe not used +CGAP_ENV_INDEXER = 'cgap-indexer' # to be used by ELB Indexer CGAP_ENV_DEV_NEW = 'cgap-dev' CGAP_ENV_HOTSEAT_NEW = 'cgap-hotseat' @@ -43,6 +45,9 @@ FOURFRONT_STG_OR_PRD_TOKENS = ['webprod', 'blue', 'green'] FOURFRONT_STG_OR_PRD_NAMES = ['staging', 'stagging', 'data'] +# We should know which BS Envs are indexing envs +INDEXER_ENVS = [FF_ENV_INDEXER, CGAP_ENV_INDEXER] + # Done this way because it's safer going forward. CGAP_STG_OR_PRD_TOKENS = [] CGAP_STG_OR_PRD_NAMES = [CGAP_ENV_WEBPROD, CGAP_ENV_PRODUCTION_GREEN, CGAP_ENV_PRODUCTION_BLUE, diff --git a/test/test_deployment_utils.py b/test/test_deployment_utils.py index f80affe8d..f5053ad81 100644 --- a/test/test_deployment_utils.py +++ b/test/test_deployment_utils.py @@ -277,6 +277,43 @@ def now(cls): MockFileStream.reset() + # For this test, we check if the 'indexer' option being set correctly sets the ENCODED.INDEXER option + with mock.patch("os.path.exists") as mock_exists: + mock_exists.return_value = True + with mock.patch("io.open", side_effect=mocked_open): + TestDeployer.build_ini_file_from_template(some_template_file_name, some_ini_file_name, indexer=True) + + assert MockFileStream.FILE_SYSTEM[some_ini_file_name] == [ + '[Foo]', + 'DATABASE = "snow_white"', + 'SOME_URL = "http://user@unittest:6543/"', + 'OOPS = "$NOT_AN_ENV_VAR"', + 'HMMM = "${NOT_AN_ENV_VAR_EITHER}"', + 'SHHH = "my-secret"', + 'VERSION = "v-12345-bundle-version"', + 'PROJECT_VERSION = "11.22.33"', + 'ENCODED.INDEXER = "true"' + ] + + MockFileStream.reset() + + with mock.patch("os.path.exists") as mock_exists: + mock_exists.return_value = True + with mock.patch("io.open", side_effect=mocked_open): + + # for this test, we're going to pretend we deployed with bs_name == 'fourfront-indexer', + # which should throw an exception + def mocked_os_get(val, default): + if val == 'ENCODED_BS_ENV': + return 'fourfront-indexer' + else: + return default + + with mock.patch("os.environ.get", side_effect=mocked_os_get): + with pytest.raises(RuntimeError): + TestDeployer.build_ini_file_from_template(some_template_file_name, + some_ini_file_name, indexer=True) + # Uncomment this for debugging... # assert False, "PASSED" From 7e9a937b127d26472a0687b40c23c322d7e0ba19 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 27 Apr 2020 15:26:12 -0400 Subject: [PATCH 2/9] C4-137 small fix to entry point --- dcicutils/deployment_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dcicutils/deployment_utils.py b/dcicutils/deployment_utils.py index 624341ff3..b9ef54efd 100644 --- a/dcicutils/deployment_utils.py +++ b/dcicutils/deployment_utils.py @@ -254,7 +254,9 @@ def main(cls): help="an ElasticSearch namespace", default=None) parser.add_argument("--indexer", - help="") + help="whether or not to deploy an indexer", + action='store_true', + default=False) args = parser.parse_args() template_file_name = cls.environment_template_filename(args.env) ini_file_name = args.target @@ -263,7 +265,8 @@ def main(cls): cls.build_ini_file_from_template(template_file_name, ini_file_name, bs_env=args.bs_env, bs_mirror_env=args.bs_mirror_env, s3_bucket_env=args.s3_bucket_env, data_set=args.data_set, - es_server=args.es_server, es_namespace=args.es_namespace) + es_server=args.es_server, es_namespace=args.es_namespace, + indexer=args.indexer) except Exception as e: PRINT("Error (%s): %s" % (e.__class__.__name__, e)) sys.exit(1) From 0e2e017a74a8786c5679fba69ad78f355270ceb7 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 27 Apr 2020 16:24:09 -0400 Subject: [PATCH 3/9] C4-141 wrap test_app minimally --- dcicutils/misc_utils.py | 50 ++++++++++++++++++++ poetry.lock | 102 ++++++++++++++++++++++++++++++++++++---- pyproject.toml | 1 + 3 files changed, 145 insertions(+), 8 deletions(-) diff --git a/dcicutils/misc_utils.py b/dcicutils/misc_utils.py index 8a9d8400e..4073bf210 100644 --- a/dcicutils/misc_utils.py +++ b/dcicutils/misc_utils.py @@ -3,6 +3,8 @@ """ import os +import logging +from webtest import TestApp # Using PRINT(...) for debugging, rather than its more familiar lowercase form) for intended programmatic output, @@ -11,6 +13,54 @@ PRINT = print +class VirtualApp(TestApp): + """ Wrapper class for TestApp, which we use a handler to the Encoded Application where we can submit + requests to it simulating a number of conditions, including permissions. + """ + logging.basicConfig() + + + def __init__(self, app, environ): + """ Builds an encoded application, allowing you to submit requests to an encoded application + + :param app: return value of get_app(config_uri, app_name) + :param environ: options to pass to the application. Usually permissions. + """ + self.virtual_app = TestApp(app, environ) + + def get(self, url, **kwargs): + """ Wrapper for TestApp.get that logs the outgoing GET + + :param url: url to GET + :param kwargs: args to pass to the GET + :return: result of GET + """ + logging.info('OUTGOING HTTP GET: %s' % url) + return self.virtual_app.get(url, **kwargs) + + def post_json(self, url, obj, **kwargs): + """ Wrapper for TestApp.post_json that logs the outgoing POST + + :param url: url to POST to + :param obj: object body to POST + :param kwargs: args to pass to the POST + :return: result of POST + """ + logging.info('OUTGOING HTTP POST on url: %s with object: %s' % (url, obj)) + return self.virtual_app.post_json(url, obj, **kwargs) + + def patch_json(self, url, fields, **kwargs): + """ Wrapper for TestApp.patch_json that logs the outgoing PATCH + + :param url: url to PATCH to, should contain an object uuid + :param fields: fields to PATCH on uuid in URL + :param kwargs: args to pass to the PATCH + :return: result of PATCH + """ + logging.info('OUTGOING HTTP PATCH on url: %s with changes: %s' % (url, fields)) + return self.virtual_app.patch_json(url, fields, **kwargs) + + def ignored(*args, **kwargs): """ This is useful for defeating flake warnings. diff --git a/poetry.lock b/poetry.lock index dac7179c7..61835a654 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,16 +31,31 @@ version = "0.4.2" [package.dependencies] requests = ">=0.14.0" +[[package]] +category = "main" +description = "Screen-scraping library" +name = "beautifulsoup4" +optional = false +python-versions = "*" +version = "4.9.0" + +[package.dependencies] +soupsieve = [">1.2", "<2.0"] + +[package.extras] +html5lib = ["html5lib"] +lxml = ["lxml"] + [[package]] category = "main" description = "The AWS SDK for Python" name = "boto3" optional = false python-versions = "*" -version = "1.12.42" +version = "1.12.47" [package.dependencies] -botocore = ">=1.15.42,<1.16.0" +botocore = ">=1.15.47,<1.16.0" jmespath = ">=0.7.1,<1.0.0" s3transfer = ">=0.3.0,<0.4.0" @@ -50,7 +65,7 @@ description = "Low-level, data-driven core of boto 3." name = "botocore" optional = false python-versions = "*" -version = "1.15.42" +version = "1.15.47" [package.dependencies] docutils = ">=0.10,<0.16" @@ -527,6 +542,14 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" version = "1.14.0" +[[package]] +category = "main" +description = "A modern CSS selector implementation for Beautiful Soup." +name = "soupsieve" +optional = false +python-versions = "*" +version = "1.9.5" + [[package]] category = "main" description = "Structured Logging for Python" @@ -586,6 +609,18 @@ brotli = ["brotlipy (>=0.6.0)"] secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=0.14)", "ipaddress"] socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] +[[package]] +category = "main" +description = "Waitress WSGI server" +name = "waitress" +optional = false +python-versions = "*" +version = "1.4.3" + +[package.extras] +docs = ["Sphinx (>=1.8.1)", "docutils", "pylons-sphinx-themes (>=1.0.9)"] +testing = ["nose", "coverage (>=5.0)"] + [[package]] category = "dev" description = "Measures number of Terminal column cells of wide-character codes" @@ -594,6 +629,36 @@ optional = false python-versions = "*" version = "0.1.9" +[[package]] +category = "main" +description = "WSGI request and response object" +name = "webob" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*" +version = "1.8.6" + +[package.extras] +docs = ["Sphinx (>=1.7.5)", "pylons-sphinx-themes"] +testing = ["pytest (>=3.1.0)", "coverage", "pytest-cov", "pytest-xdist"] + +[[package]] +category = "main" +description = "Helper to test WSGI applications" +name = "webtest" +optional = false +python-versions = "*" +version = "2.0.35" + +[package.dependencies] +WebOb = ">=1.2" +beautifulsoup4 = "*" +six = "*" +waitress = ">=0.8.5" + +[package.extras] +docs = ["Sphinx (>=1.8.1)", "docutils", "pylons-sphinx-themes (>=1.0.8)"] +tests = ["nose (<1.3.0)", "coverage", "mock", "pastedeploy", "wsgiproxy2", "pyquery"] + [[package]] category = "dev" description = "Backport of pathlib-compatible object wrapper for zip files" @@ -608,7 +673,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["pathlib2", "unittest2", "jaraco.itertools", "func-timeout"] [metadata] -content-hash = "8de09338da2ff26e20e9dd4834286cbb00ac20ac6c1dd5d20c27bff1cf5d99da" +content-hash = "54f0dfe07783f74fdccacd1ff65649cdfea9eb5f6aa8a1c1bd2aed262cf0db04" python-versions = ">=3.4,<3.7" [metadata.files] @@ -623,13 +688,18 @@ attrs = [ aws-requests-auth = [ {file = "aws-requests-auth-0.4.2.tar.gz", hash = "sha256:112c85fe938a01e28f7e1a87168615b6977b28596362b1dcbafbf4f2cc69f720"}, ] +beautifulsoup4 = [ + {file = "beautifulsoup4-4.9.0-py2-none-any.whl", hash = "sha256:a4bbe77fd30670455c5296242967a123ec28c37e9702a8a81bd2f20a4baf0368"}, + {file = "beautifulsoup4-4.9.0-py3-none-any.whl", hash = "sha256:d4e96ac9b0c3a6d3f0caae2e4124e6055c5dcafde8e2f831ff194c104f0775a0"}, + {file = "beautifulsoup4-4.9.0.tar.gz", hash = "sha256:594ca51a10d2b3443cbac41214e12dbb2a1cd57e1a7344659849e2e20ba6a8d8"}, +] boto3 = [ - {file = "boto3-1.12.42-py2.py3-none-any.whl", hash = "sha256:c205c9d69beb43f1dee6f8c30029a418afe1f82fc52a254d9f3b5ab24ee5dd00"}, - {file = "boto3-1.12.42.tar.gz", hash = "sha256:bd005143eadea91dcba536caffcdd19d9a4dbefa7f59ddd503ef0ef2e5079c36"}, + {file = "boto3-1.12.47-py2.py3-none-any.whl", hash = "sha256:7f8b822e383c0d7656488d3b6fdc3e9c42a56fab3ed1a27c2cbc65876093cb21"}, + {file = "boto3-1.12.47.tar.gz", hash = "sha256:84daba6d2f0c8d55477ba0b8196ffa7eb7f79d9099f768ac93eb968955877a3f"}, ] botocore = [ - {file = "botocore-1.15.42-py2.py3-none-any.whl", hash = "sha256:1b7730de543a751c2491f1510688f3c34a8b9669998d8b88f8facf6c3be3c790"}, - {file = "botocore-1.15.42.tar.gz", hash = "sha256:2ce77c2b11253b64a3d7ec0aa696c064d6ed83c32e6288fc2d59f485f8119828"}, + {file = "botocore-1.15.47-py2.py3-none-any.whl", hash = "sha256:cceeb6d2a1bbbd062ab54552ded5065a12b14e845aa35613fc91fd68312020c0"}, + {file = "botocore-1.15.47.tar.gz", hash = "sha256:6c6e9db7a6e420431794faee111923e4627b4920d4d9d8b16e1a578a389b2283"}, ] certifi = [ {file = "certifi-2020.4.5.1-py2.py3-none-any.whl", hash = "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304"}, @@ -836,6 +906,10 @@ six = [ {file = "six-1.14.0-py2.py3-none-any.whl", hash = "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"}, {file = "six-1.14.0.tar.gz", hash = "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a"}, ] +soupsieve = [ + {file = "soupsieve-1.9.5-py2.py3-none-any.whl", hash = "sha256:bdb0d917b03a1369ce964056fc195cfdff8819c40de04695a80bc813c3cfa1f5"}, + {file = "soupsieve-1.9.5.tar.gz", hash = "sha256:e2c1c5dee4a1c36bcb790e0fabd5492d874b8ebd4617622c4f6a731701060dda"}, +] structlog = [ {file = "structlog-19.2.0-py2.py3-none-any.whl", hash = "sha256:6640e6690fc31d5949bc614c1a630464d3aaa625284aeb7c6e486c3010d73e12"}, {file = "structlog-19.2.0.tar.gz", hash = "sha256:4287058cf4ce1a59bc5dea290d6386d37f29a37529c9a51cdf7387e51710152b"}, @@ -856,10 +930,22 @@ urllib3 = [ {file = "urllib3-1.25.9-py2.py3-none-any.whl", hash = "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"}, {file = "urllib3-1.25.9.tar.gz", hash = "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527"}, ] +waitress = [ + {file = "waitress-1.4.3-py2.py3-none-any.whl", hash = "sha256:77ff3f3226931a1d7d8624c5371de07c8e90c7e5d80c5cc660d72659aaf23f38"}, + {file = "waitress-1.4.3.tar.gz", hash = "sha256:045b3efc3d97c93362173ab1dfc159b52cfa22b46c3334ffc805dbdbf0e4309e"}, +] wcwidth = [ {file = "wcwidth-0.1.9-py2.py3-none-any.whl", hash = "sha256:cafe2186b3c009a04067022ce1dcd79cb38d8d65ee4f4791b8888d6599d1bbe1"}, {file = "wcwidth-0.1.9.tar.gz", hash = "sha256:ee73862862a156bf77ff92b09034fc4825dd3af9cf81bc5b360668d425f3c5f1"}, ] +webob = [ + {file = "WebOb-1.8.6-py2.py3-none-any.whl", hash = "sha256:a3c89a8e9ba0aeb17382836cdb73c516d0ecf6630ec40ec28288f3ed459ce87b"}, + {file = "WebOb-1.8.6.tar.gz", hash = "sha256:aa3a917ed752ba3e0b242234b2a373f9c4e2a75d35291dcbe977649bd21fd108"}, +] +webtest = [ + {file = "WebTest-2.0.35-py2.py3-none-any.whl", hash = "sha256:44ddfe99b5eca4cf07675e7222c81dd624d22f9a26035d2b93dc8862dc1153c6"}, + {file = "WebTest-2.0.35.tar.gz", hash = "sha256:aac168b5b2b4f200af4e35867cf316712210e3d5db81c1cbdff38722647bb087"}, +] zipp = [ {file = "zipp-1.2.0-py2.py3-none-any.whl", hash = "sha256:e0d9e63797e483a30d27e09fffd308c59a700d365ec34e93cc100844168bf921"}, {file = "zipp-1.2.0.tar.gz", hash = "sha256:c70410551488251b0fee67b460fb9a536af8d6f9f008ad10ac51f615b6a521b1"}, diff --git a/pyproject.toml b/pyproject.toml index f35edc2d4..85d48be9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ urllib3 = "^1.24.3" structlog = "^19.2.0" requests = "^2.21.0" toml = ">=0.10.0,<1" +webtest = "^2.0.34" [tool.poetry.dev-dependencies] pytest = ">=4.5.0" From 224f6749a7b616a3d1c4d5dc7a43f073ff0ad734 Mon Sep 17 00:00:00 2001 From: Kent Pitman Date: Mon, 27 Apr 2020 17:18:54 -0400 Subject: [PATCH 4/9] Make some proposed changes to resolve my code review comemnts. --- dcicutils/deployment_utils.py | 12 +-- pyproject.toml | 2 +- test/ini_files/blue.ini | 3 +- test/ini_files/cg_any.ini | 3 +- test/ini_files/cgap.ini | 3 +- test/ini_files/cgapdev.ini | 3 +- test/ini_files/cgaptest.ini | 3 +- test/ini_files/cgapwolf.ini | 3 +- test/ini_files/ff_any.ini | 3 +- test/ini_files/green.ini | 3 +- test/ini_files/mastertest.ini | 3 +- test/ini_files/webdev.ini | 3 +- test/ini_files/webprod.ini | 3 +- test/ini_files/webprod2.ini | 3 +- test/test_deployment_utils.py | 159 +++++++++++++++++++++++----------- 15 files changed, 129 insertions(+), 80 deletions(-) diff --git a/dcicutils/deployment_utils.py b/dcicutils/deployment_utils.py index b9ef54efd..fc50507b8 100644 --- a/dcicutils/deployment_utils.py +++ b/dcicutils/deployment_utils.py @@ -39,7 +39,6 @@ class Deployer: TEMPLATE_DIR = None INI_FILE_NAME = "production.ini" - INDEXER_ENTRY = 'ENCODED.INDEXER = "true"\n' PYPROJECT_FILE_NAME = None @classmethod @@ -138,7 +137,11 @@ def build_ini_stream_from_template(cls, template_file_name, init_file_stream, data_set = data_set or os.environ.get("ENCODED_DATA_SET", data_set_for_env(bs_env) or "MISSING_ENCODED_DATA_SET") es_namespace = es_namespace or os.environ.get("ENCODED_ES_NAMESPACE", bs_env) - indexer = indexer or 'ENCODED.INDEXER' in os.environ # set this env variable to deploy an indexer + # Set ENCODED_INDEXER to 'true' to deploy an indexer. + # If the value is missing, the empty string, or any other thing besides 'true' (in any case), + # this value will default to the empty string, causing the line not to appear in the output file + # because there is a special case that suppresses output of empty values. -kmp 27-Apr-2020 + indexer = "true" if indexer or os.environ.get('ENCODED_INDEXER', "false").upper() == "TRUE" else "" # print("data_set computed = ", data_set) @@ -151,6 +154,7 @@ def build_ini_stream_from_template(cls, template_file_name, init_file_stream, 'S3_BUCKET_ENV': s3_bucket_env, 'DATA_SET': data_set, 'ES_NAMESPACE': es_namespace, + 'INDEXER': indexer, } # if we specify an indexer name for bs_env, we did the deployment wrong and should bail @@ -182,10 +186,6 @@ def build_ini_stream_from_template(cls, template_file_name, init_file_stream, if not cls.EMPTY_ASSIGNMENT.match(expanded_line): init_file_stream.write(expanded_line) - # if we are an indexer, set the application.indexer option - if indexer: - init_file_stream.write(cls.INDEXER_ENTRY) - finally: for key in extra_vars.keys(): diff --git a/pyproject.toml b/pyproject.toml index f35edc2d4..af60bdbe0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "0.17.0" +version = "0.18.0" description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" diff --git a/test/ini_files/blue.ini b/test/ini_files/blue.ini index 6834af971..6cf8c5551 100644 --- a/test/ini_files/blue.ini +++ b/test/ini_files/blue.ini @@ -7,7 +7,6 @@ blob_bucket = elasticbeanstalk-fourfront-webprod-blobs system_bucket = elasticbeanstalk-fourfront-webprod-system # blob_store_profile_name = encoded-4dn-files accession_factory = encoded.server_defaults.enc_accession -indexer.processes = $(python3 -c 'import multiprocessing; print(max(1, multiprocessing.cpu_count() - 2));') elasticsearch.server = search-fourfront-blue-xkkzdrxkrunz35shbemkgrmhku.us-east-1.es.amazonaws.com:80 snovault.app_version = ask-pip env.name = fourfront-blue @@ -15,7 +14,7 @@ mirror.env.name = fourfront-green encoded_version = ${PROJECT_VERSION} eb_app_version = ${APP_VERSION} mpindexer = true -indexer = true +indexer = ${INDEXER} indexer.namespace = fourfront-blue elasticsearch.aws_auth = true production = true diff --git a/test/ini_files/cg_any.ini b/test/ini_files/cg_any.ini index 74f2e4f78..c360e5896 100644 --- a/test/ini_files/cg_any.ini +++ b/test/ini_files/cg_any.ini @@ -7,7 +7,6 @@ blob_bucket = elasticbeanstalk-${BS_ENV}-blobs system_bucket = elasticbeanstalk-${BS_ENV}-system # blob_store_profile_name = encoded-4dn-files accession_factory = encoded.server_defaults.enc_accession -indexer.processes = $(python3 -c 'import multiprocessing; print(max(1, multiprocessing.cpu_count() - 2));') elasticsearch.server = ${ES_SERVER} snovault.app_version = ask-pip env.name = ${BS_ENV} @@ -15,7 +14,7 @@ indexer.namespace = ${ES_NAMESPACE} encoded_version = ${PROJECT_VERSION} eb_app_version = ${APP_VERSION} mpindexer = true -indexer = true +indexer = ${INDEXER} elasticsearch.aws_auth = true production = true diff --git a/test/ini_files/cgap.ini b/test/ini_files/cgap.ini index a6e9dc2bd..c4468ee0f 100644 --- a/test/ini_files/cgap.ini +++ b/test/ini_files/cgap.ini @@ -7,7 +7,6 @@ blob_bucket = elasticbeanstalk-fourfront-cgap-blobs system_bucket = elasticbeanstalk-fourfront-cgap-system # blob_store_profile_name = encoded-4dn-files accession_factory = encoded.server_defaults.enc_accession -indexer.processes = $(python3 -c 'import multiprocessing; print(max(1, multiprocessing.cpu_count() - 2));') elasticsearch.server = search-fourfront-cgap-ewf7r7u2nq3xkgyozdhns4bkni.us-east-1.es.amazonaws.com:80 snovault.app_version = ask-pip env.name = fourfront-cgap @@ -15,7 +14,7 @@ indexer.namespace = fourfront-cgap encoded_version = ${PROJECT_VERSION} eb_app_version = ${APP_VERSION} mpindexer = true -indexer = true +indexer = ${INDEXER} elasticsearch.aws_auth = true production = true diff --git a/test/ini_files/cgapdev.ini b/test/ini_files/cgapdev.ini index 397d717c8..5b59904af 100644 --- a/test/ini_files/cgapdev.ini +++ b/test/ini_files/cgapdev.ini @@ -7,7 +7,6 @@ blob_bucket = elasticbeanstalk-fourfront-cgapdev-blobs system_bucket = elasticbeanstalk-fourfront-cgapdev-system # blob_store_profile_name = encoded-4dn-files accession_factory = encoded.server_defaults.enc_accession -indexer.processes = $(python3 -c 'import multiprocessing; print(max(1, multiprocessing.cpu_count() - 2));') elasticsearch.server = search-fourfront-cgapdev-gnv2sgdngkjbcemdadmaoxcsae.us-east-1.es.amazonaws.com:80 snovault.app_version = ask-pip env.name = fourfront-cgapdev @@ -15,7 +14,7 @@ indexer.namespace = fourfront-cgapdev encoded_version = ${PROJECT_VERSION} eb_app_version = ${APP_VERSION} mpindexer = true -indexer = true +indexer = ${INDEXER} elasticsearch.aws_auth = true production = true diff --git a/test/ini_files/cgaptest.ini b/test/ini_files/cgaptest.ini index d3f7836e7..0025166ec 100644 --- a/test/ini_files/cgaptest.ini +++ b/test/ini_files/cgaptest.ini @@ -7,7 +7,6 @@ blob_bucket = elasticbeanstalk-fourfront-cgaptest-blobs system_bucket = elasticbeanstalk-fourfront-cgaptest-system # blob_store_profile_name = encoded-4dn-files accession_factory = encoded.server_defaults.enc_accession -indexer.processes = $(python3 -c 'import multiprocessing; print(max(1, multiprocessing.cpu_count() - 2));') elasticsearch.server = search-fourfront-cgaptest-dxiczz2zv7f3nshshvevcvmpmy.us-east-1.es.amazonaws.com:80 snovault.app_version = ask-pip env.name = fourfront-cgaptest @@ -15,7 +14,7 @@ indexer.namespace = fourfront-cgaptest encoded_version = ${PROJECT_VERSION} eb_app_version = ${APP_VERSION} mpindexer = true -indexer = true +indexer = ${INDEXER} elasticsearch.aws_auth = true production = true diff --git a/test/ini_files/cgapwolf.ini b/test/ini_files/cgapwolf.ini index 40e240ed8..a7bc06938 100644 --- a/test/ini_files/cgapwolf.ini +++ b/test/ini_files/cgapwolf.ini @@ -7,7 +7,6 @@ blob_bucket = elasticbeanstalk-fourfront-cgapwolf-blobs system_bucket = elasticbeanstalk-fourfront-cgapwolf-system # blob_store_profile_name = encoded-4dn-files accession_factory = encoded.server_defaults.enc_accession -indexer.processes = $(python3 -c 'import multiprocessing; print(max(1, multiprocessing.cpu_count() - 2));') elasticsearch.server = search-fourfront-cgapwolf-r5kkbokabymtguuwjzspt2kiqa.us-east-1.es.amazonaws.com:80 snovault.app_version = ask-pip env.name = fourfront-cgapwolf @@ -15,7 +14,7 @@ indexer.namespace = fourfront-cgapwolf encoded_version = ${PROJECT_VERSION} eb_app_version = ${APP_VERSION} mpindexer = true -indexer = true +indexer = ${INDEXER} elasticsearch.aws_auth = true production = true diff --git a/test/ini_files/ff_any.ini b/test/ini_files/ff_any.ini index 07d2f658e..fdd98ba81 100644 --- a/test/ini_files/ff_any.ini +++ b/test/ini_files/ff_any.ini @@ -7,7 +7,6 @@ blob_bucket = elasticbeanstalk-${S3_BUCKET_ENV}-blobs system_bucket = elasticbeanstalk-${S3_BUCKET_ENV}-system # blob_store_profile_name = encoded-4dn-files accession_factory = encoded.server_defaults.enc_accession -indexer.processes = $(python3 -c 'import multiprocessing; print(max(1, multiprocessing.cpu_count() - 2));') elasticsearch.server = ${ES_SERVER} snovault.app_version = ask-pip env.name = ${BS_ENV} @@ -15,7 +14,7 @@ mirror.env.name = ${BS_MIRROR_ENV} encoded_version = ${PROJECT_VERSION} eb_app_version = ${APP_VERSION} mpindexer = true -indexer = true +indexer = ${INDEXER} indexer.namespace = ${ES_NAMESPACE} elasticsearch.aws_auth = true production = true diff --git a/test/ini_files/green.ini b/test/ini_files/green.ini index 3caf774b1..276f6efef 100644 --- a/test/ini_files/green.ini +++ b/test/ini_files/green.ini @@ -7,7 +7,6 @@ blob_bucket = elasticbeanstalk-fourfront-webprod-blobs system_bucket = elasticbeanstalk-fourfront-webprod-system # blob_store_profile_name = encoded-4dn-files accession_factory = encoded.server_defaults.enc_accession -indexer.processes = $(python3 -c 'import multiprocessing; print(max(1, multiprocessing.cpu_count() - 2));') elasticsearch.server = search-fourfront-green-cghpezl64x4uma3etijfknh7ja.us-east-1.es.amazonaws.com:80 snovault.app_version = ask-pip env.name = fourfront-green @@ -15,7 +14,7 @@ mirror.env.name = fourfront-blue encoded_version = ${PROJECT_VERSION} eb_app_version = ${APP_VERSION} mpindexer = true -indexer = true +indexer = ${INDEXER} indexer.namespace = fourfront-green elasticsearch.aws_auth = true production = true diff --git a/test/ini_files/mastertest.ini b/test/ini_files/mastertest.ini index cec298803..bea68e315 100644 --- a/test/ini_files/mastertest.ini +++ b/test/ini_files/mastertest.ini @@ -7,14 +7,13 @@ blob_bucket = elasticbeanstalk-fourfront-mastertest-blobs system_bucket = elasticbeanstalk-fourfront-mastertest-system # blob_store_profile_name = encoded-4dn-files accession_factory = encoded.server_defaults.enc_accession -indexer.processes = $(python3 -c 'import multiprocessing; print(max(1, multiprocessing.cpu_count() - 2));') elasticsearch.server = search-fourfront-mastertest-wusehbixktyxtbagz5wzefffp4.us-east-1.es.amazonaws.com:80 snovault.app_version = ask-pip env.name = fourfront-mastertest encoded_version = ${PROJECT_VERSION} eb_app_version = ${APP_VERSION} mpindexer = true -indexer = true +indexer = ${INDEXER} indexer.namespace = fourfront-mastertest elasticsearch.aws_auth = true production = true diff --git a/test/ini_files/webdev.ini b/test/ini_files/webdev.ini index 1125e2f2b..578b81d9f 100644 --- a/test/ini_files/webdev.ini +++ b/test/ini_files/webdev.ini @@ -7,14 +7,13 @@ blob_bucket = elasticbeanstalk-fourfront-webdev-blobs system_bucket = elasticbeanstalk-fourfront-webdev-system # blob_store_profile_name = encoded-4dn-files accession_factory = encoded.server_defaults.enc_accession -indexer.processes = $(python3 -c 'import multiprocessing; print(max(1, multiprocessing.cpu_count() - 2));') elasticsearch.server = search-fourfront-webdev-5uqlmdvvshqew46o46kcc2lxmy.us-east-1.es.amazonaws.com:80 snovault.app_version = ask-pip env.name = fourfront-webdev encoded_version = ${PROJECT_VERSION} eb_app_version = ${APP_VERSION} mpindexer = true -indexer = true +indexer = ${INDEXER} indexer.namespace = fourfront-webdev elasticsearch.aws_auth = true production = true diff --git a/test/ini_files/webprod.ini b/test/ini_files/webprod.ini index 41561ff3c..271f1cb7d 100644 --- a/test/ini_files/webprod.ini +++ b/test/ini_files/webprod.ini @@ -7,7 +7,6 @@ blob_bucket = elasticbeanstalk-fourfront-webprod-blobs system_bucket = elasticbeanstalk-fourfront-webprod-system # blob_store_profile_name = encoded-4dn-files accession_factory = encoded.server_defaults.enc_accession -indexer.processes = $(python3 -c 'import multiprocessing; print(max(1, multiprocessing.cpu_count() - 2));') elasticsearch.server = search-fourfront-webprod-hmrrlalm4ifyhl4bzbvl73hwv4.us-east-1.es.amazonaws.com:80 snovault.app_version = ask-pip env.name = fourfront-webprod @@ -15,7 +14,7 @@ mirror.env.name = fourfront-webprod2 encoded_version = ${PROJECT_VERSION} eb_app_version = ${APP_VERSION} mpindexer = true -indexer = true +indexer = ${INDEXER} indexer.namespace = fourfront-webprod elasticsearch.aws_auth = true production = true diff --git a/test/ini_files/webprod2.ini b/test/ini_files/webprod2.ini index 60f344e14..9c8b30db3 100644 --- a/test/ini_files/webprod2.ini +++ b/test/ini_files/webprod2.ini @@ -7,7 +7,6 @@ blob_bucket = elasticbeanstalk-fourfront-webprod-blobs system_bucket = elasticbeanstalk-fourfront-webprod-system # blob_store_profile_name = encoded-4dn-files accession_factory = encoded.server_defaults.enc_accession -indexer.processes = $(python3 -c 'import multiprocessing; print(max(1, multiprocessing.cpu_count() - 2));') elasticsearch.server = search-fourfront-webprod2-fkav4x4wjvhgejtcg6ilrmczpe.us-east-1.es.amazonaws.com:80 snovault.app_version = ask-pip env.name = fourfront-webprod2 @@ -15,7 +14,7 @@ mirror.env.name = fourfront-webprod encoded_version = ${PROJECT_VERSION} eb_app_version = ${APP_VERSION} mpindexer = true -indexer = true +indexer = ${INDEXER} indexer.namespace = fourfront-webprod2 elasticsearch.aws_auth = true production = true diff --git a/test/test_deployment_utils.py b/test/test_deployment_utils.py index f5053ad81..02118a1ff 100644 --- a/test/test_deployment_utils.py +++ b/test/test_deployment_utils.py @@ -153,13 +153,14 @@ def mocked_open(filename, mode='r', encoding=None): print("reading mocked TEMPLATE FILE", some_ini_file_name) return StringIO( '[Foo]\n' - 'DATABASE = "${RDS_DB_NAME}"\n' - 'SOME_URL = "http://${RDS_USERNAME}@$RDS_HOSTNAME:$RDS_PORT/"\n' - 'OOPS = "$NOT_AN_ENV_VAR"\n' - 'HMMM = "${NOT_AN_ENV_VAR_EITHER}"\n' - 'SHHH = "$RDS_PASSWORD"\n' - 'VERSION = "${APP_VERSION}"\n' - 'PROJECT_VERSION = "${PROJECT_VERSION}"\n' + 'database = "${RDS_DB_NAME}"\n' + 'some_url = "http://${RDS_USERNAME}@$RDS_HOSTNAME:$RDS_PORT/"\n' + 'oops = "$NOT_AN_ENV_VAR"\n' + 'hmmm = "${NOT_AN_ENV_VAR_EITHER}"\n' + 'shhh = "$RDS_PASSWORD"\n' + 'version = "${APP_VERSION}"\n' + 'project_version = "${PROJECT_VERSION}"\n' + 'indexer = ${INDEXER}' ) elif filename == TestDeployer.PYPROJECT_FILE_NAME: @@ -201,13 +202,13 @@ def mocked_exists(filename): # all the "%" substitutions would have to be on the final line, not line-by-line where needed. assert MockFileStream.FILE_SYSTEM[some_ini_file_name] == [ '[Foo]', - 'DATABASE = "snow_white"', - 'SOME_URL = "http://user@unittest:6543/"', - 'OOPS = "$NOT_AN_ENV_VAR"', - 'HMMM = "${NOT_AN_ENV_VAR_EITHER}"', - 'SHHH = "my-secret"', - 'VERSION = "%s"' % MOCKED_BUNDLE_VERSION, - 'PROJECT_VERSION = "%s"' % MOCKED_PROJECT_VERSION, + 'database = "snow_white"', + 'some_url = "http://user@unittest:6543/"', + 'oops = "$NOT_AN_ENV_VAR"', + 'hmmm = "${NOT_AN_ENV_VAR_EITHER}"', + 'shhh = "my-secret"', + 'version = "%s"' % MOCKED_BUNDLE_VERSION, + 'project_version = "%s"' % MOCKED_PROJECT_VERSION, ] MockFileStream.reset() @@ -226,13 +227,13 @@ def mocked_exists(filename): assert MockFileStream.FILE_SYSTEM[some_ini_file_name] == [ '[Foo]', - 'DATABASE = "snow_white"', - 'SOME_URL = "http://user@unittest:6543/"', - 'OOPS = "$NOT_AN_ENV_VAR"', - 'HMMM = "${NOT_AN_ENV_VAR_EITHER}"', - 'SHHH = "my-secret"', - 'VERSION = "%s"' % MOCKED_LOCAL_GIT_VERSION, # This is the result of no manifest file existing - 'PROJECT_VERSION = "%s"' % MOCKED_PROJECT_VERSION, + 'database = "snow_white"', + 'some_url = "http://user@unittest:6543/"', + 'oops = "$NOT_AN_ENV_VAR"', + 'hmmm = "${NOT_AN_ENV_VAR_EITHER}"', + 'shhh = "my-secret"', + 'version = "%s"' % MOCKED_LOCAL_GIT_VERSION, # This is the result of no manifest file existing + 'project_version = "%s"' % MOCKED_PROJECT_VERSION, ] MockFileStream.reset() @@ -266,17 +267,63 @@ def now(cls): assert MockFileStream.FILE_SYSTEM[some_ini_file_name] == [ '[Foo]', - 'DATABASE = "snow_white"', - 'SOME_URL = "http://user@unittest:6543/"', - 'OOPS = "$NOT_AN_ENV_VAR"', - 'HMMM = "${NOT_AN_ENV_VAR_EITHER}"', - 'SHHH = "my-secret"', - 'VERSION = "unknown-version-at-20010203045506000000"', # We mocked datetime.datetime.now() to get this - 'PROJECT_VERSION = "%s"' % MOCKED_PROJECT_VERSION, + 'database = "snow_white"', + 'some_url = "http://user@unittest:6543/"', + 'oops = "$NOT_AN_ENV_VAR"', + 'hmmm = "${NOT_AN_ENV_VAR_EITHER}"', + 'shhh = "my-secret"', + 'version = "unknown-version-at-20010203045506000000"', # We mocked datetime.datetime.now() to get this + 'project_version = "%s"' % MOCKED_PROJECT_VERSION, ] MockFileStream.reset() + for truth in ["TRUE", "True", "true"]: + + # For this test, we check if the 'indexer' option being set correctly sets the ENCODED.INDEXER option + with mock.patch("os.path.exists") as mock_exists: + mock_exists.return_value = True + with mock.patch("io.open", side_effect=mocked_open): + with override_environ(ENCODED_INDEXER=truth): + TestDeployer.build_ini_file_from_template(some_template_file_name, some_ini_file_name) + + assert MockFileStream.FILE_SYSTEM[some_ini_file_name] == [ + '[Foo]', + 'database = "snow_white"', + 'some_url = "http://user@unittest:6543/"', + 'oops = "$NOT_AN_ENV_VAR"', + 'hmmm = "${NOT_AN_ENV_VAR_EITHER}"', + 'shhh = "my-secret"', + 'version = "v-12345-bundle-version"', + 'project_version = "11.22.33"', + 'indexer = true', # the value will have been canonicalized + ] + + MockFileStream.reset() + + for falsity in ["FALSE", "False", "false", "", None, "misspelling"]: + + # For this test, we check if the 'indexer' option being set correctly sets the ENCODED.INDEXER option + with mock.patch("os.path.exists") as mock_exists: + mock_exists.return_value = True + with mock.patch("io.open", side_effect=mocked_open): + with override_environ(ENCODED_INDEXER=falsity): + TestDeployer.build_ini_file_from_template(some_template_file_name, some_ini_file_name) + + assert MockFileStream.FILE_SYSTEM[some_ini_file_name] == [ + '[Foo]', + 'database = "snow_white"', + 'some_url = "http://user@unittest:6543/"', + 'oops = "$NOT_AN_ENV_VAR"', + 'hmmm = "${NOT_AN_ENV_VAR_EITHER}"', + 'shhh = "my-secret"', + 'version = "v-12345-bundle-version"', + 'project_version = "11.22.33"', + # (The 'indexer =' line will be suppressed.) + ] + + MockFileStream.reset() + # For this test, we check if the 'indexer' option being set correctly sets the ENCODED.INDEXER option with mock.patch("os.path.exists") as mock_exists: mock_exists.return_value = True @@ -285,35 +332,49 @@ def now(cls): assert MockFileStream.FILE_SYSTEM[some_ini_file_name] == [ '[Foo]', - 'DATABASE = "snow_white"', - 'SOME_URL = "http://user@unittest:6543/"', - 'OOPS = "$NOT_AN_ENV_VAR"', - 'HMMM = "${NOT_AN_ENV_VAR_EITHER}"', - 'SHHH = "my-secret"', - 'VERSION = "v-12345-bundle-version"', - 'PROJECT_VERSION = "11.22.33"', - 'ENCODED.INDEXER = "true"' + 'database = "snow_white"', + 'some_url = "http://user@unittest:6543/"', + 'oops = "$NOT_AN_ENV_VAR"', + 'hmmm = "${NOT_AN_ENV_VAR_EITHER}"', + 'shhh = "my-secret"', + 'version = "v-12345-bundle-version"', + 'project_version = "11.22.33"', + 'indexer = true', ] MockFileStream.reset() - with mock.patch("os.path.exists") as mock_exists: - mock_exists.return_value = True - with mock.patch("io.open", side_effect=mocked_open): - - # for this test, we're going to pretend we deployed with bs_name == 'fourfront-indexer', - # which should throw an exception - def mocked_os_get(val, default): - if val == 'ENCODED_BS_ENV': - return 'fourfront-indexer' - else: - return default + # For these next three tests, we're going to pretend we deployed with + # bs_name == 'fourfront-indexer' in various ways. We expect an exception to be raised. - with mock.patch("os.environ.get", side_effect=mocked_os_get): - with pytest.raises(RuntimeError): + with pytest.raises(RuntimeError): + with mock.patch("os.path.exists") as mock_exists: + mock_exists.return_value = True + with mock.patch("io.open", side_effect=mocked_open): + with override_environ(ENCODED_BS_ENV='fourfront-indexer'): TestDeployer.build_ini_file_from_template(some_template_file_name, some_ini_file_name, indexer=True) + MockFileStream.reset() + + with pytest.raises(RuntimeError): + with mock.patch("os.path.exists") as mock_exists: + mock_exists.return_value = True + with mock.patch("io.open", side_effect=mocked_open): + TestDeployer.build_ini_file_from_template(some_template_file_name, + some_ini_file_name, + bs_env='fourfront-indexer', indexer=True) + + MockFileStream.reset() + + with pytest.raises(RuntimeError): + with mock.patch("os.path.exists") as mock_exists: + mock_exists.return_value = True + with mock.patch("io.open", side_effect=mocked_open): + TestDeployer.build_ini_file_from_template(some_template_file_name, + some_ini_file_name, + bs_env='fourfront-indexer') + # Uncomment this for debugging... # assert False, "PASSED" From b01062300077a867b420fb6cd39ffcb7f42d5ef0 Mon Sep 17 00:00:00 2001 From: Kent Pitman Date: Mon, 27 Apr 2020 19:07:13 -0400 Subject: [PATCH 5/9] Adjust VirtualApp and add tests. --- dcicutils/misc_utils.py | 35 +++++++--- test/test_misc_utils.py | 149 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 173 insertions(+), 11 deletions(-) diff --git a/dcicutils/misc_utils.py b/dcicutils/misc_utils.py index 4073bf210..353fc6ca0 100644 --- a/dcicutils/misc_utils.py +++ b/dcicutils/misc_utils.py @@ -4,7 +4,12 @@ import os import logging -from webtest import TestApp +import webtest # importing the library makes it easier to mock testing + + +# Is this the right place for this? I feel like this should be done in an application, not a library. +# -kmp 27-Apr-2020 +logging.basicConfig() # Using PRINT(...) for debugging, rather than its more familiar lowercase form) for intended programmatic output, @@ -13,20 +18,30 @@ PRINT = print -class VirtualApp(TestApp): - """ Wrapper class for TestApp, which we use a handler to the Encoded Application where we can submit - requests to it simulating a number of conditions, including permissions. +class VirtualApp(): """ - logging.basicConfig() + Wrapper class for TestApp, to allow custom control over submitting Encoded requests, + simulating a number of conditions, including permissions. + IMPORTANT: We use webtest.TestApp is used as substrate technology here, but use of this class + occurs in the main application, not just in testing. Among other things, we have + renamed the app here in order to avoid confusions created by the name when it is used + in production settings. + """ def __init__(self, app, environ): - """ Builds an encoded application, allowing you to submit requests to an encoded application + """ + Builds an encoded application, allowing you to submit requests to an encoded application :param app: return value of get_app(config_uri, app_name) :param environ: options to pass to the application. Usually permissions. """ - self.virtual_app = TestApp(app, environ) + # NOTE: The TestApp class that we're wrapping takes a richer set of initialization parameters + # (including relative_to, use_unicode, cookiejar, parser_features, json_encoder, and lint), + # but we'll add them conservatively here. If there is a need for any of them, we should add + # them explicitly here one-by-one as the need is shown so we have tight control of what + # we're depending on and what we're not. -kmp 27-Apr-2020 + self.wrapped_app = webtest.TestApp(app, environ) def get(self, url, **kwargs): """ Wrapper for TestApp.get that logs the outgoing GET @@ -36,7 +51,7 @@ def get(self, url, **kwargs): :return: result of GET """ logging.info('OUTGOING HTTP GET: %s' % url) - return self.virtual_app.get(url, **kwargs) + return self.wrapped_app.get(url, **kwargs) def post_json(self, url, obj, **kwargs): """ Wrapper for TestApp.post_json that logs the outgoing POST @@ -47,7 +62,7 @@ def post_json(self, url, obj, **kwargs): :return: result of POST """ logging.info('OUTGOING HTTP POST on url: %s with object: %s' % (url, obj)) - return self.virtual_app.post_json(url, obj, **kwargs) + return self.wrapped_app.post_json(url, obj, **kwargs) def patch_json(self, url, fields, **kwargs): """ Wrapper for TestApp.patch_json that logs the outgoing PATCH @@ -58,7 +73,7 @@ def patch_json(self, url, fields, **kwargs): :return: result of PATCH """ logging.info('OUTGOING HTTP PATCH on url: %s with changes: %s' % (url, fields)) - return self.virtual_app.patch_json(url, fields, **kwargs) + return self.wrapped_app.patch_json(url, fields, **kwargs) def ignored(*args, **kwargs): diff --git a/test/test_misc_utils.py b/test/test_misc_utils.py index b18ee91c9..14803727d 100644 --- a/test/test_misc_utils.py +++ b/test/test_misc_utils.py @@ -1,8 +1,11 @@ import io +import json import os -from dcicutils.misc_utils import PRINT, ignored, get_setting_from_context +import webtest +from dcicutils.misc_utils import PRINT, ignored, get_setting_from_context, VirtualApp from unittest import mock + def test_uppercase_print(): # This is just a synonym, so the easiest thing is just to test that fact. assert PRINT == print @@ -70,3 +73,147 @@ def test_get_setting_from_context(): assert get_setting_from_context(sample_settings, ini_var='pie.color', env_var=None, default='green') == '' assert get_setting_from_context(sample_settings, ini_var='pie.color', env_var=False, default='green') == 'green' + + +class FakeTestApp: + def __init__(self, app, extra_environ=None): + self.app = app + self.extra_environ = extra_environ or {} + self.calls = [] + + def get(self, url, **kwargs): + call_info = {'op': 'get', 'url': url, 'kwargs': kwargs} + self.calls.append(call_info) + return json.dumps({"result_of": call_info}) + + def post_json(self, url, obj, **kwargs): + call_info = {'op': 'post_json', 'url': url, 'obj': obj, 'kwargs': kwargs} + self.calls.append(call_info) + return json.dumps({"result_of": call_info}) + + def patch_json(self, url, obj, **kwargs): + call_info = {'op': 'patch_json', 'url': url, 'obj': obj, 'kwargs': kwargs} + self.calls.append(call_info) + return json.dumps({"result_of": call_info}) + +class FakeApp: + pass + + +def test_virtual_app_creation(): + with mock.patch.object(webtest, "TestApp", FakeTestApp): + app = FakeApp() + environ = {'some': 'stuff'} + + vapp = VirtualApp(app, environ) + + assert isinstance(vapp, VirtualApp) + assert not isinstance(vapp, webtest.TestApp) + + assert isinstance(vapp.wrapped_app, webtest.TestApp) # the mocked one, anyway. + assert vapp.wrapped_app.app is app + assert vapp.wrapped_app.extra_environ is environ + + return vapp + + +def test_virtual_app_get(): + + with mock.patch.object(webtest, "TestApp", FakeTestApp): + app = FakeApp() + environ = {'some': 'stuff'} + vapp = VirtualApp(app, environ) + + log_info = [] + + with mock.patch("logging.info") as mock_info: + mock_info.side_effect = lambda msg: log_info.append(msg) + vapp.get("http://no.such.place/") + vapp.get("http://no.such.place/", params={'foo': 'bar'}) + + assert log_info == [ + 'OUTGOING HTTP GET: http://no.such.place/', + 'OUTGOING HTTP GET: http://no.such.place/', + ] + assert vapp.wrapped_app.calls == [ + { + 'op': 'get', + 'url': 'http://no.such.place/', + 'kwargs': {}, + }, + { + 'op': 'get', + 'url': 'http://no.such.place/', + 'kwargs': {'params': {'foo': 'bar'}}, + }, + ] + +def test_virtual_app_post_json(): + + with mock.patch.object(webtest, "TestApp", FakeTestApp): + app = FakeApp() + environ = {'some': 'stuff'} + vapp = VirtualApp(app, environ) + + log_info = [] + + with mock.patch("logging.info") as mock_info: + mock_info.side_effect = lambda msg: log_info.append(msg) + vapp.post_json("http://no.such.place/", {'beta': 'gamma'}) + vapp.post_json("http://no.such.place/", {'alpha': 'omega'}, params={'foo': 'bar'}) + + assert log_info == [ + ("OUTGOING HTTP POST on url: %s with object: %s" + % ("http://no.such.place/", {'beta': 'gamma'})), + ("OUTGOING HTTP POST on url: %s with object: %s" + % ("http://no.such.place/", {'alpha': 'omega'})), + ] + assert vapp.wrapped_app.calls == [ + { + 'op': 'post_json', + 'url': 'http://no.such.place/', + 'obj': {'beta': 'gamma'}, + 'kwargs': {}, + }, + { + 'op': 'post_json', + 'url': 'http://no.such.place/', + 'obj': {'alpha': 'omega'}, + 'kwargs': {'params': {'foo': 'bar'}}, + }, + ] + +def test_virtual_app_patch_json(): + + with mock.patch.object(webtest, "TestApp", FakeTestApp): + app = FakeApp() + environ = {'some': 'stuff'} + vapp = VirtualApp(app, environ) + + log_info = [] + + with mock.patch("logging.info") as mock_info: + mock_info.side_effect = lambda msg: log_info.append(msg) + vapp.patch_json("http://no.such.place/", {'beta': 'gamma'}) + vapp.patch_json("http://no.such.place/", {'alpha': 'omega'}, params={'foo': 'bar'}) + + assert log_info == [ + ("OUTGOING HTTP PATCH on url: %s with changes: %s" + % ("http://no.such.place/", {'beta': 'gamma'})), + ("OUTGOING HTTP PATCH on url: %s with changes: %s" + % ("http://no.such.place/", {'alpha': 'omega'})), + ] + assert vapp.wrapped_app.calls == [ + { + 'op': 'patch_json', + 'url': 'http://no.such.place/', + 'obj': {'beta': 'gamma'}, + 'kwargs': {}, + }, + { + 'op': 'patch_json', + 'url': 'http://no.such.place/', + 'obj': {'alpha': 'omega'}, + 'kwargs': {'params': {'foo': 'bar'}}, + }, + ] From cf90fe66acad98ed180667075fe658688a2db237 Mon Sep 17 00:00:00 2001 From: Kent Pitman Date: Mon, 27 Apr 2020 19:14:50 -0400 Subject: [PATCH 6/9] Satisfy flake8 --- test/test_misc_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test_misc_utils.py b/test/test_misc_utils.py index 14803727d..441e81e97 100644 --- a/test/test_misc_utils.py +++ b/test/test_misc_utils.py @@ -96,6 +96,7 @@ def patch_json(self, url, obj, **kwargs): self.calls.append(call_info) return json.dumps({"result_of": call_info}) + class FakeApp: pass @@ -148,6 +149,7 @@ def test_virtual_app_get(): }, ] + def test_virtual_app_post_json(): with mock.patch.object(webtest, "TestApp", FakeTestApp): @@ -183,6 +185,7 @@ def test_virtual_app_post_json(): }, ] + def test_virtual_app_patch_json(): with mock.patch.object(webtest, "TestApp", FakeTestApp): From 977614cd1c10eb13c26c992e58c218a23fb319ac Mon Sep 17 00:00:00 2001 From: Kent Pitman Date: Mon, 27 Apr 2020 19:58:13 -0400 Subject: [PATCH 7/9] Better tests that also test return values in the virtual app. --- test/test_misc_utils.py | 92 +++++++++++++++++++++++++++++++++++------ 1 file changed, 80 insertions(+), 12 deletions(-) diff --git a/test/test_misc_utils.py b/test/test_misc_utils.py index 441e81e97..5b7270aa9 100644 --- a/test/test_misc_utils.py +++ b/test/test_misc_utils.py @@ -75,6 +75,19 @@ def test_get_setting_from_context(): assert get_setting_from_context(sample_settings, ini_var='pie.color', env_var=False, default='green') == 'green' +class FakeResponse: + + def __init__(self, json): + self._json = json + + def json(self): + return self._json + + @property + def content(self): + return json.dumps(self.json, indent=2, default=str) + + class FakeTestApp: def __init__(self, app, extra_environ=None): self.app = app @@ -84,17 +97,17 @@ def __init__(self, app, extra_environ=None): def get(self, url, **kwargs): call_info = {'op': 'get', 'url': url, 'kwargs': kwargs} self.calls.append(call_info) - return json.dumps({"result_of": call_info}) + return FakeResponse({'processed': call_info}) def post_json(self, url, obj, **kwargs): call_info = {'op': 'post_json', 'url': url, 'obj': obj, 'kwargs': kwargs} self.calls.append(call_info) - return json.dumps({"result_of": call_info}) + return FakeResponse({'processed': call_info}) def patch_json(self, url, obj, **kwargs): call_info = {'op': 'patch_json', 'url': url, 'obj': obj, 'kwargs': kwargs} self.calls.append(call_info) - return json.dumps({"result_of": call_info}) + return FakeResponse({'processed': call_info}) class FakeApp: @@ -129,8 +142,24 @@ def test_virtual_app_get(): with mock.patch("logging.info") as mock_info: mock_info.side_effect = lambda msg: log_info.append(msg) - vapp.get("http://no.such.place/") - vapp.get("http://no.such.place/", params={'foo': 'bar'}) + + response1 = vapp.get("http://no.such.place/") + assert response1.json() == { + 'processed': { + 'op': 'get', + 'url': 'http://no.such.place/', + 'kwargs': {}, + } + } + + response2 = vapp.get("http://no.such.place/", params={'foo': 'bar'}) + assert response2.json() == { + 'processed': { + 'op': 'get', + 'url': 'http://no.such.place/', + 'kwargs': {'params': {'foo': 'bar'}}, + } + } assert log_info == [ 'OUTGOING HTTP GET: http://no.such.place/', @@ -146,7 +175,8 @@ def test_virtual_app_get(): 'op': 'get', 'url': 'http://no.such.place/', 'kwargs': {'params': {'foo': 'bar'}}, - }, + } + , ] @@ -161,8 +191,26 @@ def test_virtual_app_post_json(): with mock.patch("logging.info") as mock_info: mock_info.side_effect = lambda msg: log_info.append(msg) - vapp.post_json("http://no.such.place/", {'beta': 'gamma'}) - vapp.post_json("http://no.such.place/", {'alpha': 'omega'}, params={'foo': 'bar'}) + + response1 = vapp.post_json("http://no.such.place/", {'beta': 'gamma'}) + assert response1.json() == { + 'processed': { + 'op': 'post_json', + 'url': 'http://no.such.place/', + 'obj': {'beta': 'gamma'}, + 'kwargs': {}, + } + } + + response2 = vapp.post_json("http://no.such.place/", {'alpha': 'omega'}, params={'foo': 'bar'}) + assert response2.json() == { + 'processed': { + 'op': 'post_json', + 'url': 'http://no.such.place/', + 'obj': {'alpha': 'omega'}, + 'kwargs': {'params': {'foo': 'bar'}}, + } + } assert log_info == [ ("OUTGOING HTTP POST on url: %s with object: %s" @@ -182,7 +230,8 @@ def test_virtual_app_post_json(): 'url': 'http://no.such.place/', 'obj': {'alpha': 'omega'}, 'kwargs': {'params': {'foo': 'bar'}}, - }, + } + , ] @@ -197,8 +246,26 @@ def test_virtual_app_patch_json(): with mock.patch("logging.info") as mock_info: mock_info.side_effect = lambda msg: log_info.append(msg) - vapp.patch_json("http://no.such.place/", {'beta': 'gamma'}) - vapp.patch_json("http://no.such.place/", {'alpha': 'omega'}, params={'foo': 'bar'}) + + response1 = vapp.patch_json("http://no.such.place/", {'beta': 'gamma'}) + assert response1.json() == { + 'processed': { + 'op': 'patch_json', + 'url': 'http://no.such.place/', + 'obj': {'beta': 'gamma'}, + 'kwargs': {}, + } + } + + response2 = vapp.patch_json("http://no.such.place/", {'alpha': 'omega'}, params={'foo': 'bar'}) + assert response2.json() == { + 'processed': { + 'op': 'patch_json', + 'url': 'http://no.such.place/', + 'obj': {'alpha': 'omega'}, + 'kwargs': {'params': {'foo': 'bar'}}, + } + } assert log_info == [ ("OUTGOING HTTP PATCH on url: %s with changes: %s" @@ -218,5 +285,6 @@ def test_virtual_app_patch_json(): 'url': 'http://no.such.place/', 'obj': {'alpha': 'omega'}, 'kwargs': {'params': {'foo': 'bar'}}, - }, + } + , ] From 6f9ec85a61950897134f859a8b2d7b95bcc275e8 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 28 Apr 2020 09:58:54 -0400 Subject: [PATCH 8/9] C4-142 make expand_es_metadata recoverable from expired signature --- dcicutils/ff_utils.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/dcicutils/ff_utils.py b/dcicutils/ff_utils.py index 7cceefde5..4973d98ee 100644 --- a/dcicutils/ff_utils.py +++ b/dcicutils/ff_utils.py @@ -12,6 +12,7 @@ ) from .misc_utils import PRINT import requests +from elasticsearch.exceptions import AuthorizationException # urlparse import differs between py2 and 3 if sys.version_info[0] < 3: import urlparse @@ -695,7 +696,7 @@ def delete_field(obj_id, del_field, key=None, ff_env=None): def get_es_search_generator(es_client, index, body, page_size=200): """ - Simple generator behind get_es_metada which takes an es_client (from + Simple generator behind get_es_metadata which takes an es_client (from es_utils create_es_client), a string index, and a dict query body. Also takes an optional string page_size, which controls pagination size NOTE: 'index' must be namespaced @@ -881,7 +882,8 @@ def expand_es_metadata(uuid_list, key=None, ff_env=None, store_frame='raw', add_ add_pc_wfr (bool): Include workflow_runs and linked items (processed/ref files, wf, software...) ignore_field(list): Remove keys from items, so any linking through these fields, ie relations use_generator (bool): Use a generator when getting es. Less memory used but takes longer - es_client: optional result from es_utils.create_es_client + es_client: optional result from es_utils.create_es_client - note this could be regenerated + in this method if the signature expires Returns: dict: contains all item types as keys, and with values of list of dictionaries i.e. @@ -928,8 +930,21 @@ def remove_keys(my_dict, remove_list): while uuid_list: uuids_to_check = [] # uuids to add to uuid_list if not if not in item_uuids - for es_item in get_es_metadata(uuid_list, es_client=es_client, chunk_size=chunk, - is_generator=use_generator, key=auth): + + # get the next page of data, recreating the es_client if need be + try: + current_page = get_es_metadata(uuid_list, es_client=es_client, chunk_size=chunk, + is_generator=use_generator, key=auth) + except AuthorizationException: # our signature expired, recreate the es_client with a fresh signature + if es_url: + es_client = es_utils.create_es_client(es_url, use_aws_auth=True) + else: # recreate client and try again - if we fail here, exception should propagate + es_url = get_health_page(key=auth)['elasticsearch'] + es_client = es_utils.create_es_client(es_url, use_aws_auth=True) + + current_page = get_es_metadata(uuid_list, es_client=es_client, chunk_size=chunk, + is_generator=use_generator, key=auth) + for es_item in current_page: # get object type via es result and schema for storing obj_type = es_item['object']['@type'][0] obj_key = schema_name[obj_type] From 06a9caac22559d25037a88feefe8d27a4ce96805 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 28 Apr 2020 10:38:10 -0400 Subject: [PATCH 9/9] C4-141 add exception handling, fix tests to account for it --- dcicutils/misc_utils.py | 45 ++++++++++++++++++++++++++++++++++++----- test/test_misc_utils.py | 13 ++++++------ 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/dcicutils/misc_utils.py b/dcicutils/misc_utils.py index 353fc6ca0..aff3946e9 100644 --- a/dcicutils/misc_utils.py +++ b/dcicutils/misc_utils.py @@ -18,7 +18,32 @@ PRINT = print -class VirtualApp(): +class VirtualAppError(Exception): + """ Special Exception to be raised by VirtualApp that contains some additional info """ + + def __init__(self, msg, url, body, e): + super(VirtualAppError, self).__init__(msg) + self.msg = msg + self.query_url = url + self.query_body = body + self.raw_exception = e + + def __repr__(self): + return "Exception encountered on VirtualApp\n" \ + "URL: %s\n" \ + "BODY: %s\n" \ + "MSG: %s\n" \ + "Raw Exception: %s\n" % (self.query_url, self.query_body, self.msg, self.raw_exception) + + def __str__(self): + return self.__repr__() + + +class _VirtualAppHelper(webtest.TestApp): # effectively disguises 'TestApp' + pass + + +class VirtualApp: """ Wrapper class for TestApp, to allow custom control over submitting Encoded requests, simulating a number of conditions, including permissions. @@ -28,6 +53,7 @@ class VirtualApp(): renamed the app here in order to avoid confusions created by the name when it is used in production settings. """ + HELPER_CLASS = _VirtualAppHelper def __init__(self, app, environ): """ @@ -41,7 +67,7 @@ def __init__(self, app, environ): # but we'll add them conservatively here. If there is a need for any of them, we should add # them explicitly here one-by-one as the need is shown so we have tight control of what # we're depending on and what we're not. -kmp 27-Apr-2020 - self.wrapped_app = webtest.TestApp(app, environ) + self.wrapped_app = self.HELPER_CLASS(app, environ) def get(self, url, **kwargs): """ Wrapper for TestApp.get that logs the outgoing GET @@ -51,7 +77,10 @@ def get(self, url, **kwargs): :return: result of GET """ logging.info('OUTGOING HTTP GET: %s' % url) - return self.wrapped_app.get(url, **kwargs) + try: + return self.wrapped_app.get(url, **kwargs) + except webtest.AppError as e: + raise VirtualAppError(msg='HTTP GET failed.', url=url, body='', e=str(e)) def post_json(self, url, obj, **kwargs): """ Wrapper for TestApp.post_json that logs the outgoing POST @@ -62,7 +91,10 @@ def post_json(self, url, obj, **kwargs): :return: result of POST """ logging.info('OUTGOING HTTP POST on url: %s with object: %s' % (url, obj)) - return self.wrapped_app.post_json(url, obj, **kwargs) + try: + return self.wrapped_app.post_json(url, obj, **kwargs) + except webtest.AppError as e: + raise VirtualAppError(msg='HTTP POST failed.', url=url, body=obj, e=str(e)) def patch_json(self, url, fields, **kwargs): """ Wrapper for TestApp.patch_json that logs the outgoing PATCH @@ -73,7 +105,10 @@ def patch_json(self, url, fields, **kwargs): :return: result of PATCH """ logging.info('OUTGOING HTTP PATCH on url: %s with changes: %s' % (url, fields)) - return self.wrapped_app.patch_json(url, fields, **kwargs) + try: + return self.wrapped_app.patch_json(url, fields, **kwargs) + except webtest.AppError as e: + raise VirtualAppError(msg='HTTP PATCH failed.', url=url, body=fields, e=str(e)) def ignored(*args, **kwargs): diff --git a/test/test_misc_utils.py b/test/test_misc_utils.py index 5b7270aa9..d410476b1 100644 --- a/test/test_misc_utils.py +++ b/test/test_misc_utils.py @@ -2,7 +2,7 @@ import json import os import webtest -from dcicutils.misc_utils import PRINT, ignored, get_setting_from_context, VirtualApp +from dcicutils.misc_utils import PRINT, ignored, get_setting_from_context, VirtualApp, _VirtualAppHelper from unittest import mock @@ -115,7 +115,7 @@ class FakeApp: def test_virtual_app_creation(): - with mock.patch.object(webtest, "TestApp", FakeTestApp): + with mock.patch.object(VirtualApp, "HELPER_CLASS", FakeTestApp): app = FakeApp() environ = {'some': 'stuff'} @@ -123,8 +123,9 @@ def test_virtual_app_creation(): assert isinstance(vapp, VirtualApp) assert not isinstance(vapp, webtest.TestApp) + assert not isinstance(vapp, _VirtualAppHelper) - assert isinstance(vapp.wrapped_app, webtest.TestApp) # the mocked one, anyway. + assert isinstance(vapp.wrapped_app, FakeTestApp) # the mocked one, anyway. assert vapp.wrapped_app.app is app assert vapp.wrapped_app.extra_environ is environ @@ -133,7 +134,7 @@ def test_virtual_app_creation(): def test_virtual_app_get(): - with mock.patch.object(webtest, "TestApp", FakeTestApp): + with mock.patch.object(VirtualApp, "HELPER_CLASS", FakeTestApp): app = FakeApp() environ = {'some': 'stuff'} vapp = VirtualApp(app, environ) @@ -182,7 +183,7 @@ def test_virtual_app_get(): def test_virtual_app_post_json(): - with mock.patch.object(webtest, "TestApp", FakeTestApp): + with mock.patch.object(VirtualApp, "HELPER_CLASS", FakeTestApp): app = FakeApp() environ = {'some': 'stuff'} vapp = VirtualApp(app, environ) @@ -237,7 +238,7 @@ def test_virtual_app_post_json(): def test_virtual_app_patch_json(): - with mock.patch.object(webtest, "TestApp", FakeTestApp): + with mock.patch.object(VirtualApp, "HELPER_CLASS", FakeTestApp): app = FakeApp() environ = {'some': 'stuff'} vapp = VirtualApp(app, environ)