From 2ec997c9eb0972ddef810a25de709e715b5d9aa9 Mon Sep 17 00:00:00 2001 From: Hans Donner Date: Tue, 7 Mar 2023 15:33:56 +0100 Subject: [PATCH] mypy config to exclude, and test selection via markers - mypy was configred via `files` which fails when specifying a target dir. Now turned into an exclude in the config, and adding explicit files to the Makefile when a service is selected. - added SKIP_MYPY to be able to skip mypy check (as long as the mypy chnage is not complete) - using conftest.py to give all the tests default markers - the include/exclude from the makefile is now in the conftest.py marking as well - for services, also picking up test that reference that service in its name (specific pattern) --- Makefile | 36 +++++------ setup.cfg | 132 +++++++++++++++++++++++++++++++++++++++- tests/conftest.py | 149 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 299 insertions(+), 18 deletions(-) create mode 100644 tests/conftest.py diff --git a/Makefile b/Makefile index 6dfae9180b20..8f917f1e33e9 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,23 @@ SHELL := /bin/bash ifdef SERVICE_NAME -TEST_SERVICE = test_${SERVICE_NAME} +TEST_SERVICE = and ${SERVICE_NAME} +TEST_SERVICE_DIR = test_${SERVICE_NAME} TF_SERVICE_NAME = ${SERVICE_NAME} +MYPY_EXPLICIT_FILES ?= moto/${SERVICE_NAME}/*.py* endif -TF_SERVICE_NAME ?= "default" +TF_SERVICE_NAME ?= default TEST_NAMES ?= "*" +# Parallel tests will be run separate +# -m tags are specified in tests/conftest.py, or on individual files/tests ifeq ($(TEST_SERVER_MODE), true) - # exclude test_kinesisvideoarchivedmedia - # because testing with moto_server is difficult with data-endpoint - TEST_EXCLUDE := --ignore tests/test_kinesisvideoarchivedmedia --ignore tests/test_acm --ignore tests/test_amp --ignore tests/test_awslambda --ignore tests/test_batch --ignore tests/test_ec2 --ignore tests/test_sqs - # Parallel tests will be run separate - PARALLEL_TESTS := ./tests/test_acm/ ./tests/test_acmpca/ ./tests/test_amp/ ./tests/test_awslambda ./tests/test_batch ./tests/test_ec2 ./tests/test_sqs + TEST_EXCLUDE := -m "not parallel and not parallel_server_only and not skip_server ${TEST_SERVICE}" + PARALLEL_TESTS := -m "parallel or parallel_server_only ${TEST_SERVICE}" else - TEST_EXCLUDE := --ignore tests/test_batch --ignore tests/test_ec2 --ignore tests/test_sqs - PARALLEL_TESTS := ./tests/test_batch ./tests/test_ec2 ./tests/test_sqs + TEST_EXCLUDE := -m "not parallel ${TEST_SERVICE}" + PARALLEL_TESTS := -m "parallel ${TEST_SERVICE}" endif init: @@ -25,15 +26,15 @@ init: lint: @echo "Running flake8..." - flake8 moto/${SERVICE_NAME} tests/${TEST_SERVICE} + flake8 moto/${SERVICE_NAME} tests/${TEST_SERVICE_DIR} @echo "Running black... " $(eval black_version := $(shell grep "^black==" requirements-dev.txt | sed "s/black==//")) @echo "(Make sure you have black-$(black_version) installed, as other versions will produce different results)" - black --check moto/${SERVICE_NAME} tests/${TEST_SERVICE} + black --check moto/${SERVICE_NAME} tests/${TEST_SERVICE_DIR} @echo "Running pylint..." - pylint -j 0 moto/${SERVICE_NAME} tests/${TEST_SERVICE} + pylint -j 0 moto/${SERVICE_NAME} tests/${TEST_SERVICE_DIR} @echo "Running MyPy..." - mypy --install-types --non-interactive moto/${SERVICE_NAME} + if [[ "${SKIP_MYPY}" == "" ]]; then mypy --install-types --non-interactive ${MYPY_EXPLICIT_FILES}; else echo "Skipping"; fi format: black moto/ tests/ @@ -41,13 +42,14 @@ format: test-only: rm -f .coverage rm -rf cover - pytest -sv --cov=moto --cov-report xml ./tests/${TEST_SERVICE} $(TEST_EXCLUDE) + pytest -sv --cov=moto --cov-report xml ./tests $(TEST_EXCLUDE) ${PYTEST_ARGS} # https://github.com/aws/aws-xray-sdk-python/issues/196 - Run these tests separately without Coverage enabled - if [[ "${SERVICE_NAME}" == "" || "${SERVICE_NAME}" == "xray" ]]; then pytest -sv ./tests/test_xray; else echo "Skipping"; fi - if [[ "$(filter %${SERVICE_NAME},$(PARALLEL_TESTS))" != "" ]]; then MOTO_CALL_RESET_API=false pytest --cov=moto --cov-report xml --cov-append -n 4 $(filter %${SERVICE_NAME},$(PARALLEL_TESTS)); else echo "Skipping"; fi + if [[ "${SERVICE_NAME}" == "" || "${SERVICE_NAME}" == "xray" ]]; then pytest -sv ./tests/test_xray ${PYTEST_ARGS}; else echo "Skipping"; fi + MOTO_CALL_RESET_API=false pytest --cov=moto --cov-report xml --cov-append -n 4 $(PARALLEL_TESTS) ${PYTEST_ARGS} test: lint test-only - @echo "USAGE: make test [SERVICE_NAME=s3] (defaults to all)" + @echo "USAGE: make test [SERVICE_NAME=s3] [SKIP_MYPY=true]" + @echo " defaults to all services and running mypy (not all services are mypy proof, see also setup.cfg)" terraformtests: @echo "Make sure that the MotoServer is already running on port 4566 (moto_server -p 4566)" diff --git a/setup.cfg b/setup.cfg index f617f0fe1d1a..c3fea90bc566 100644 --- a/setup.cfg +++ b/setup.cfg @@ -211,7 +211,130 @@ universal=1 [tool:pytest] markers = + # special cases network: marks tests which require network connection + parallel: can be run in parallel + parallel_server_only: can be run in parallel (in server mode only) + skip_server: should not be run in server mode + # services + acm: acm service + acmpca: acmpca service + amp: amp service + apigateway: apigateway service + apigatewayv2: apigatewayv2 service + applicationautoscaling: applicationautoscaling service + appsync: appsync service + athena: athena service + autoscaling: autoscaling service + awslambda: awslambda service + batch: batch service + batch_simple: batch_simple service + budgets: budgets service + ce: ce service + cloudformation: cloudformation service + cloudfront: cloudfront service + cloudtrail: cloudtrail service + cloudwatch: cloudwatch service + codebuild: codebuild service + codecommit: codecommit service + codepipeline: codepipeline service + cognitoidentity: cognitoidentity service + cognitoidp: cognitoidp service + comprehend: comprehend service + config: config service + databrew: databrew service + datapipeline: datapipeline service + datasync: datasync service + dax: dax service + dms: dms service + ds: ds service + dynamodb: dynamodb service + dynamodb_v20111205: dynamodb_v20111205 service + dynamodbstreams: dynamodbstreams service + ebs: ebs service + ec2: ec2 service + ec2instanceconnect: ec2instanceconnect service + ecr: ecr service + ecs: ecs service + efs: efs service + eks: eks service + elasticache: elasticache service + elasticbeanstalk: elasticbeanstalk service + elastictranscoder: elastictranscoder service + elb: elb service + elbv2: elbv2 service + emr: emr service + emrcontainers: emrcontainers service + emrserverless: emrserverless service + es: es service + events: events service + firehose: firehose service + forecast: forecast service + glacier: glacier service + glue: glue service + greengrass: greengrass service + guardduty: guardduty service + iam: iam service + identitystore: identitystore service + instance_metadata: instance_metadata service + iot: iot service + iotdata: iotdata service + kinesis: kinesis service + kinesisvideo: kinesisvideo service + kinesisvideoarchivedmedia: kinesisvideoarchivedmedia service + kms: kms service + logs: logs service + managedblockchain: managedblockchain service + mediaconnect: mediaconnect service + medialive: medialive service + mediapackage: mediapackage service + mediastore: mediastore service + mediastoredata: mediastoredata service + meteringmarketplace: meteringmarketplace service + moto_api: moto_api service + moto_server: moto_server service + mq: mq service + neptune: neptune service + opsworks: opsworks service + organizations: organizations service + packages: packages service + personalize: personalize service + pinpoint: pinpoint service + polly: polly service + quicksight: quicksight service + ram: ram service + rds: rds service + redshift: redshift service + redshiftdata: redshiftdata service + rekognition: rekognition service + resourcegroups: resourcegroups service + resourcegroupstaggingapi: resourcegroupstaggingapi service + route53: route53 service + route53resolver: route53resolver service + s3: s3 service + s3bucket_path: s3bucket_path service + s3control: s3control service + sagemaker: sagemaker service + sdb: sdb service + secretsmanager: secretsmanager service + servicediscovery: servicediscovery service + servicequotas: servicequotas service + ses: ses service + signer: signer service + sns: sns service + sqs: sqs service + ssm: ssm service + ssoadmin: ssoadmin service + stepfunctions: stepfunctions service + sts: sts service + support: support service + swf: swf service + textract: textract service + timestreamwrite: timestreamwrite service + transcribe: transcribe service + utilities: utilities service + wafv2: wafv2 service + xray: xray service [coverage:run] relative_files = True @@ -229,7 +352,14 @@ disable = W,C,R,E enable = anomalous-backslash-in-string, arguments-renamed, dangerous-default-value, deprecated-module, function-redefined, import-self, redefined-builtin, redefined-outer-name, reimported, pointless-statement, super-with-arguments, unused-argument, unused-import, unused-variable, useless-else-on-loop, wildcard-import [mypy] -files= moto/a*,moto/b*,moto/c*,moto/d*,moto/e*,moto/f*,moto/moto_api,moto/neptune +files= moto/a*,moto/b*,moto/c*,moto/d*,moto/e*,moto/moto_api,moto/neptune +exclude = (?x)( + ^moto/[g-l] + | ^moto/m(?!oto_api) # all m*, except moto_api + | ^moto/n(?!eptune) # all n*, except neptune + | ^moto/[o-z] + ) + show_column_numbers=True show_error_codes = True disable_error_code=abstract diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000000..7686148126d4 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,149 @@ +import pytest + +SERVICES = ( + "acm", + "acmpca", + "amp", + "apigateway", + "apigatewayv2", + "applicationautoscaling", + "appsync", + "athena", + "autoscaling", + "awslambda", + "batch", + "batch_simple", + "budgets", + "ce", + "cloudformation", + "cloudfront", + "cloudtrail", + "cloudwatch", + "codebuild", + "codecommit", + "codepipeline", + "cognitoidentity", + "cognitoidp", + "comprehend", + "config", + "databrew", + "datapipeline", + "datasync", + "dax", + "dms", + "ds", + "dynamodb", + "dynamodb_v20111205", + "dynamodbstreams", + "ebs", + "ec2", + "ec2instanceconnect", + "ecr", + "ecs", + "efs", + "eks", + "elasticache", + "elasticbeanstalk", + "elastictranscoder", + "elb", + "elbv2", + "emr", + "emrcontainers", + "emrserverless", + "es", + "events", + "firehose", + "forecast", + "glacier", + "glue", + "greengrass", + "guardduty", + "iam", + "identitystore", + "instance_metadata", + "iot", + "iotdata", + "kinesis", + "kinesisvideo", + "kinesisvideoarchivedmedia", + "kms", + "logs", + "managedblockchain", + "mediaconnect", + "medialive", + "mediapackage", + "mediastore", + "mediastoredata", + "meteringmarketplace", + "moto_api", + "moto_server", + "mq", + "neptune", + "opsworks", + "organizations", + "packages", + "personalize", + "pinpoint", + "polly", + "quicksight", + "ram", + "rds", + "redshift", + "redshiftdata", + "rekognition", + "resourcegroups", + "resourcegroupstaggingapi", + "route53", + "route53resolver", + "s3", + "s3bucket_path", + "s3control", + "sagemaker", + "sdb", + "secretsmanager", + "servicediscovery", + "servicequotas", + "ses", + "signer", + "sns", + "sqs", + "ssm", + "ssoadmin", + "stepfunctions", + "sts", + "support", + "swf", + "textract", + "timestreamwrite", + "transcribe", + "utilities", + "wafv2", + "xray", +) + + +EXTRA_MARKERS_TEST_DIRS = { + "acm": (pytest.mark.parallel_server_only,), + "acmpca": (pytest.mark.parallel_server_only,), + "amp": (pytest.mark.parallel_server_only,), + "awslambda": (pytest.mark.parallel_server_only,), + "batch": (pytest.mark.parallel,), + "ec2": (pytest.mark.parallel,), + # exclude test_kinesisvideoarchivedmedia + # because testing with moto_server is difficult with data-endpoint + "kinesisvideoarchivedmedia": (pytest.mark.skip_server,), + "sqs": (pytest.mark.parallel,), +} + + +def pytest_collection_modifyitems(items): # noqa: SC200 + for item in items: + for service, markers in EXTRA_MARKERS_TEST_DIRS.items(): + if f"tests/test_{service}/" in item.nodeid: # noqa: SC200 + for marker in markers: + item.add_marker(marker) + for service in SERVICES: + if f"tests/test_{service}/" in item.nodeid: # noqa: SC200 + item.add_marker(getattr(pytest.mark, service)) + if f"_{service}.py" in item.nodeid: + item.add_marker(getattr(pytest.mark, service))