Waiting to fix missing warning outputs in tox

Author: Kudbettin

tests/functional/run_functional_tests.py

@@ -33,6 +33,7 @@ class Config(object):
 
     test_result = ''
     expected = ''
+    unexpected = ''
 
     if not os.path.isfile('{}/plan.out.json'.format(directory)) or not os.path.isfile('{}/test.feature'.format(directory)):
         test_result = colorful.orange('skipped')
@@ -44,6 +45,10 @@ class Config(object):
             with open('{}/.expected'.format(directory)) as expected_file:
                 expected = expected_file.read().split('\n')
 
+        if os.path.isfile('{}/.unexpected'.format(directory)):
+            with open('{}/.unexpected'.format(directory)) as unexpected_file:
+                unexpected = unexpected_file.read().split('\n')
+
         if not os.path.isfile('{}/.no_early_exit'.format(directory)):
             parameters.append('-q')
 
@@ -81,11 +86,26 @@ class Config(object):
 
                         test_result = colorful.red('failed')
                         failure_happened = True
-                    else:
-                        test_result = colorful.green('passed')
 
-                else:
+                if unexpected:
+                    unexpected_failures = [
+                        unexp for unexp in unexpected 
+                        if re.findall(unexp, str(test_process.stdout))
+                    ]
+
+                    if unexpected_failures:
+                        print('\nOutput: {}'.format(test_process.stdout))
+                        print('Found;')
+                        for failure in expected_failures:
+                            print('\t{}'.format(colorful.yellow(failure)))
+                        print('in the test output. This was unexpected.\n')
+
+                        test_result = colorful.red('failed')
+                        failure_happened = True
+                
+                if not failure_happened:
                     test_result = colorful.green('passed')
+
             else:
                 print('Output: {}'.format(test_process.stdout))
                 test_result = colorful.red('failed')

tests/functional/test_issue-279/.expected

@@ -0,0 +1 @@
+warning: aws_autoscaling_group.bar \(aws_autoscaling_group\) does not have health_check_type property.

tests/functional/test_issue-279/.unexpected

@@ -0,0 +1,23 @@
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to us-east-1a.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to 1.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to None.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to False.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to 300.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to None.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to $Latest.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to True.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to True.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to None.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to 1.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to 1Minute.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to None.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to 1.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to None.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to None.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to False.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to None.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to None.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to None.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to None.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to 10m.
+warning: resource property in aws_autoscaling_group.bar resource does not match with \(^ELB$\)|\(^EC2$\) case insensitive regex. It is set to None.

tests/functional/test_issue-279/plan.out.json

@@ -0,0 +1 @@
+{"format_version":"0.1","terraform_version":"0.12.25","planned_values":{"root_module":{"resources":[{"address":"aws_autoscaling_group.bar","mode":"managed","type":"aws_autoscaling_group","name":"bar","provider_name":"aws","schema_version":0,"values":{"availability_zones":["us-east-1a"],"desired_capacity":1,"enabled_metrics":null,"force_delete":false,"health_check_grace_period":300,"initial_lifecycle_hook":[],"launch_configuration":null,"launch_template":[{"version":"$Latest"}],"max_instance_lifetime":null,"max_size":1,"metrics_granularity":"1Minute","min_elb_capacity":null,"min_size":1,"mixed_instances_policy":[],"name_prefix":null,"placement_group":null,"protect_from_scale_in":false,"suspended_processes":null,"tag":[],"tags":null,"termination_policies":null,"timeouts":null,"wait_for_capacity_timeout":"10m","wait_for_elb_capacity":null}},{"address":"aws_launch_template.foobar","mode":"managed","type":"aws_launch_template","name":"foobar","provider_name":"aws","schema_version":0,"values":{"block_device_mappings":[],"capacity_reservation_specification":[],"cpu_options":[],"credit_specification":[],"description":null,"disable_api_termination":null,"ebs_optimized":null,"elastic_gpu_specifications":[],"elastic_inference_accelerator":[],"hibernation_options":[],"iam_instance_profile":[],"image_id":"ami-1a2b3c","instance_initiated_shutdown_behavior":null,"instance_market_options":[],"instance_type":"t2.micro","kernel_id":null,"key_name":null,"license_specification":[],"monitoring":[],"name_prefix":"foobar","network_interfaces":[],"placement":[],"ram_disk_id":null,"security_group_names":null,"tag_specifications":[],"tags":null,"user_data":null,"vpc_security_group_ids":null}}]}},"resource_changes":[{"address":"aws_autoscaling_group.bar","mode":"managed","type":"aws_autoscaling_group","name":"bar","provider_name":"aws","change":{"actions":["create"],"before":null,"after":{"availability_zones":["us-east-1a"],"desired_capacity":1,"enabled_metrics":null,"force_delete":false,"health_check_grace_period":300,"initial_lifecycle_hook":[],"launch_configuration":null,"launch_template":[{"version":"$Latest"}],"max_instance_lifetime":null,"max_size":1,"metrics_granularity":"1Minute","min_elb_capacity":null,"min_size":1,"mixed_instances_policy":[],"name_prefix":null,"placement_group":null,"protect_from_scale_in":false,"suspended_processes":null,"tag":[],"tags":null,"termination_policies":null,"timeouts":null,"wait_for_capacity_timeout":"10m","wait_for_elb_capacity":null},"after_unknown":{"arn":true,"availability_zones":[false],"default_cooldown":true,"health_check_type":true,"id":true,"initial_lifecycle_hook":[],"launch_template":[{"id":true,"name":true}],"load_balancers":true,"mixed_instances_policy":[],"name":true,"service_linked_role_arn":true,"tag":[],"target_group_arns":true,"vpc_zone_identifier":true}}},{"address":"aws_launch_template.foobar","mode":"managed","type":"aws_launch_template","name":"foobar","provider_name":"aws","change":{"actions":["create"],"before":null,"after":{"block_device_mappings":[],"capacity_reservation_specification":[],"cpu_options":[],"credit_specification":[],"description":null,"disable_api_termination":null,"ebs_optimized":null,"elastic_gpu_specifications":[],"elastic_inference_accelerator":[],"hibernation_options":[],"iam_instance_profile":[],"image_id":"ami-1a2b3c","instance_initiated_shutdown_behavior":null,"instance_market_options":[],"instance_type":"t2.micro","kernel_id":null,"key_name":null,"license_specification":[],"monitoring":[],"name_prefix":"foobar","network_interfaces":[],"placement":[],"ram_disk_id":null,"security_group_names":null,"tag_specifications":[],"tags":null,"user_data":null,"vpc_security_group_ids":null},"after_unknown":{"arn":true,"block_device_mappings":[],"capacity_reservation_specification":[],"cpu_options":[],"credit_specification":[],"default_version":true,"elastic_gpu_specifications":[],"elastic_inference_accelerator":[],"hibernation_options":[],"iam_instance_profile":[],"id":true,"instance_market_options":[],"latest_version":true,"license_specification":[],"metadata_options":true,"monitoring":[],"name":true,"network_interfaces":[],"placement":[],"tag_specifications":[]}}}],"configuration":{"root_module":{"resources":[{"address":"aws_autoscaling_group.bar","mode":"managed","type":"aws_autoscaling_group","name":"bar","provider_config_key":"aws","expressions":{"availability_zones":{"constant_value":["us-east-1a"]},"desired_capacity":{"constant_value":1},"launch_template":[{"id":{"references":["aws_launch_template.foobar"]},"version":{"constant_value":"$Latest"}}],"max_size":{"constant_value":1},"min_size":{"constant_value":1}},"schema_version":0},{"address":"aws_launch_template.foobar","mode":"managed","type":"aws_launch_template","name":"foobar","provider_config_key":"aws","expressions":{"image_id":{"constant_value":"ami-1a2b3c"},"instance_type":{"constant_value":"t2.micro"},"name_prefix":{"constant_value":"foobar"}},"schema_version":0}]}}}

tests/functional/test_issue-279/test.feature

@@ -0,0 +1,6 @@
+Feature: ASG
+  @warning
+  Scenario: Health checks not enabled for ASG
+    Given I have aws_autoscaling_group defined
+    Then it must contain health_check_type
+    And its value must match the "(^ELB$)|(^EC2$)" regex