diff --git a/moto/es/models.py b/moto/es/models.py index 4cc65d6fb652..729a63f72677 100644 --- a/moto/es/models.py +++ b/moto/es/models.py @@ -1,155 +1,10 @@ -from typing import Any, Dict, List - from moto.core.base_backend import BackendDict, BaseBackend -from moto.core.common_models import BaseModel -from moto.moto_api._internal import mock_random -from moto.utilities.utils import get_partition - -from .exceptions import DomainNotFound - - -class Domain(BaseModel): - def __init__( - self, - region_name: str, - domain_name: str, - es_version: str, - elasticsearch_cluster_config: Dict[str, Any], - ebs_options: Dict[str, Any], - access_policies: Dict[str, Any], - snapshot_options: Dict[str, Any], - vpc_options: Dict[str, Any], - cognito_options: Dict[str, Any], - encryption_at_rest_options: Dict[str, Any], - node_to_node_encryption_options: Dict[str, Any], - advanced_options: Dict[str, Any], - log_publishing_options: Dict[str, Any], - domain_endpoint_options: Dict[str, Any], - advanced_security_options: Dict[str, Any], - auto_tune_options: Dict[str, Any], - ): - self.domain_id = mock_random.get_random_hex(8) - self.region_name = region_name - self.domain_name = domain_name - self.es_version = es_version - self.elasticsearch_cluster_config = elasticsearch_cluster_config - self.ebs_options = ebs_options - self.access_policies = access_policies - self.snapshot_options = snapshot_options - self.vpc_options = vpc_options - self.cognito_options = cognito_options - self.encryption_at_rest_options = encryption_at_rest_options - self.node_to_node_encryption_options = node_to_node_encryption_options - self.advanced_options = advanced_options - self.log_publishing_options = log_publishing_options - self.domain_endpoint_options = domain_endpoint_options - self.advanced_security_options = advanced_security_options - self.auto_tune_options = auto_tune_options - if self.auto_tune_options: - self.auto_tune_options["State"] = "ENABLED" - - @property - def arn(self) -> str: - return f"arn:{get_partition(self.region_name)}:es:{self.region_name}:domain/{self.domain_id}" - - def to_json(self) -> Dict[str, Any]: - return { - "DomainId": self.domain_id, - "DomainName": self.domain_name, - "ARN": self.arn, - "Created": True, - "Deleted": False, - "Processing": False, - "UpgradeProcessing": False, - "ElasticsearchVersion": self.es_version, - "ElasticsearchClusterConfig": self.elasticsearch_cluster_config, - "EBSOptions": self.ebs_options, - "AccessPolicies": self.access_policies, - "SnapshotOptions": self.snapshot_options, - "VPCOptions": self.vpc_options, - "CognitoOptions": self.cognito_options, - "EncryptionAtRestOptions": self.encryption_at_rest_options, - "NodeToNodeEncryptionOptions": self.node_to_node_encryption_options, - "AdvancedOptions": self.advanced_options, - "LogPublishingOptions": self.log_publishing_options, - "DomainEndpointOptions": self.domain_endpoint_options, - "AdvancedSecurityOptions": self.advanced_security_options, - "AutoTuneOptions": self.auto_tune_options, - } class ElasticsearchServiceBackend(BaseBackend): - """Implementation of ElasticsearchService APIs.""" - def __init__(self, region_name: str, account_id: str): + # Functionality is part of OpenSearch, as that includes all of ES functionality + more super().__init__(region_name, account_id) - self.domains: Dict[str, Domain] = dict() - - def create_elasticsearch_domain( - self, - domain_name: str, - elasticsearch_version: str, - elasticsearch_cluster_config: Dict[str, Any], - ebs_options: Dict[str, Any], - access_policies: Dict[str, Any], - snapshot_options: Dict[str, Any], - vpc_options: Dict[str, Any], - cognito_options: Dict[str, Any], - encryption_at_rest_options: Dict[str, Any], - node_to_node_encryption_options: Dict[str, Any], - advanced_options: Dict[str, Any], - log_publishing_options: Dict[str, Any], - domain_endpoint_options: Dict[str, Any], - advanced_security_options: Dict[str, Any], - auto_tune_options: Dict[str, Any], - ) -> Dict[str, Any]: - # TODO: Persist/Return other attributes - new_domain = Domain( - region_name=self.region_name, - domain_name=domain_name, - es_version=elasticsearch_version, - elasticsearch_cluster_config=elasticsearch_cluster_config, - ebs_options=ebs_options, - access_policies=access_policies, - snapshot_options=snapshot_options, - vpc_options=vpc_options, - cognito_options=cognito_options, - encryption_at_rest_options=encryption_at_rest_options, - node_to_node_encryption_options=node_to_node_encryption_options, - advanced_options=advanced_options, - log_publishing_options=log_publishing_options, - domain_endpoint_options=domain_endpoint_options, - advanced_security_options=advanced_security_options, - auto_tune_options=auto_tune_options, - ) - self.domains[domain_name] = new_domain - return new_domain.to_json() - - def delete_elasticsearch_domain(self, domain_name: str) -> None: - if domain_name not in self.domains: - raise DomainNotFound(domain_name) - del self.domains[domain_name] - - def describe_elasticsearch_domain(self, domain_name: str) -> Dict[str, Any]: - if domain_name not in self.domains: - raise DomainNotFound(domain_name) - return self.domains[domain_name].to_json() - - def list_domain_names(self) -> List[Dict[str, str]]: - """ - The engine-type parameter is not yet supported. - Pagination is not yet implemented. - """ - return [{"DomainName": domain.domain_name} for domain in self.domains.values()] - - def describe_elasticsearch_domains( - self, domain_names: List[str] - ) -> List[Dict[str, Any]]: - queried_domains = [] - for domain_name in domain_names: - if domain_name in self.domains: - queried_domains.append(self.domains[domain_name].to_json()) - return queried_domains es_backends = BackendDict(ElasticsearchServiceBackend, "es") diff --git a/moto/es/responses.py b/moto/es/responses.py deleted file mode 100644 index 35293b33d36e..000000000000 --- a/moto/es/responses.py +++ /dev/null @@ -1,107 +0,0 @@ -import json -import re -from typing import Any - -from moto.core.common_types import TYPE_RESPONSE -from moto.core.responses import BaseResponse - -from .exceptions import InvalidDomainName -from .models import ElasticsearchServiceBackend, es_backends - - -class ElasticsearchServiceResponse(BaseResponse): - """Handler for ElasticsearchService requests and responses.""" - - def __init__(self) -> None: - super().__init__(service_name="es") - - @property - def es_backend(self) -> ElasticsearchServiceBackend: - """Return backend instance specific for this region.""" - return es_backends[self.current_account][self.region] - - @classmethod - def list_domains(cls, request: Any, full_url: str, headers: Any) -> TYPE_RESPONSE: # type: ignore - response = ElasticsearchServiceResponse() - response.setup_class(request, full_url, headers) - if request.method == "GET": - return response.list_domain_names() - - @classmethod - def domains(cls, request: Any, full_url: str, headers: Any) -> TYPE_RESPONSE: # type: ignore - response = ElasticsearchServiceResponse() - response.setup_class(request, full_url, headers) - if request.method == "POST": - return response.create_elasticsearch_domain() - - @classmethod - def domain(cls, request: Any, full_url: str, headers: Any) -> TYPE_RESPONSE: # type: ignore - response = ElasticsearchServiceResponse() - response.setup_class(request, full_url, headers) - if request.method == "DELETE": - return response.delete_elasticsearch_domain() - if request.method == "GET": - return response.describe_elasticsearch_domain() - - def create_elasticsearch_domain(self) -> TYPE_RESPONSE: - params = json.loads(self.body) - domain_name = params.get("DomainName") - if not re.match(r"^[a-z][a-z0-9\-]+$", domain_name): - raise InvalidDomainName(domain_name) - elasticsearch_version = params.get("ElasticsearchVersion") - elasticsearch_cluster_config = params.get("ElasticsearchClusterConfig") - ebs_options = params.get("EBSOptions") - access_policies = params.get("AccessPolicies") - snapshot_options = params.get("SnapshotOptions") - vpc_options = params.get("VPCOptions") - cognito_options = params.get("CognitoOptions") - encryption_at_rest_options = params.get("EncryptionAtRestOptions") - node_to_node_encryption_options = params.get("NodeToNodeEncryptionOptions") - advanced_options = params.get("AdvancedOptions") - log_publishing_options = params.get("LogPublishingOptions") - domain_endpoint_options = params.get("DomainEndpointOptions") - advanced_security_options = params.get("AdvancedSecurityOptions") - auto_tune_options = params.get("AutoTuneOptions") - domain_status = self.es_backend.create_elasticsearch_domain( - domain_name=domain_name, - elasticsearch_version=elasticsearch_version, - elasticsearch_cluster_config=elasticsearch_cluster_config, - ebs_options=ebs_options, - access_policies=access_policies, - snapshot_options=snapshot_options, - vpc_options=vpc_options, - cognito_options=cognito_options, - encryption_at_rest_options=encryption_at_rest_options, - node_to_node_encryption_options=node_to_node_encryption_options, - advanced_options=advanced_options, - log_publishing_options=log_publishing_options, - domain_endpoint_options=domain_endpoint_options, - advanced_security_options=advanced_security_options, - auto_tune_options=auto_tune_options, - ) - return 200, {}, json.dumps({"DomainStatus": domain_status}) - - def delete_elasticsearch_domain(self) -> TYPE_RESPONSE: - domain_name = self.path.split("/")[-1] - self.es_backend.delete_elasticsearch_domain(domain_name=domain_name) - return 200, {}, json.dumps(dict()) - - def describe_elasticsearch_domain(self) -> TYPE_RESPONSE: - domain_name = self.path.split("/")[-1] - if not re.match(r"^[a-z][a-z0-9\-]+$", domain_name): - raise InvalidDomainName(domain_name) - domain_status = self.es_backend.describe_elasticsearch_domain( - domain_name=domain_name - ) - return 200, {}, json.dumps({"DomainStatus": domain_status}) - - def list_domain_names(self) -> TYPE_RESPONSE: - domain_names = self.es_backend.list_domain_names() - return 200, {}, json.dumps({"DomainNames": domain_names}) - - def describe_elasticsearch_domains(self) -> TYPE_RESPONSE: - domain_names = self._get_param("DomainNames") - domain_list = self.es_backend.describe_elasticsearch_domains( - domain_names=domain_names, - ) - return 200, {}, json.dumps({"DomainStatusList": domain_list}) diff --git a/moto/es/urls.py b/moto/es/urls.py index fbee7fcb7060..4c1ae5294eaf 100644 --- a/moto/es/urls.py +++ b/moto/es/urls.py @@ -1,17 +1,15 @@ from moto.opensearch.responses import OpenSearchServiceResponse -from .responses import ElasticsearchServiceResponse - url_bases = [ r"https?://es\.(.+)\.amazonaws\.com", ] url_paths = { - "{0}/2015-01-01/domain$": ElasticsearchServiceResponse.list_domains, - "{0}/2015-01-01/es/domain$": ElasticsearchServiceResponse.domains, - "{0}/2015-01-01/es/domain/(?P[^/]+)": ElasticsearchServiceResponse.domain, - "{0}/2015-01-01/es/domain-info$": ElasticsearchServiceResponse.dispatch, + "{0}/2015-01-01/domain$": OpenSearchServiceResponse.list_domains, + "{0}/2015-01-01/es/domain$": OpenSearchServiceResponse.domains, + "{0}/2015-01-01/es/domain/(?P[^/]+)": OpenSearchServiceResponse.domain, + "{0}/2015-01-01/es/domain-info$": OpenSearchServiceResponse.list_domains, "{0}/2021-01-01/domain$": OpenSearchServiceResponse.dispatch, "{0}/2021-01-01/opensearch/compatibleVersions": OpenSearchServiceResponse.dispatch, "{0}/2021-01-01/opensearch/domain": OpenSearchServiceResponse.dispatch, diff --git a/moto/opensearch/exceptions.py b/moto/opensearch/exceptions.py index 55504a02d373..97f9e37adaad 100644 --- a/moto/opensearch/exceptions.py +++ b/moto/opensearch/exceptions.py @@ -4,6 +4,8 @@ class ResourceNotFoundException(JsonRESTError): + code = 409 + def __init__(self, name: str): super().__init__("ResourceNotFoundException", f"Domain not found: {name}") diff --git a/moto/opensearch/models.py b/moto/opensearch/models.py index 2e7242983b6a..4927dcbb5809 100644 --- a/moto/opensearch/models.py +++ b/moto/opensearch/models.py @@ -85,6 +85,9 @@ def __init__( auto_tune_options: Dict[str, Any], off_peak_window_options: Dict[str, Any], software_update_options: Dict[str, bool], + is_es: bool, + elasticsearch_version: Optional[str], + elasticsearch_cluster_config: Optional[str], ): self.domain_id = f"{account_id}/{domain_name}" self.domain_name = domain_name @@ -113,10 +116,16 @@ def __init__( advanced_security_options or default_advanced_security_options ) self.auto_tune_options = auto_tune_options or {"State": "ENABLE_IN_PROGRESS"} + if not self.auto_tune_options.get("State"): + self.auto_tune_options["State"] = "ENABLED" self.off_peak_windows_options = off_peak_window_options self.software_update_options = ( software_update_options or default_software_update_options ) + self.engine_type = "Elasticsearch" if is_es else "OpenSearch" + self.is_es = is_es + self.elasticsearch_version = elasticsearch_version + self.elasticsearch_cluster_config = elasticsearch_cluster_config self.deleted = False self.processing = False @@ -157,6 +166,8 @@ def dct_options(self) -> Dict[str, Any]: "AutoTuneOptions": self.auto_tune_options, "OffPeakWindowsOptions": self.off_peak_windows_options, "SoftwareUpdateOptions": self.software_update_options, + "ElasticsearchVersion": self.elasticsearch_version, + "ElasticsearchClusterConfig": self.elasticsearch_cluster_config, } def to_dict(self) -> Dict[str, Any]: @@ -259,6 +270,9 @@ def create_domain( auto_tune_options: Dict[str, Any], off_peak_window_options: Dict[str, Any], software_update_options: Dict[str, Any], + is_es: bool, + elasticsearch_version: Optional[str], + elasticsearch_cluster_config: Optional[str], ) -> OpenSearchDomain: domain = OpenSearchDomain( account_id=self.account_id, @@ -280,6 +294,9 @@ def create_domain( auto_tune_options=auto_tune_options, off_peak_window_options=off_peak_window_options, software_update_options=software_update_options, + is_es=is_es, + elasticsearch_version=elasticsearch_version, + elasticsearch_cluster_config=elasticsearch_cluster_config, ) self.domains[domain_name] = domain if tag_list: @@ -355,23 +372,17 @@ def remove_tags(self, arn: str, tag_keys: List[str]) -> None: def list_domain_names(self, engine_type: str) -> List[Dict[str, str]]: domains = [] + if engine_type and engine_type not in ["Elasticsearch", "OpenSearch"]: + raise EngineTypeNotFoundException(engine_type) for domain in self.domains.values(): if engine_type: - if engine_type in domain.engine_version.options: + if engine_type == domain.engine_type: domains.append( - { - "DomainName": domain.domain_name, - "EngineType": engine_type.split("_")[0], - } + {"DomainName": domain.domain_name, "EngineType": engine_type} ) - else: - raise EngineTypeNotFoundException(domain.domain_name) else: domains.append( - { - "DomainName": domain.domain_name, - "EngineType": domain.engine_version.options.split("_")[0], - } + {"DomainName": domain.domain_name, "EngineType": domain.engine_type} ) return domains diff --git a/moto/opensearch/responses.py b/moto/opensearch/responses.py index 4034a16b2ba9..efb6f9badc1d 100644 --- a/moto/opensearch/responses.py +++ b/moto/opensearch/responses.py @@ -1,8 +1,12 @@ """Handles incoming opensearch requests, invokes methods, returns responses.""" import json +import re +from typing import Any +from moto.core.common_types import TYPE_RESPONSE from moto.core.responses import BaseResponse +from moto.es.exceptions import InvalidDomainName from .models import OpenSearchServiceBackend, opensearch_backends @@ -18,8 +22,35 @@ def opensearch_backend(self) -> OpenSearchServiceBackend: """Return backend instance specific for this region.""" return opensearch_backends[self.current_account][self.region] + @classmethod + def list_domains(cls, request: Any, full_url: str, headers: Any) -> TYPE_RESPONSE: # type: ignore + response = cls() + response.setup_class(request, full_url, headers) + if request.method == "GET": + return 200, {}, response.list_domain_names() + if request.method == "POST": + return 200, {}, response.describe_domains() + + @classmethod + def domains(cls, request: Any, full_url: str, headers: Any) -> TYPE_RESPONSE: # type: ignore + response = cls() + response.setup_class(request, full_url, headers) + if request.method == "POST": + return 200, {}, response.create_domain() + + @classmethod + def domain(cls, request: Any, full_url: str, headers: Any) -> TYPE_RESPONSE: # type: ignore + response = cls() + response.setup_class(request, full_url, headers) + if request.method == "DELETE": + return 200, {}, response.delete_domain() + if request.method == "GET": + return 200, {}, response.describe_domain() + def create_domain(self) -> str: domain_name = self._get_param("DomainName") + if not re.match(r"^[a-z][a-z0-9\-]+$", domain_name): + raise InvalidDomainName(domain_name) engine_version = self._get_param("EngineVersion") cluster_config = self._get_param("ClusterConfig") ebs_options = self._get_param("EBSOptions") @@ -37,6 +68,10 @@ def create_domain(self) -> str: auto_tune_options = self._get_param("AutoTuneOptions") off_peak_window_options = self._get_param("OffPeakWindowOptions") software_update_options = self._get_param("SoftwareUpdateOptions") + # ElasticSearch specific options + is_es = self.parsed_url.path.endswith("/es/domain") + elasticsearch_version = self._get_param("ElasticsearchVersion") + elasticsearch_cluster_config = self._get_param("ElasticsearchClusterConfig") domain = self.opensearch_backend.create_domain( domain_name=domain_name, engine_version=engine_version, @@ -56,6 +91,9 @@ def create_domain(self) -> str: auto_tune_options=auto_tune_options, off_peak_window_options=off_peak_window_options, software_update_options=software_update_options, + is_es=is_es, + elasticsearch_version=elasticsearch_version, + elasticsearch_cluster_config=elasticsearch_cluster_config, ) return json.dumps(dict(DomainStatus=domain.to_dict())) @@ -67,21 +105,23 @@ def get_compatible_versions(self) -> str: return json.dumps(dict(CompatibleVersions=compatible_versions)) def delete_domain(self) -> str: - domain_name = self._get_param("DomainName") + domain_name = self.path.split("/")[-1] domain = self.opensearch_backend.delete_domain( domain_name=domain_name, ) return json.dumps(dict(DomainStatus=domain.to_dict())) def describe_domain(self) -> str: - domain_name = self._get_param("DomainName") + domain_name = self.path.split("/")[-1] + if not re.match(r"^[a-z][a-z0-9\-]+$", domain_name): + raise InvalidDomainName(domain_name) domain = self.opensearch_backend.describe_domain( domain_name=domain_name, ) return json.dumps(dict(DomainStatus=domain.to_dict())) def describe_domain_config(self) -> str: - domain_name = self._get_param("DomainName") + domain_name = self.path.split("/")[-1] domain = self.opensearch_backend.describe_domain_config( domain_name=domain_name, ) diff --git a/tests/test_es/test_es.py b/tests/test_es/test_es.py index 259f444da912..a696d3e39a13 100644 --- a/tests/test_es/test_es.py +++ b/tests/test_es/test_es.py @@ -5,6 +5,7 @@ from botocore.exceptions import ClientError from moto import mock_aws +from tests import DEFAULT_ACCOUNT_ID # See our Development Tips on writing tests for hints on how to write good tests: # http://docs.getmoto.org/en/latest/docs/contributing/development_tips/tests.html @@ -33,7 +34,10 @@ def test_create_elasticsearch_domain_minimal(): domain = resp["DomainStatus"] assert domain["DomainName"] == "motosearch" - assert domain["ARN"] == f"arn:aws:es:us-east-2:domain/{domain['DomainId']}" + assert ( + domain["ARN"] + == f"arn:aws:es:us-east-2:{DEFAULT_ACCOUNT_ID}:domain/{domain['DomainName']}" + ) assert domain["Created"] is True assert domain["Deleted"] is False assert domain["Processing"] is False @@ -188,7 +192,10 @@ def test_describe_elasticsearch_domain(): domain = resp["DomainStatus"] assert domain["DomainName"] == "motosearch" - assert domain["ARN"] == f"arn:aws:es:ap-southeast-1:domain/{domain['DomainId']}" + assert ( + domain["ARN"] + == f"arn:aws:es:ap-southeast-1:{DEFAULT_ACCOUNT_ID}:domain/{domain['DomainName']}" + ) assert domain["Created"] is True assert domain["Deleted"] is False assert domain["Processing"] is False @@ -204,6 +211,18 @@ def test_list_domain_names_initial(): assert resp["DomainNames"] == [] +@mock_aws +def test_list_domain_names_enginetype(): + client = boto3.client("es", region_name="us-east-1") + client.create_elasticsearch_domain(DomainName="elasticsearch-domain") + + resp = client.list_domain_names(EngineType="Elasticsearch") + assert len(resp["DomainNames"]) == 1 + + resp = client.list_domain_names(EngineType="OpenSearch") + assert len(resp["DomainNames"]) == 0 + + @mock_aws def test_list_domain_names_with_multiple_domains(): client = boto3.client("es", region_name="eu-west-1") @@ -214,7 +233,9 @@ def test_list_domain_names_with_multiple_domains(): assert len(resp["DomainNames"]) == 4 for name in domain_names: - assert {"DomainName": name} in resp["DomainNames"] + assert {"DomainName": name, "EngineType": "Elasticsearch"} in resp[ + "DomainNames" + ] @mock_aws @@ -240,3 +261,33 @@ def test_describe_elasticsearch_domains(): # Test for invalid domain name resp = client.describe_elasticsearch_domains(DomainNames=["invalid"]) assert len(resp["DomainStatusList"]) == 0 + + +@mock_aws +def test_list_domain_names_opensearch(): + opensearch_client = boto3.client("opensearch", region_name="us-east-2") + status = opensearch_client.create_domain(DomainName="moto-opensearch")[ + "DomainStatus" + ] + assert status["Created"] + assert "DomainId" in status + assert "DomainName" in status + assert status["DomainName"] == "moto-opensearch" + + # ensure that elasticsearch client can describe opensearch domains as well + es_client = boto3.client("es", region_name="us-east-2") + domain_names = es_client.list_domain_names()["DomainNames"] + assert len(domain_names) == 1 + assert domain_names[0]["DomainName"] == "moto-opensearch" + assert domain_names[0]["EngineType"] == "OpenSearch" + + +@mock_aws +def test_list_domain_names_opensearch_deleted(): + opensearch_client = boto3.client("opensearch", region_name="us-east-2") + opensearch_client.create_domain(DomainName="moto-opensearch") + opensearch_client.delete_domain(DomainName="moto-opensearch") + + # ensure that elasticsearch client can describe opensearch domains as well + es_client = boto3.client("es", region_name="us-east-2") + assert es_client.list_domain_names()["DomainNames"] == [] diff --git a/tests/test_opensearch/test_opensearch.py b/tests/test_opensearch/test_opensearch.py index 22955de1b435..7dd80864a80e 100644 --- a/tests/test_opensearch/test_opensearch.py +++ b/tests/test_opensearch/test_opensearch.py @@ -117,8 +117,17 @@ def test_describe_domain(): def test_delete_domain(): client = boto3.client("opensearch", region_name="eu-west-1") client.create_domain(DomainName="testdn") - client.delete_domain(DomainName="testdn") + resp = client.delete_domain(DomainName="testdn") + status = resp["DomainStatus"] + assert "DomainId" in status + assert "DomainName" in status + assert status["Deleted"] # assume deleted completion + assert status["DomainName"] == "testdn" + +@mock_aws +def test_delete_invalid_domain(): + client = boto3.client("opensearch", region_name="eu-west-1") with pytest.raises(ClientError) as exc: client.describe_domain(DomainName="testdn") err = exc.value.response["Error"] @@ -216,7 +225,7 @@ def test_list_unknown_domain_names_engine_type(): client.list_domain_names(EngineType="unknown") err = exc.value.response["Error"] assert err["Code"] == "EngineTypeNotFoundException" - assert err["Message"] == "Engine Type not found: testdn" + assert err["Message"] == "Engine Type not found: unknown" @mock_aws