From d1308826d074f55cf4b6cc1251a3aecc2b359282 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 8 Jan 2025 21:57:08 +0530 Subject: [PATCH 1/2] Removing UpdateClassification array. AppendClassification (being renamed to addOrUpdate) will be responsible for add/update based on classification state in atlas. Incase classification doesnt exist , add flow will be used, otherwise any change or no change in config in a pre-existing classification will be treated for update flow. For Remove Classification Flow, not going to throw any error, instead will be silently logging it in case the classification in the array doesnt exist with Atlas. --- .../model/instance/AtlasEntityHeader.java | 21 ++----- .../graph/v2/ClassificationAssociator.java | 60 +++++++++++-------- 2 files changed, 41 insertions(+), 40 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java index c628451c2c..e9985354ed 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java @@ -61,10 +61,7 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable { private String displayText = null; private List classificationNames = null; private List classifications = null; - private List appendClassifications = null; - - - private List updateClassifications = null; + private List addOrUpdateClassifications = null; private List removeClassifications = null; private List meaningNames = null; private List meanings = null; @@ -106,20 +103,12 @@ public AtlasEntityHeader(String typeName, Map attributes) { setLabels(null); } - public List getAppendClassifications() { - return appendClassifications; - } - - public void setAppendClassifications(List appendClassifications) { - this.appendClassifications = appendClassifications; - } - - public List getUpdateClassifications() { - return updateClassifications; + public List getAddOrUpdateClassifications() { + return addOrUpdateClassifications; } - public void setUpdateClassifications(List updateClassifications) { - this.updateClassifications = updateClassifications; + public void setAddOrUpdateClassifications(List addOrUpdateClassifications) { + this.addOrUpdateClassifications = addOrUpdateClassifications; } public List getRemoveClassifications() { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java index b0e85782df..76e4b0548b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java @@ -54,7 +54,6 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; import static org.apache.atlas.AtlasConfiguration.ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES; @@ -242,45 +241,58 @@ public void setClassifications(Map map, boolean overr RequestContext.get().setDelayTagNotifications(false); } + private Map> validateAndTransfer(AtlasEntityHeader incomingEntityHeader, AtlasEntityHeader entityToBeChanged) throws AtlasBaseException { Map> operationListMap = new HashMap<>(); - Set requiredClassificationKeys = Stream.concat( - Optional.ofNullable(incomingEntityHeader.getRemoveClassifications()).orElse(Collections.emptyList()).stream(), - Optional.ofNullable(incomingEntityHeader.getUpdateClassifications()).orElse(Collections.emptyList()).stream() - ).filter(classification -> classification.getEntityGuid().equals(entityToBeChanged.getGuid())) - .map(this::generateClassificationComparisonKey) - .collect(Collectors.toSet()); + // Initialize required collections + Set preExistingClassificationKeys = new HashSet<>(); + List filteredRemoveClassifications = new ArrayList<>(); - Set preExistingClassificationKeys = Optional.ofNullable(entityToBeChanged.getClassifications()) - .orElse(Collections.emptyList()) - .stream() - .filter(classification -> classification.getEntityGuid().equals(entityToBeChanged.getGuid())) - .map(this::generateClassificationComparisonKey) - .collect(Collectors.toSet()); + ListOps listOps = new ListOps<>(); - Set diff = requiredClassificationKeys.stream() - .filter(key -> !preExistingClassificationKeys.contains(key)) - .collect(Collectors.toSet()); + // First loop: Process pre-existing classifications and removeClassifications + for (AtlasClassification classification : Optional.ofNullable(entityToBeChanged.getClassifications()).orElse(Collections.emptyList())) { + if (entityToBeChanged.getGuid().equals(classification.getEntityGuid())) { + String key = generateClassificationComparisonKey(classification); + preExistingClassificationKeys.add(key); // Track pre-existing keys + } + } - if (!diff.isEmpty()) { - String firstTypeName = diff.iterator().next().split("\\|")[1]; - throw new AtlasBaseException(AtlasErrorCode.CLASSIFICATION_NOT_ASSOCIATED_WITH_ENTITY, firstTypeName); + for (AtlasClassification classification : Optional.ofNullable(incomingEntityHeader.getRemoveClassifications()).orElse(Collections.emptyList())) { + if (entityToBeChanged.getGuid().equals(classification.getEntityGuid())) { + String key = generateClassificationComparisonKey(classification); + // If the classification doesn't exist in pre-existing keys, log it + if (!preExistingClassificationKeys.contains(key)) { + String typeName = key.split("\\|")[1]; + LOG.info("Classification {} is not associated with entity {}", typeName, entityToBeChanged.getGuid()); + } else { + filteredRemoveClassifications.add(classification); + } + } } - List filteredClassifications = Optional.ofNullable(incomingEntityHeader.getAppendClassifications()) + // Process incoming classifications for add/update + List filteredClassifications = Optional.ofNullable(incomingEntityHeader.getAddOrUpdateClassifications()) .orElse(Collections.emptyList()) .stream() .filter(classification -> classification.getEntityGuid().equals(entityToBeChanged.getGuid())) - .filter(appendClassification -> !preExistingClassificationKeys.contains(generateClassificationComparisonKey(appendClassification))) .collect(Collectors.toList()); - bucket(PROCESS_DELETE, operationListMap, incomingEntityHeader.getRemoveClassifications()); - bucket(PROCESS_UPDATE, operationListMap, incomingEntityHeader.getUpdateClassifications()); - bucket(PROCESS_ADD, operationListMap, filteredClassifications); + // Filter incoming and existing classifications for updates and additions + List incomingClassifications = listOps.filter(incomingEntityHeader.getGuid(), filteredClassifications); + List entityClassifications = listOps.filter(entityToBeChanged.getGuid(), entityToBeChanged.getClassifications()); + + // Bucket the operations (add, update, delete) + bucket(PROCESS_DELETE, operationListMap, filteredRemoveClassifications); + bucket(PROCESS_UPDATE, operationListMap, listOps.intersect(incomingClassifications, entityClassifications)); + bucket(PROCESS_ADD, operationListMap, listOps.subtract(incomingClassifications, entityClassifications)); + return operationListMap; } + + private String generateClassificationComparisonKey(AtlasClassification classification) { return classification.getEntityGuid() + "|" + classification.getTypeName(); } From 83658034d01fd9c28d59c7749236d0fc2db68a3b Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 10 Jan 2025 11:03:46 +0530 Subject: [PATCH 2/2] Removed trailing new-lines and comments --- .../store/graph/v2/ClassificationAssociator.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java index 76e4b0548b..a7d9f34d06 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java @@ -241,17 +241,14 @@ public void setClassifications(Map map, boolean overr RequestContext.get().setDelayTagNotifications(false); } - private Map> validateAndTransfer(AtlasEntityHeader incomingEntityHeader, AtlasEntityHeader entityToBeChanged) throws AtlasBaseException { Map> operationListMap = new HashMap<>(); - // Initialize required collections Set preExistingClassificationKeys = new HashSet<>(); List filteredRemoveClassifications = new ArrayList<>(); ListOps listOps = new ListOps<>(); - // First loop: Process pre-existing classifications and removeClassifications for (AtlasClassification classification : Optional.ofNullable(entityToBeChanged.getClassifications()).orElse(Collections.emptyList())) { if (entityToBeChanged.getGuid().equals(classification.getEntityGuid())) { String key = generateClassificationComparisonKey(classification); @@ -272,18 +269,15 @@ private Map> validateAndTransfer(AtlasEntityHe } } - // Process incoming classifications for add/update List filteredClassifications = Optional.ofNullable(incomingEntityHeader.getAddOrUpdateClassifications()) .orElse(Collections.emptyList()) .stream() .filter(classification -> classification.getEntityGuid().equals(entityToBeChanged.getGuid())) .collect(Collectors.toList()); - // Filter incoming and existing classifications for updates and additions List incomingClassifications = listOps.filter(incomingEntityHeader.getGuid(), filteredClassifications); List entityClassifications = listOps.filter(entityToBeChanged.getGuid(), entityToBeChanged.getClassifications()); - // Bucket the operations (add, update, delete) bucket(PROCESS_DELETE, operationListMap, filteredRemoveClassifications); bucket(PROCESS_UPDATE, operationListMap, listOps.intersect(incomingClassifications, entityClassifications)); bucket(PROCESS_ADD, operationListMap, listOps.subtract(incomingClassifications, entityClassifications)); @@ -291,8 +285,6 @@ private Map> validateAndTransfer(AtlasEntityHe return operationListMap; } - - private String generateClassificationComparisonKey(AtlasClassification classification) { return classification.getEntityGuid() + "|" + classification.getTypeName(); }