Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.atlas.repository.store.bootstrap; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEnumDef; import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.ObjectUtils; import org.codehaus.jackson.annotate.JsonAutoDetect; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.annotate.JsonSerialize; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; import java.io.File; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE; import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY; /** * Class that handles initial loading of models and patches into typedef store */ public class AtlasTypeDefStoreInitializer { private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeDefStoreInitializer.class); public void initializeStore(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, String typesDirName) { File typesDir = new File(typesDirName); File[] typeDefFiles = typesDir.exists() ? typesDir.listFiles() : null; if (typeDefFiles == null || typeDefFiles.length == 0) { LOG.info("Types directory {} does not exist or not readable or has no typedef files", typesDirName); return; } // sort the files by filename Arrays.sort(typeDefFiles); for (File typeDefFile : typeDefFiles) { if (!typeDefFile.isFile()) { continue; } try { String jsonStr = new String(Files.readAllBytes(typeDefFile.toPath()), StandardCharsets.UTF_8); AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class); if (typesDef == null || typesDef.isEmpty()) { LOG.info("No type in file {}", typeDefFile.getAbsolutePath()); continue; } AtlasTypesDef typesToCreate = getTypesToCreate(typesDef, typeRegistry); AtlasTypesDef typesToUpdate = getTypesToUpdate(typesDef, typeRegistry); if (!typesToCreate.isEmpty() || !typesToUpdate.isEmpty()) { typeDefStore.createUpdateTypesDef(typesToCreate, typesToUpdate); LOG.info("Created/Updated types defined in file {}", typeDefFile.getAbsolutePath()); } else { LOG.info("No new type in file {}", typeDefFile.getAbsolutePath()); } } catch (Throwable t) { LOG.error("error while registering types in file {}", typeDefFile.getAbsolutePath(), t); } } applyTypePatches(typeDefStore, typeRegistry, typesDirName); } public static AtlasTypesDef getTypesToCreate(AtlasTypesDef typesDef, AtlasTypeRegistry typeRegistry) { AtlasTypesDef typesToCreate = new AtlasTypesDef(); if (CollectionUtils.isNotEmpty(typesDef.getEnumDefs())) { for (AtlasEnumDef enumDef : typesDef.getEnumDefs()) { if (!typeRegistry.isRegisteredType(enumDef.getName())) { typesToCreate.getEnumDefs().add(enumDef); } } } if (CollectionUtils.isNotEmpty(typesDef.getStructDefs())) { for (AtlasStructDef structDef : typesDef.getStructDefs()) { if (!typeRegistry.isRegisteredType(structDef.getName())) { typesToCreate.getStructDefs().add(structDef); } } } if (CollectionUtils.isNotEmpty(typesDef.getClassificationDefs())) { for (AtlasClassificationDef classificationDef : typesDef.getClassificationDefs()) { if (!typeRegistry.isRegisteredType(classificationDef.getName())) { typesToCreate.getClassificationDefs().add(classificationDef); } } } if (CollectionUtils.isNotEmpty(typesDef.getEntityDefs())) { for (AtlasEntityDef entityDef : typesDef.getEntityDefs()) { if (!typeRegistry.isRegisteredType(entityDef.getName())) { typesToCreate.getEntityDefs().add(entityDef); } } } return typesToCreate; } public static AtlasTypesDef getTypesToUpdate(AtlasTypesDef typesDef, AtlasTypeRegistry typeRegistry) { AtlasTypesDef typesToUpdate = new AtlasTypesDef(); if (CollectionUtils.isNotEmpty(typesDef.getStructDefs())) { for (AtlasStructDef newStructDef : typesDef.getStructDefs()) { AtlasStructDef oldStructDef = typeRegistry.getStructDefByName(newStructDef.getName()); if (oldStructDef == null) { continue; } if (updateTypeAttributes(oldStructDef, newStructDef)) { typesToUpdate.getStructDefs().add(newStructDef); } } } if (CollectionUtils.isNotEmpty(typesDef.getClassificationDefs())) { for (AtlasClassificationDef newClassifDef : typesDef.getClassificationDefs()) { AtlasClassificationDef oldClassifDef = typeRegistry .getClassificationDefByName(newClassifDef.getName()); if (oldClassifDef == null) { continue; } if (updateTypeAttributes(oldClassifDef, newClassifDef)) { typesToUpdate.getClassificationDefs().add(newClassifDef); } } } if (CollectionUtils.isNotEmpty(typesDef.getEntityDefs())) { for (AtlasEntityDef newEntityDef : typesDef.getEntityDefs()) { AtlasEntityDef oldEntityDef = typeRegistry.getEntityDefByName(newEntityDef.getName()); if (oldEntityDef == null) { continue; } if (updateTypeAttributes(oldEntityDef, newEntityDef)) { typesToUpdate.getEntityDefs().add(newEntityDef); } } } if (CollectionUtils.isNotEmpty(typesDef.getEnumDefs())) { for (AtlasEnumDef newEnumDef : typesDef.getEnumDefs()) { AtlasEnumDef oldEnumDef = typeRegistry.getEnumDefByName(newEnumDef.getName()); if (oldEnumDef == null) { continue; } if (isTypeUpdateApplicable(oldEnumDef, newEnumDef)) { if (CollectionUtils.isNotEmpty(oldEnumDef.getElementDefs())) { for (AtlasEnumElementDef oldEnumElem : oldEnumDef.getElementDefs()) { if (!newEnumDef.hasElement(oldEnumElem.getValue())) { newEnumDef.addElement(oldEnumElem); } } } typesToUpdate.getEnumDefs().add(newEnumDef); } } } return typesToUpdate; } private static boolean updateTypeAttributes(AtlasStructDef oldStructDef, AtlasStructDef newStructDef) { boolean ret = isTypeUpdateApplicable(oldStructDef, newStructDef); if (ret) { // make sure that all attributes in oldDef are in newDef as well if (CollectionUtils.isNotEmpty(oldStructDef.getAttributeDefs())) { for (AtlasAttributeDef oldAttrDef : oldStructDef.getAttributeDefs()) { if (!newStructDef.hasAttribute(oldAttrDef.getName())) { newStructDef.addAttribute(oldAttrDef); } } } } return ret; } private static boolean isTypeUpdateApplicable(AtlasBaseTypeDef oldTypeDef, AtlasBaseTypeDef newTypeDef) { String oldTypeVersion = oldTypeDef.getTypeVersion(); String newTypeVersion = newTypeDef.getTypeVersion(); return ObjectUtils.compare(newTypeVersion, oldTypeVersion) > 0; } private void applyTypePatches(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, String typesDirName) { String typePatchesDirName = typesDirName + File.separator + "patches"; File typePatchesDir = new File(typePatchesDirName); File[] typePatchFiles = typePatchesDir.exists() ? typePatchesDir.listFiles() : null; if (typePatchFiles == null || typePatchFiles.length == 0) { LOG.info("Type patches directory {} does not exist or not readable or has no patches", typePatchesDirName); return; } // sort the files by filename Arrays.sort(typePatchFiles); PatchHandler[] patchHandlers = new PatchHandler[] { new AddAttributePatchHandler(typeDefStore, typeRegistry), new UpdateTypeDefOptionsPatchHandler(typeDefStore, typeRegistry), new UpdateAttributePatchHandler(typeDefStore, typeRegistry) }; Map<String, PatchHandler> patchHandlerRegistry = new HashMap<>(); for (PatchHandler patchHandler : patchHandlers) { for (String supportedAction : patchHandler.getSupportedActions()) { patchHandlerRegistry.put(supportedAction, patchHandler); } } for (File typePatchFile : typePatchFiles) { if (!typePatchFile.isFile()) { continue; } LOG.info("Applying patches in file {}", typePatchFile.getAbsolutePath()); try { String jsonStr = new String(Files.readAllBytes(typePatchFile.toPath()), StandardCharsets.UTF_8); TypeDefPatches patches = AtlasType.fromJson(jsonStr, TypeDefPatches.class); if (patches == null || CollectionUtils.isEmpty(patches.getPatches())) { LOG.info("No patches in file {}", typePatchFile.getAbsolutePath()); continue; } for (TypeDefPatch patch : patches.getPatches()) { PatchHandler patchHandler = patchHandlerRegistry.get(patch.getAction()); if (patchHandler == null) { LOG.error("Unknown patch action {} in file {}. Ignored", patch.getAction(), typePatchFile.getAbsolutePath()); continue; } try { patchHandler.applyPatch(patch); } catch (AtlasBaseException excp) { LOG.error("Failed to apply {} patch in file {}. Ignored", patch.getAction(), typePatchFile.getAbsolutePath(), excp); } } } catch (Throwable t) { LOG.error("Failed to apply patches in file {}. Ignored", typePatchFile.getAbsolutePath(), t); } } } /** * typedef patch details */ @JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) @XmlRootElement @XmlAccessorType(XmlAccessType.PROPERTY) static class TypeDefPatch { private String action; private String typeName; private String applyToVersion; private String updateToVersion; private Map<String, Object> params; private List<AtlasAttributeDef> attributeDefs; private Map<String, String> typeDefOptions; public String getAction() { return action; } public void setAction(String action) { this.action = action; } public String getTypeName() { return typeName; } public void setTypeName(String typeName) { this.typeName = typeName; } public String getApplyToVersion() { return applyToVersion; } public void setApplyToVersion(String applyToVersion) { this.applyToVersion = applyToVersion; } public String getUpdateToVersion() { return updateToVersion; } public void setUpdateToVersion(String updateToVersion) { this.updateToVersion = updateToVersion; } public Map<String, Object> getParams() { return params; } public void setParams(Map<String, Object> params) { this.params = params; } public List<AtlasAttributeDef> getAttributeDefs() { return attributeDefs; } public void setAttributeDefs(List<AtlasAttributeDef> attributeDefs) { this.attributeDefs = attributeDefs; } public Map<String, String> getTypeDefOptions() { return typeDefOptions; } public void setTypeDefOptions(Map<String, String> typeDefOptions) { this.typeDefOptions = typeDefOptions; } } /** * list of typedef patches */ @JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) @XmlRootElement @XmlAccessorType(XmlAccessType.PROPERTY) static class TypeDefPatches { private List<TypeDefPatch> patches; public List<TypeDefPatch> getPatches() { return patches; } public void setPatches(List<TypeDefPatch> patches) { this.patches = patches; } } abstract class PatchHandler { protected final AtlasTypeDefStore typeDefStore; protected final AtlasTypeRegistry typeRegistry; protected final String[] supportedActions; protected PatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, String[] supportedActions) { this.typeDefStore = typeDefStore; this.typeRegistry = typeRegistry; this.supportedActions = supportedActions; } public String[] getSupportedActions() { return supportedActions; } public abstract void applyPatch(TypeDefPatch patch) throws AtlasBaseException; protected boolean isPatchApplicable(TypeDefPatch patch, AtlasBaseTypeDef currentTypeDef) { String currentVersion = currentTypeDef.getTypeVersion(); String applyToVersion = patch.getApplyToVersion(); return currentVersion == null || currentVersion.equalsIgnoreCase(applyToVersion) || currentVersion.startsWith(applyToVersion + "."); } } class AddAttributePatchHandler extends PatchHandler { public AddAttributePatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) { super(typeDefStore, typeRegistry, new String[] { "ADD_ATTRIBUTE" }); } @Override public void applyPatch(TypeDefPatch patch) throws AtlasBaseException { String typeName = patch.getTypeName(); AtlasBaseTypeDef typeDef = typeRegistry.getTypeDefByName(typeName); if (typeDef == null) { throw new AtlasBaseException(AtlasErrorCode.PATCH_FOR_UNKNOWN_TYPE, patch.getAction(), typeName); } if (isPatchApplicable(patch, typeDef)) { if (typeDef.getClass().equals(AtlasEntityDef.class)) { AtlasEntityDef updatedDef = new AtlasEntityDef((AtlasEntityDef) typeDef); for (AtlasAttributeDef attributeDef : patch.getAttributeDefs()) { updatedDef.addAttribute(attributeDef); } updatedDef.setTypeVersion(patch.getUpdateToVersion()); typeDefStore.updateEntityDefByName(typeName, updatedDef); } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { AtlasClassificationDef updatedDef = new AtlasClassificationDef( (AtlasClassificationDef) typeDef); for (AtlasAttributeDef attributeDef : patch.getAttributeDefs()) { updatedDef.addAttribute(attributeDef); } updatedDef.setTypeVersion(patch.getUpdateToVersion()); typeDefStore.updateClassificationDefByName(typeName, updatedDef); } else if (typeDef.getClass().equals(AtlasStructDef.class)) { AtlasStructDef updatedDef = new AtlasStructDef((AtlasStructDef) typeDef); for (AtlasAttributeDef attributeDef : patch.getAttributeDefs()) { updatedDef.addAttribute(attributeDef); } updatedDef.setTypeVersion(patch.getUpdateToVersion()); typeDefStore.updateStructDefByName(typeName, updatedDef); } else { throw new AtlasBaseException(AtlasErrorCode.PATCH_NOT_APPLICABLE_FOR_TYPE, patch.getAction(), typeDef.getClass().getSimpleName()); } } else { LOG.info("patch skipped: typeName={}; applyToVersion={}; updateToVersion={}", patch.getTypeName(), patch.getApplyToVersion(), patch.getUpdateToVersion()); } } } class UpdateAttributePatchHandler extends PatchHandler { public UpdateAttributePatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) { super(typeDefStore, typeRegistry, new String[] { "UPDATE_ATTRIBUTE" }); } @Override public void applyPatch(TypeDefPatch patch) throws AtlasBaseException { String typeName = patch.getTypeName(); AtlasBaseTypeDef typeDef = typeRegistry.getTypeDefByName(typeName); if (typeDef == null) { throw new AtlasBaseException(AtlasErrorCode.PATCH_FOR_UNKNOWN_TYPE, patch.getAction(), typeName); } if (isPatchApplicable(patch, typeDef)) { if (typeDef.getClass().equals(AtlasEntityDef.class)) { AtlasEntityDef updatedDef = new AtlasEntityDef((AtlasEntityDef) typeDef); addOrUpdateAttributes(updatedDef, patch.getAttributeDefs()); updatedDef.setTypeVersion(patch.getUpdateToVersion()); typeDefStore.updateEntityDefByName(typeName, updatedDef); } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { AtlasClassificationDef updatedDef = new AtlasClassificationDef( (AtlasClassificationDef) typeDef); addOrUpdateAttributes(updatedDef, patch.getAttributeDefs()); updatedDef.setTypeVersion(patch.getUpdateToVersion()); typeDefStore.updateClassificationDefByName(typeName, updatedDef); } else if (typeDef.getClass().equals(AtlasStructDef.class)) { AtlasStructDef updatedDef = new AtlasStructDef((AtlasStructDef) typeDef); addOrUpdateAttributes(updatedDef, patch.getAttributeDefs()); updatedDef.setTypeVersion(patch.getUpdateToVersion()); typeDefStore.updateStructDefByName(typeName, updatedDef); } else { throw new AtlasBaseException(AtlasErrorCode.PATCH_NOT_APPLICABLE_FOR_TYPE, patch.getAction(), typeDef.getClass().getSimpleName()); } } else { LOG.info("patch skipped: typeName={}; applyToVersion={}; updateToVersion={}", patch.getTypeName(), patch.getApplyToVersion(), patch.getUpdateToVersion()); } } private void addOrUpdateAttributes(AtlasStructDef structDef, List<AtlasAttributeDef> attributesToUpdate) { for (AtlasAttributeDef attributeToUpdate : attributesToUpdate) { String attrName = attributeToUpdate.getName(); if (structDef.hasAttribute(attrName)) { structDef.removeAttribute(attrName); } structDef.addAttribute(attributeToUpdate); } } } class UpdateTypeDefOptionsPatchHandler extends PatchHandler { public UpdateTypeDefOptionsPatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) { super(typeDefStore, typeRegistry, new String[] { "UPDATE_TYPEDEF_OPTIONS" }); } @Override public void applyPatch(TypeDefPatch patch) throws AtlasBaseException { String typeName = patch.getTypeName(); AtlasBaseTypeDef typeDef = typeRegistry.getTypeDefByName(typeName); if (typeDef == null) { throw new AtlasBaseException(AtlasErrorCode.PATCH_FOR_UNKNOWN_TYPE, patch.getAction(), typeName); } if (MapUtils.isEmpty(patch.getTypeDefOptions())) { throw new AtlasBaseException(AtlasErrorCode.PATCH_INVALID_DATA, patch.getAction(), typeName); } if (isPatchApplicable(patch, typeDef)) { if (typeDef.getClass().equals(AtlasEntityDef.class)) { AtlasEntityDef updatedDef = new AtlasEntityDef((AtlasEntityDef) typeDef); if (updatedDef.getOptions() == null) { updatedDef.setOptions(patch.getTypeDefOptions()); } else { updatedDef.getOptions().putAll(patch.getTypeDefOptions()); } updatedDef.setTypeVersion(patch.getUpdateToVersion()); typeDefStore.updateEntityDefByName(typeName, updatedDef); } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) { AtlasClassificationDef updatedDef = new AtlasClassificationDef( (AtlasClassificationDef) typeDef); if (updatedDef.getOptions() == null) { updatedDef.setOptions(patch.getTypeDefOptions()); } else { updatedDef.getOptions().putAll(patch.getTypeDefOptions()); } updatedDef.setTypeVersion(patch.getUpdateToVersion()); typeDefStore.updateClassificationDefByName(typeName, updatedDef); } else if (typeDef.getClass().equals(AtlasStructDef.class)) { AtlasStructDef updatedDef = new AtlasStructDef((AtlasStructDef) typeDef); if (updatedDef.getOptions() == null) { updatedDef.setOptions(patch.getTypeDefOptions()); } else { updatedDef.getOptions().putAll(patch.getTypeDefOptions()); } updatedDef.setTypeVersion(patch.getUpdateToVersion()); typeDefStore.updateStructDefByName(typeName, updatedDef); } else if (typeDef.getClass().equals(AtlasEnumDef.class)) { AtlasEnumDef updatedDef = new AtlasEnumDef((AtlasEnumDef) typeDef); if (updatedDef.getOptions() == null) { updatedDef.setOptions(patch.getTypeDefOptions()); } else { updatedDef.getOptions().putAll(patch.getTypeDefOptions()); } updatedDef.setTypeVersion(patch.getUpdateToVersion()); typeDefStore.updateEnumDefByName(typeName, updatedDef); } else { throw new AtlasBaseException(AtlasErrorCode.PATCH_NOT_APPLICABLE_FOR_TYPE, patch.getAction(), typeDef.getClass().getSimpleName()); } } else { LOG.info("patch skipped: typeName={}; applyToVersion={}; updateToVersion={}", patch.getTypeName(), patch.getApplyToVersion(), patch.getUpdateToVersion()); } } } }