Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 701E8200BC5 for ; Tue, 8 Nov 2016 01:00:30 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 6E8B3160AEC; Tue, 8 Nov 2016 00:00:30 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id EC14B160AF9 for ; Tue, 8 Nov 2016 01:00:27 +0100 (CET) Received: (qmail 77241 invoked by uid 500); 8 Nov 2016 00:00:27 -0000 Mailing-List: contact commits-help@atlas.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@atlas.incubator.apache.org Delivered-To: mailing list commits@atlas.incubator.apache.org Received: (qmail 77228 invoked by uid 99); 8 Nov 2016 00:00:27 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd4-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 08 Nov 2016 00:00:27 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd4-us-west.apache.org (ASF Mail Server at spamd4-us-west.apache.org) with ESMTP id 0F2DFC00B6 for ; Tue, 8 Nov 2016 00:00:26 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd4-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -6.219 X-Spam-Level: X-Spam-Status: No, score=-6.219 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-2.999] autolearn=disabled Received: from mx1-lw-us.apache.org ([10.40.0.8]) by localhost (spamd4-us-west.apache.org [10.40.0.11]) (amavisd-new, port 10024) with ESMTP id JaTlr_bqCnxk for ; Tue, 8 Nov 2016 00:00:14 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-us.apache.org (ASF Mail Server at mx1-lw-us.apache.org) with SMTP id 929765FC86 for ; Tue, 8 Nov 2016 00:00:13 +0000 (UTC) Received: (qmail 76908 invoked by uid 99); 8 Nov 2016 00:00:13 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 08 Nov 2016 00:00:13 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id CAD93E02E4; Tue, 8 Nov 2016 00:00:12 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: madhan@apache.org To: commits@atlas.incubator.apache.org Date: Tue, 08 Nov 2016 00:00:15 -0000 Message-Id: In-Reply-To: <5c956ddbb71347668ec4ec5e55b840c7@git.apache.org> References: <5c956ddbb71347668ec4ec5e55b840c7@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [4/4] incubator-atlas git commit: ATLAS-1272: updated types bootstrap to load from new format typedef JSON files archived-at: Tue, 08 Nov 2016 00:00:30 -0000 ATLAS-1272: updated types bootstrap to load from new format typedef JSON files Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/def9e385 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/def9e385 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/def9e385 Branch: refs/heads/master Commit: def9e385cc17c2bceb9450c82947370fadc09c3c Parents: 2ea3a45 Author: Madhan Neethiraj Authored: Sun Nov 6 19:32:16 2016 -0800 Committer: Madhan Neethiraj Committed: Mon Nov 7 14:06:57 2016 -0800 ---------------------------------------------------------------------- addons/falcon-bridge/pom.xml | 18 - .../atlas/falcon/bridge/FalconBridge.java | 46 +- .../falcon/model/FalconDataModelGenerator.java | 214 -------- .../apache/atlas/falcon/hook/FalconHookIT.java | 22 +- addons/hdfs-model/pom.xml | 18 - .../atlas/fs/model/FSDataModelGenerator.java | 38 -- .../org/apache/atlas/fs/model/FSDataModel.scala | 95 ---- .../apache/atlas/fs/model/HDFSModelTest.java | 83 ---- addons/hive-bridge/pom.xml | 18 - .../atlas/hive/bridge/ColumnLineageUtils.java | 5 +- .../atlas/hive/bridge/HiveMetaStoreBridge.java | 101 ++-- .../org/apache/atlas/hive/hook/HiveHook.java | 9 +- .../hive/model/HiveDataModelGenerator.java | 350 ------------- .../java/org/apache/atlas/hive/HiveITBase.java | 4 +- .../hive/bridge/HiveMetaStoreBridgeTest.java | 6 +- .../org/apache/atlas/hive/hook/HiveHookIT.java | 73 ++- addons/models/0010-base_model.json | 97 ++++ addons/models/0020-fs_model.json | 191 +++++++ addons/models/0030-hive_model.json | 498 +++++++++++++++++++ addons/models/0040-sqoop_model.json | 97 ++++ addons/models/0050-falcon_model.json | 147 ++++++ addons/models/0060-hbase_model.json | 24 + addons/models/0070-kafka_model.json | 57 +++ addons/models/0080-storm_model.json | 150 ++++++ .../patches/001-hive_column_add_position.json | 21 + addons/sqoop-bridge/pom.xml | 18 - .../org/apache/atlas/sqoop/hook/SqoopHook.java | 46 +- .../sqoop/model/SqoopDataModelGenerator.java | 187 ------- .../apache/atlas/sqoop/hook/SqoopHookIT.java | 26 - addons/storm-bridge/pom.xml | 18 - .../apache/atlas/storm/hook/StormAtlasHook.java | 6 +- .../storm/model/StormDataModelGenerator.java | 41 -- .../atlas/storm/model/StormDataModel.scala | 104 ---- .../atlas/storm/hook/StormAtlasHookIT.java | 56 --- .../src/main/assemblies/standalone-package.xml | 30 +- .../java/org/apache/atlas/AtlasErrorCode.java | 4 +- .../atlas/model/typedef/AtlasStructDef.java | 26 +- .../atlas/model/typedef/AtlasTypesDef.java | 2 + .../org/apache/atlas/type/AtlasStructType.java | 14 +- .../apache/atlas/type/AtlasTypeRegistry.java | 8 + .../apache/atlas/RepositoryMetadataModule.java | 4 - .../graph/GraphBackedSearchIndexer.java | 4 +- .../bootstrap/AtlasTypeDefStoreInitializer.java | 364 ++++++++++++++ .../store/graph/AtlasTypeDefGraphStore.java | 13 + .../store/graph/v1/AtlasStructDefStoreV1.java | 35 +- .../atlas/services/AtlasPatchHandler.java | 173 ------- .../atlas/services/AtlasTypeAttributePatch.java | 296 ----------- .../apache/atlas/services/AtlasTypePatch.java | 104 ---- .../atlas/services/DefaultMetadataService.java | 82 +-- .../services/IBootstrapTypesRegistrar.java | 27 - .../atlas/services/ReservedTypesRegistrar.java | 135 ----- .../ReservedTypesRegistrationException.java | 26 - .../org/apache/atlas/BaseRepositoryTest.java | 42 ++ .../test/java/org/apache/atlas/TestUtils.java | 2 - .../store/graph/AtlasTypeDefGraphStoreTest.java | 4 +- .../DefaultMetadataServiceMockTest.java | 151 ------ .../services/ReservedTypesRegistrarTest.java | 103 ---- 57 files changed, 1917 insertions(+), 2616 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/falcon-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/pom.xml b/addons/falcon-bridge/pom.xml index de3e810..96743e7 100644 --- a/addons/falcon-bridge/pom.xml +++ b/addons/falcon-bridge/pom.xml @@ -355,24 +355,6 @@ 1.2.1 false - - - org.apache.atlas.falcon.model.FalconDataModelGenerator - - - atlas.conf - ${project.build.directory}/../../../typesystem/target/test-classes - - - - ${project.build.directory}/models/falcon_model.json - - - package - - java - - http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java index 05d072f..d1a7e87 100644 --- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java +++ b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java @@ -21,11 +21,8 @@ package org.apache.atlas.falcon.bridge; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasConstants; import org.apache.atlas.falcon.Util.EventUtil; -import org.apache.atlas.falcon.model.FalconDataModelGenerator; import org.apache.atlas.falcon.model.FalconDataTypes; -import org.apache.atlas.fs.model.FSDataTypes; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; -import org.apache.atlas.hive.model.HiveDataModelGenerator; import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.typesystem.Referenceable; import org.apache.commons.collections.CollectionUtils; @@ -61,6 +58,15 @@ import java.util.Map; public class FalconBridge { private static final Logger LOG = LoggerFactory.getLogger(FalconBridge.class); + public static final String COLO = "colo"; + public static final String TAGS = "tags"; + public static final String GROUPS = "groups"; + public static final String PIPELINES = "pipelines"; + public static final String WFPROPERTIES = "workflow-properties"; + public static final String RUNSON = "runs-on"; + public static final String STOREDIN = "stored-in"; + public static final String FREQUENCY = "frequency"; + /** * Creates cluster entity * @@ -77,14 +83,14 @@ public class FalconBridge { clusterRef.set(AtlasClient.DESCRIPTION, cluster.getDescription()); clusterRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, cluster.getName()); - clusterRef.set(FalconDataModelGenerator.COLO, cluster.getColo()); + clusterRef.set(FalconBridge.COLO, cluster.getColo()); if (cluster.getACL() != null) { clusterRef.set(AtlasClient.OWNER, cluster.getACL().getGroup()); } if (StringUtils.isNotEmpty(cluster.getTags())) { - clusterRef.set(FalconDataModelGenerator.TAGS, + clusterRef.set(FalconBridge.TAGS, EventUtil.convertKeyValueStringToMap(cluster.getTags())); } @@ -100,19 +106,19 @@ public class FalconBridge { String feedQualifiedName = getFeedQualifiedName(feed.getName(), (String) clusterReferenceable.get(AtlasClient.NAME)); feedEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, feedQualifiedName); - feedEntity.set(FalconDataModelGenerator.FREQUENCY, feed.getFrequency().toString()); - feedEntity.set(FalconDataModelGenerator.STOREDIN, clusterReferenceable); + feedEntity.set(FalconBridge.FREQUENCY, feed.getFrequency().toString()); + feedEntity.set(FalconBridge.STOREDIN, clusterReferenceable); if (feed.getACL() != null) { feedEntity.set(AtlasClient.OWNER, feed.getACL().getOwner()); } if (StringUtils.isNotEmpty(feed.getTags())) { - feedEntity.set(FalconDataModelGenerator.TAGS, + feedEntity.set(FalconBridge.TAGS, EventUtil.convertKeyValueStringToMap(feed.getTags())); } if (feed.getGroups() != null) { - feedEntity.set(FalconDataModelGenerator.GROUPS, feed.getGroups()); + feedEntity.set(FalconBridge.GROUPS, feed.getGroups()); } return feedEntity; @@ -165,7 +171,7 @@ public class FalconBridge { feedCreateEntity.set(AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS, outputs); } - feedCreateEntity.set(FalconDataModelGenerator.STOREDIN, clusterReferenceable); + feedCreateEntity.set(FalconBridge.STOREDIN, clusterReferenceable); entities.add(feedCreateEntity); } @@ -244,7 +250,7 @@ public class FalconBridge { processEntity.set(AtlasClient.NAME, process.getName()); processEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, getProcessQualifiedName(process.getName(), cluster.getName())); - processEntity.set(FalconDataModelGenerator.FREQUENCY, process.getFrequency().toString()); + processEntity.set(FalconBridge.FREQUENCY, process.getFrequency().toString()); if (!inputs.isEmpty()) { processEntity.set(AtlasClient.PROCESS_ATTRIBUTE_INPUTS, inputs); @@ -254,7 +260,7 @@ public class FalconBridge { } // set cluster - processEntity.set(FalconDataModelGenerator.RUNSON, clusterReferenceable); + processEntity.set(FalconBridge.RUNSON, clusterReferenceable); // Set user if (process.getACL() != null) { @@ -262,15 +268,15 @@ public class FalconBridge { } if (StringUtils.isNotEmpty(process.getTags())) { - processEntity.set(FalconDataModelGenerator.TAGS, + processEntity.set(FalconBridge.TAGS, EventUtil.convertKeyValueStringToMap(process.getTags())); } if (process.getPipelines() != null) { - processEntity.set(FalconDataModelGenerator.PIPELINES, process.getPipelines()); + processEntity.set(FalconBridge.PIPELINES, process.getPipelines()); } - processEntity.set(FalconDataModelGenerator.WFPROPERTIES, + processEntity.set(FalconBridge.WFPROPERTIES, getProcessEntityWFProperties(process.getWorkflow(), process.getName())); @@ -319,7 +325,7 @@ public class FalconBridge { private static List fillHDFSDataSet(final String pathUri, final String clusterName) { List entities = new ArrayList<>(); - Referenceable ref = new Referenceable(FSDataTypes.HDFS_PATH().toString()); + Referenceable ref = new Referenceable(HiveMetaStoreBridge.HDFS_PATH); ref.set("path", pathUri); // Path path = new Path(pathUri); // ref.set("name", path.getName()); @@ -352,7 +358,7 @@ public class FalconBridge { tableRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, HiveMetaStoreBridge.getTableQualifiedName(clusterName, dbName, tableName)); tableRef.set(AtlasClient.NAME, tableName.toLowerCase()); - tableRef.set(HiveDataModelGenerator.DB, dbRef); + tableRef.set(HiveMetaStoreBridge.DB, dbRef); entities.add(tableRef); return entities; @@ -364,7 +370,7 @@ public class FalconBridge { Referenceable clusterRef = new Referenceable(FalconDataTypes.FALCON_CLUSTER.getName()); clusterRef.set(AtlasClient.NAME, String.format("%s", clusterName)); clusterRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, clusterName); - clusterRef.set(FalconDataModelGenerator.COLO, colo); + clusterRef.set(FalconBridge.COLO, colo); return clusterRef; } @@ -375,8 +381,8 @@ public class FalconBridge { feedDatasetRef.set(AtlasClient.NAME, feed.getName()); feedDatasetRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, getFeedQualifiedName(feed.getName(), (String) clusterReference.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME))); - feedDatasetRef.set(FalconDataModelGenerator.STOREDIN, clusterReference); - feedDatasetRef.set(FalconDataModelGenerator.FREQUENCY, feed.getFrequency()); + feedDatasetRef.set(FalconBridge.STOREDIN, clusterReference); + feedDatasetRef.set(FalconBridge.FREQUENCY, feed.getFrequency()); return feedDatasetRef; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java deleted file mode 100644 index fc0a9b2..0000000 --- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java +++ /dev/null @@ -1,214 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.falcon.model; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import org.apache.atlas.AtlasClient; -import org.apache.atlas.AtlasException; -import org.apache.atlas.addons.ModelDefinitionDump; -import org.apache.atlas.typesystem.TypesDef; -import org.apache.atlas.typesystem.json.TypesSerialization; -import org.apache.atlas.typesystem.types.AttributeDefinition; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.EnumType; -import org.apache.atlas.typesystem.types.EnumTypeDefinition; -import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.StructType; -import org.apache.atlas.typesystem.types.StructTypeDefinition; -import org.apache.atlas.typesystem.types.TraitType; -import org.apache.atlas.typesystem.types.utils.TypesUtil; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -/** - * Utility that generates falcon data model. - */ -public class FalconDataModelGenerator { - - private static final Logger LOG = LoggerFactory.getLogger(FalconDataModelGenerator.class); - public static final String FREQUENCY = "frequency"; - - private final Map> classTypeDefinitions; - - public static final String COLO = "colo"; - public static final String TAGS = "tags"; - public static final String GROUPS = "groups"; - public static final String PIPELINES = "pipelines"; - public static final String WFPROPERTIES = "workflow-properties"; - public static final String RUNSON = "runs-on"; - public static final String STOREDIN = "stored-in"; - - public FalconDataModelGenerator() { - classTypeDefinitions = new HashMap<>(); - } - - public void createDataModel() throws AtlasException { - LOG.info("Generating the Falcon Data Model"); - - // classes - createClusterEntityClass(); - createProcessEntityClass(); - createFeedCreationEntityClass(); - createFeedEntityClass(); - createReplicationFeedEntityClass(); - } - - private TypesDef getTypesDef() { - return TypesUtil.getTypesDef(ImmutableList.of(), ImmutableList.of(), - getTraitTypeDefinitions(), getClassTypeDefinitions()); - } - - public String getDataModelAsJSON() { - return TypesSerialization.toJson(getTypesDef()); - } - - private ImmutableList> getClassTypeDefinitions() { - return ImmutableList.copyOf(classTypeDefinitions.values()); - } - - private ImmutableList> getTraitTypeDefinitions() { - return ImmutableList.of(); - } - - private void createClusterEntityClass() throws AtlasException { - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition(COLO, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, - false, true, null), - // map of tags - new AttributeDefinition(TAGS, - DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()), - Multiplicity.OPTIONAL, false, null),}; - - HierarchicalTypeDefinition definition = - new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_CLUSTER.getName(), null, - ImmutableSet.of(AtlasClient.INFRASTRUCTURE_SUPER_TYPE), attributeDefinitions); - classTypeDefinitions.put(FalconDataTypes.FALCON_CLUSTER.getName(), definition); - LOG.debug("Created definition for {}", FalconDataTypes.FALCON_CLUSTER.getName()); - } - - private void createFeedCreationEntityClass() throws AtlasException { - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED, - false, false, true, null) - }; - - HierarchicalTypeDefinition definition = - new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_FEED_CREATION.getName(), null, - ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions); - classTypeDefinitions.put(FalconDataTypes.FALCON_FEED_CREATION.getName(), definition); - LOG.debug("Created definition for {}", FalconDataTypes.FALCON_FEED_CREATION.getName()); - } - - private void createFeedEntityClass() throws AtlasException { - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE), - new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED, - false, false, true, null), - new AttributeDefinition(GROUPS, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), - // map of tags - new AttributeDefinition(TAGS, - DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()), - Multiplicity.OPTIONAL, false, null) - }; - - HierarchicalTypeDefinition definition = - new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_FEED.getName(), null, - ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions); - classTypeDefinitions.put(FalconDataTypes.FALCON_FEED.getName(), definition); - LOG.debug("Created definition for {}", FalconDataTypes.FALCON_FEED.getName()); - } - - - private void createReplicationFeedEntityClass() throws AtlasException { - HierarchicalTypeDefinition definition = - new HierarchicalTypeDefinition<>(ClassType.class, - FalconDataTypes.FALCON_FEED_REPLICATION.getName(), null, - ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), null); - classTypeDefinitions.put(FalconDataTypes.FALCON_FEED_REPLICATION.getName(), definition); - LOG.debug("Created definition for {}", FalconDataTypes.FALCON_FEED_REPLICATION.getName()); - } - - private void createProcessEntityClass() throws AtlasException { - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE), - new AttributeDefinition(RUNSON, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED, - false, false, true, null), - // map of tags - new AttributeDefinition(TAGS, - DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()), - Multiplicity.OPTIONAL, false, null), - new AttributeDefinition(PIPELINES, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null), - // wf properties - new AttributeDefinition(WFPROPERTIES, - DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()), - Multiplicity.OPTIONAL, false, null),}; - - HierarchicalTypeDefinition definition = - new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_PROCESS.getName(), null, - ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions); - classTypeDefinitions.put(FalconDataTypes.FALCON_PROCESS.getName(), definition); - LOG.debug("Created definition for {}", FalconDataTypes.FALCON_PROCESS.getName()); - } - - - public String getModelAsJson() throws AtlasException { - createDataModel(); - return getDataModelAsJSON(); - } - - public static void main(String[] args) throws Exception { - FalconDataModelGenerator falconDataModelGenerator = new FalconDataModelGenerator(); - String modelAsJson = falconDataModelGenerator.getModelAsJson(); - - if (args.length == 1) { - ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson); - return; - } - - System.out.println("falconDataModelAsJSON = " + modelAsJson); - - TypesDef typesDef = falconDataModelGenerator.getTypesDef(); - for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) { - System.out.println(String.format("%s(%s) - values %s", enumType.name, EnumType.class.getSimpleName(), - Arrays.toString(enumType.enumValues))); - } - for (StructTypeDefinition structType : typesDef.structTypesAsJavaList()) { - System.out.println( - String.format("%s(%s) - attributes %s", structType.typeName, StructType.class.getSimpleName(), - Arrays.toString(structType.attributeDefinitions))); - } - for (HierarchicalTypeDefinition classType : typesDef.classTypesAsJavaList()) { - System.out.println(String.format("%s(%s) - super types [%s] - attributes %s", classType.typeName, - ClassType.class.getSimpleName(), - StringUtils.join(classType.superTypes, ","), Arrays.toString(classType.attributeDefinitions))); - } - for (HierarchicalTypeDefinition traitType : typesDef.traitTypesAsJavaList()) { - System.out.println(String.format("%s(%s) - %s", traitType.typeName, TraitType.class.getSimpleName(), - Arrays.toString(traitType.attributeDefinitions))); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java index 31b70fd..8d0a47a 100644 --- a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java +++ b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java @@ -23,9 +23,7 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasServiceException; import org.apache.atlas.falcon.bridge.FalconBridge; -import org.apache.atlas.falcon.model.FalconDataModelGenerator; import org.apache.atlas.falcon.model.FalconDataTypes; -import org.apache.atlas.fs.model.FSDataTypes; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.typesystem.Referenceable; @@ -47,7 +45,6 @@ import org.apache.falcon.entity.v0.feed.Location; import org.apache.falcon.entity.v0.feed.LocationType; import org.apache.falcon.entity.v0.process.Process; import org.apache.falcon.security.CurrentUser; -import org.apache.hadoop.hive.conf.HiveConf; import org.slf4j.Logger; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -84,24 +81,9 @@ public class FalconHookIT { AtlasService service = new AtlasService(); service.init(); STORE.registerListener(service); - registerFalconDataModel(); CurrentUser.authenticate(System.getProperty("user.name")); } - private void registerFalconDataModel() throws Exception { - if (isDataModelAlreadyRegistered()) { - LOG.info("Falcon data model is already registered!"); - return; - } - - HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(ApplicationProperties.get(), new HiveConf(), atlasClient); - hiveMetaStoreBridge.registerHiveDataModel(); - - FalconDataModelGenerator dataModelGenerator = new FalconDataModelGenerator(); - LOG.info("Registering Falcon data model"); - atlasClient.createType(dataModelGenerator.getModelAsJson()); - } - private boolean isDataModelAlreadyRegistered() throws Exception { try { atlasClient.getType(FalconDataTypes.FALCON_PROCESS.getName()); @@ -196,7 +178,7 @@ public class FalconHookIT { String inputId = ((List) processEntity.get("inputs")).get(0).getId()._getId(); Referenceable pathEntity = atlasClient.getEntity(inputId); - assertEquals(pathEntity.getTypeName(), FSDataTypes.HDFS_PATH().toString()); + assertEquals(pathEntity.getTypeName(), HiveMetaStoreBridge.HDFS_PATH.toString()); List locations = FeedHelper.getLocations(feedCluster, feed); Location dataLocation = FileSystemStorage.getLocation(locations, LocationType.DATA); @@ -243,7 +225,7 @@ public class FalconHookIT { private void assertFeedAttributes(String feedId) throws Exception { Referenceable feedEntity = atlasClient.getEntity(feedId); assertEquals(feedEntity.get(AtlasClient.OWNER), "testuser"); - assertEquals(feedEntity.get(FalconDataModelGenerator.FREQUENCY), "hours(1)"); + assertEquals(feedEntity.get(FalconBridge.FREQUENCY), "hours(1)"); assertEquals(feedEntity.get(AtlasClient.DESCRIPTION), "test input"); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hdfs-model/pom.xml ---------------------------------------------------------------------- diff --git a/addons/hdfs-model/pom.xml b/addons/hdfs-model/pom.xml index e2ab029..1376fdc 100644 --- a/addons/hdfs-model/pom.xml +++ b/addons/hdfs-model/pom.xml @@ -180,24 +180,6 @@ 1.2.1 false - - - org.apache.atlas.fs.model.FSDataModelGenerator - - - atlas.conf - ${project.build.directory}/../../../typesystem/src/test/resources/ - - - - ${project.build.directory}/models/fs_model.json - - - package - - java - - http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hdfs-model/src/main/java/org/apache/atlas/fs/model/FSDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/hdfs-model/src/main/java/org/apache/atlas/fs/model/FSDataModelGenerator.java b/addons/hdfs-model/src/main/java/org/apache/atlas/fs/model/FSDataModelGenerator.java deleted file mode 100644 index 555d565..0000000 --- a/addons/hdfs-model/src/main/java/org/apache/atlas/fs/model/FSDataModelGenerator.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.fs.model; - -import org.apache.atlas.addons.ModelDefinitionDump; -import org.apache.atlas.typesystem.TypesDef; -import org.apache.atlas.typesystem.json.TypesSerialization; - -import java.io.IOException; - -public class FSDataModelGenerator { - - public static void main(String[] args) throws IOException { - FSDataModel.main(args); - TypesDef typesDef = FSDataModel.typesDef(); - String fsTypesAsJSON = TypesSerialization.toJson(typesDef); - if (args.length == 1) { - ModelDefinitionDump.dumpModelToFile(args[0], fsTypesAsJSON); - return; - } - System.out.println("FS Data Model as JSON = " + fsTypesAsJSON); - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala ---------------------------------------------------------------------- diff --git a/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala b/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala deleted file mode 100644 index ad40340..0000000 --- a/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.fs.model - -import org.apache.atlas.{AtlasConstants, AtlasClient} -import org.apache.atlas.typesystem.TypesDef -import org.apache.atlas.typesystem.builders.TypesBuilder -import org.apache.atlas.typesystem.json.TypesSerialization -import org.apache.atlas.typesystem.types.DataTypes.MapType -import org.apache.hadoop.fs.permission.FsAction - -import scala.tools.scalap.scalax.rules.scalasig.ClassFileParser.EnumConstValue - -/** - * This represents the data model for a HDFS Path - */ -object FSDataModel extends App { - - val typesBuilder = new TypesBuilder - import typesBuilder._ - - val typesDef : TypesDef = types { - - // FS DataSet - _class(FSDataTypes.FS_PATH.toString, List(AtlasClient.DATA_SET_SUPER_TYPE)) { - //fully qualified path/URI to the filesystem path is stored in 'qualifiedName' and 'path'. - "path" ~ (string, required, indexed) - "createTime" ~ (date, optional, indexed) - "modifiedTime" ~ (date, optional, indexed) - //Is a regular file or a directory. If true, it is a file else a directory - "isFile" ~ (boolean, optional, indexed) - //Is a symlink or not - "isSymlink" ~ (boolean, optional) - //Optional and may not be set for a directory - "fileSize" ~ (long, optional) - "group" ~ (string, optional, indexed) - "posixPermissions" ~ (FSDataTypes.FS_PERMISSIONS.toString, optional, indexed) - } - - enum(FSDataTypes.FS_ACTION.toString, FsAction.values().map(x => x.name()) : _*) - - struct(FSDataTypes.FS_PERMISSIONS.toString) { - PosixPermissions.PERM_USER.toString ~ (FSDataTypes.FS_ACTION.toString, required, indexed) - PosixPermissions.PERM_GROUP.toString ~ (FSDataTypes.FS_ACTION.toString, required, indexed) - PosixPermissions.PERM_OTHER.toString ~ (FSDataTypes.FS_ACTION.toString, required, indexed) - PosixPermissions.STICKY_BIT.toString ~ (boolean, required, indexed) - } - - //HDFS DataSet - _class(FSDataTypes.HDFS_PATH.toString, List(FSDataTypes.FS_PATH.toString)) { - //Making cluster optional since path is already unique containing the namenode URI - AtlasConstants.CLUSTER_NAME_ATTRIBUTE ~ (string, optional, indexed) - "numberOfReplicas" ~ (int, optional, indexed) - "extendedAttributes" ~ (map(string, string), optional) - } - //TODO - ACLs - https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html#ACLs_Access_Control_Lists - } - - // add the types to atlas - val typesAsJSON = TypesSerialization.toJson(typesDef) - println("FS Data Model as JSON: ") - println(typesAsJSON) - -} - -object FSDataTypes extends Enumeration { - type FSDataTypes = Value - val FS_ACTION = Value("file_action") - val FS_PATH = Value("fs_path") - val HDFS_PATH = Value("hdfs_path") - val FS_PERMISSIONS = Value("fs_permissions") -} - -object PosixPermissions extends Enumeration { - type PosixPermissions = Value - val PERM_USER = Value("user") - val PERM_GROUP = Value("group") - val PERM_OTHER = Value("others") - val STICKY_BIT = Value("sticky") -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java ---------------------------------------------------------------------- diff --git a/addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java b/addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java deleted file mode 100644 index 5944da8..0000000 --- a/addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.fs.model; - -import javax.inject.Inject; - -import org.apache.atlas.RepositoryMetadataModule; -import org.apache.atlas.repository.graph.AtlasGraphProvider; -import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.services.MetadataService; -import org.apache.atlas.typesystem.TypesDef; -import org.apache.atlas.typesystem.json.TypesSerialization; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Guice; -import org.testng.annotations.Test; - -import scala.Enumeration; -import scala.collection.Iterator; - -@Test -@Guice(modules = RepositoryMetadataModule.class) -public class HDFSModelTest { - - public static final Logger LOG = LoggerFactory.getLogger(HDFSModelTest.class); - private static final String ATLAS_URL = "http://localhost:21000/"; - - @Inject - private MetadataService metadataService; - - @BeforeClass - public void setUp() throws Exception { - } - - @AfterClass - public void tearDown() throws Exception { - TypeSystem.getInstance().reset(); - AtlasGraphProvider.cleanup(); - } - - @Test - public void testCreateDataModel() throws Exception { - FSDataModel.main(new String[]{}); - TypesDef fsTypesDef = FSDataModel.typesDef(); - - String fsTypesAsJSON = TypesSerialization.toJson(fsTypesDef); - LOG.info("fsTypesAsJSON = {}", fsTypesAsJSON); - - metadataService.createType(fsTypesAsJSON); - - // verify types are registered - final Iterator valueIterator = FSDataTypes.values().iterator(); - while (valueIterator.hasNext() ) { - final Enumeration.Value typeEnum = valueIterator.next(); - String typeDefStr = metadataService.getTypeDefinition(typeEnum.toString()); - Assert.assertNotNull(typeDefStr); - - TypesDef typesDef = TypesSerialization.fromJson(typeDefStr); - Assert.assertNotNull(typesDef); - } - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/pom.xml b/addons/hive-bridge/pom.xml index 884791d..7ab47f1 100755 --- a/addons/hive-bridge/pom.xml +++ b/addons/hive-bridge/pom.xml @@ -399,24 +399,6 @@ 1.2.1 false - - - org.apache.atlas.hive.model.HiveDataModelGenerator - - - atlas.conf - ${project.build.directory}/../../../typesystem/target/test-classes - - - - ${project.build.directory}/models/hive_model.json - - - package - - java - - http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java index e4a20e1..c54fdb3 100644 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java @@ -19,8 +19,6 @@ package org.apache.atlas.hive.bridge; import org.apache.atlas.AtlasClient; -import org.apache.atlas.hive.hook.HiveHook; -import org.apache.atlas.hive.model.HiveDataModelGenerator; import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.typesystem.Referenceable; import org.apache.hadoop.hive.ql.hooks.LineageInfo; @@ -31,7 +29,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; public class ColumnLineageUtils { public static final Logger LOG = LoggerFactory.getLogger(ColumnLineageUtils.class); @@ -92,7 +89,7 @@ public class ColumnLineageUtils { if (r.getTypeName().equals(HiveDataTypes.HIVE_TABLE.getName())) { String qName = (String) r.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME); String[] qNameComps = extractComponents(qName); - for (Referenceable col : (List) r.get(HiveDataModelGenerator.COLUMNS)) { + for (Referenceable col : (List) r.get(HiveMetaStoreBridge.COLUMNS)) { String cName = (String) col.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME); String[] colQNameComps = extractComponents(cName); String colQName = colQNameComps[0] + "." + colQNameComps[1] + "." + colQNameComps[2]; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java index 2d2039b..cbc51cc 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java @@ -26,15 +26,11 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasConstants; import org.apache.atlas.AtlasServiceException; -import org.apache.atlas.fs.model.FSDataModel; -import org.apache.atlas.fs.model.FSDataTypes; import org.apache.atlas.hive.hook.HiveHook; -import org.apache.atlas.hive.model.HiveDataModelGenerator; import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Struct; import org.apache.atlas.typesystem.json.InstanceSerialization; -import org.apache.atlas.typesystem.json.TypesSerialization; import org.apache.atlas.typesystem.persistence.Id; import org.apache.atlas.utils.AuthenticationUtil; import org.apache.commons.cli.BasicParser; @@ -79,6 +75,25 @@ public class HiveMetaStoreBridge { public static final String ATLAS_ENDPOINT = "atlas.rest.address"; + public static final String COMMENT = "comment"; + public static final String PARAMETERS = "parameters"; + public static final String COLUMNS = "columns"; + public static final String POSITION = "position"; + public static final String PART_COLS = "partitionKeys"; + public static final String TABLE_ALIAS_LIST = "aliases"; + public static final String STORAGE_NUM_BUCKETS = "numBuckets"; + public static final String STORAGE_IS_STORED_AS_SUB_DIRS = "storedAsSubDirectories"; + public static final String TABLE = "table"; + public static final String DB = "db"; + public static final String STORAGE_DESC = "sd"; + public static final String STORAGE_DESC_INPUT_FMT = "inputFormat"; + public static final String STORAGE_DESC_OUTPUT_FMT = "outputFormat"; + public static final String LOCATION = "location"; + public static final String TABLE_TYPE_ATTR = "tableType"; + public static final String CREATE_TIME = "createTime"; + public static final String LAST_ACCESS_TIME = "lastAccessTime"; + public static final String HDFS_PATH = "hdfs_path"; + private static final Logger LOG = LoggerFactory.getLogger(HiveMetaStoreBridge.class); public final Hive hiveClient; @@ -174,8 +189,8 @@ public class HiveMetaStoreBridge { dbRef.set(AtlasClient.NAME, dbName); dbRef.set(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, clusterName); dbRef.set(DESCRIPTION_ATTR, hiveDB.getDescription()); - dbRef.set(HiveDataModelGenerator.LOCATION, hiveDB.getLocationUri()); - dbRef.set(HiveDataModelGenerator.PARAMETERS, hiveDB.getParameters()); + dbRef.set(LOCATION, hiveDB.getLocationUri()); + dbRef.set(PARAMETERS, hiveDB.getParameters()); dbRef.set(AtlasClient.OWNER, hiveDB.getOwnerName()); if (hiveDB.getOwnerType() != null) { dbRef.set("ownerType", hiveDB.getOwnerType().getValue()); @@ -431,7 +446,7 @@ public class HiveMetaStoreBridge { try { createDate = getTableCreatedTime(hiveTable); LOG.debug("Setting create time to {} ", createDate); - tableReference.set(HiveDataModelGenerator.CREATE_TIME, createDate); + tableReference.set(CREATE_TIME, createDate); } catch(Exception ne) { LOG.error("Error while setting createTime for the table {} ", hiveTable.getCompleteName(), ne); } @@ -441,19 +456,19 @@ public class HiveMetaStoreBridge { if ( hiveTable.getLastAccessTime() > 0) { lastAccessTime = new Date(hiveTable.getLastAccessTime() * MILLIS_CONVERT_FACTOR); } - tableReference.set(HiveDataModelGenerator.LAST_ACCESS_TIME, lastAccessTime); + tableReference.set(LAST_ACCESS_TIME, lastAccessTime); tableReference.set("retention", hiveTable.getRetention()); - tableReference.set(HiveDataModelGenerator.COMMENT, hiveTable.getParameters().get(HiveDataModelGenerator.COMMENT)); + tableReference.set(COMMENT, hiveTable.getParameters().get(COMMENT)); // add reference to the database - tableReference.set(HiveDataModelGenerator.DB, dbReference); + tableReference.set(DB, dbReference); // add reference to the StorageDescriptor Referenceable sdReferenceable = fillStorageDesc(hiveTable.getSd(), tableQualifiedName, getStorageDescQFName(tableQualifiedName), tableReference.getId()); - tableReference.set(HiveDataModelGenerator.STORAGE_DESC, sdReferenceable); + tableReference.set(STORAGE_DESC, sdReferenceable); - tableReference.set(HiveDataModelGenerator.PARAMETERS, hiveTable.getParameters()); + tableReference.set(PARAMETERS, hiveTable.getParameters()); if (hiveTable.getViewOriginalText() != null) { tableReference.set("viewOriginalText", hiveTable.getViewOriginalText()); @@ -463,14 +478,14 @@ public class HiveMetaStoreBridge { tableReference.set("viewExpandedText", hiveTable.getViewExpandedText()); } - tableReference.set(HiveDataModelGenerator.TABLE_TYPE_ATTR, hiveTable.getTableType().name()); + tableReference.set(TABLE_TYPE_ATTR, hiveTable.getTableType().name()); tableReference.set("temporary", hiveTable.isTemporary()); // add reference to the Partition Keys List partKeys = getColumns(hiveTable.getPartitionKeys(), tableReference); tableReference.set("partitionKeys", partKeys); - tableReference.set(HiveDataModelGenerator.COLUMNS, getColumns(hiveTable.getCols(), tableReference)); + tableReference.set(COLUMNS, getColumns(hiveTable.getCols(), tableReference)); return tableReference; } @@ -523,12 +538,12 @@ public class HiveMetaStoreBridge { serdeInfoStruct.set(AtlasClient.NAME, serdeInfo.getName()); serdeInfoStruct.set("serializationLib", serdeInfo.getSerializationLib()); - serdeInfoStruct.set(HiveDataModelGenerator.PARAMETERS, serdeInfo.getParameters()); + serdeInfoStruct.set(PARAMETERS, serdeInfo.getParameters()); sdReferenceable.set("serdeInfo", serdeInfoStruct); - sdReferenceable.set(HiveDataModelGenerator.STORAGE_NUM_BUCKETS, storageDesc.getNumBuckets()); + sdReferenceable.set(STORAGE_NUM_BUCKETS, storageDesc.getNumBuckets()); sdReferenceable - .set(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS, storageDesc.isStoredAsSubDirectories()); + .set(STORAGE_IS_STORED_AS_SUB_DIRS, storageDesc.isStoredAsSubDirectories()); List sortColsStruct = new ArrayList<>(); for (Order sortcol : storageDesc.getSortCols()) { @@ -543,7 +558,7 @@ public class HiveMetaStoreBridge { sdReferenceable.set("sortCols", sortColsStruct); } - sdReferenceable.set(HiveDataModelGenerator.LOCATION, storageDesc.getLocation()); + sdReferenceable.set(LOCATION, storageDesc.getLocation()); sdReferenceable.set("inputFormat", storageDesc.getInputFormat()); sdReferenceable.set("outputFormat", storageDesc.getOutputFormat()); sdReferenceable.set("compressed", storageDesc.isCompressed()); @@ -552,15 +567,15 @@ public class HiveMetaStoreBridge { sdReferenceable.set("bucketCols", storageDesc.getBucketCols()); } - sdReferenceable.set(HiveDataModelGenerator.PARAMETERS, storageDesc.getParameters()); + sdReferenceable.set(PARAMETERS, storageDesc.getParameters()); sdReferenceable.set("storedAsSubDirectories", storageDesc.isStoredAsSubDirectories()); - sdReferenceable.set(HiveDataModelGenerator.TABLE, tableId); + sdReferenceable.set(TABLE, tableId); return sdReferenceable; } public Referenceable fillHDFSDataSet(String pathUri) { - Referenceable ref = new Referenceable(FSDataTypes.HDFS_PATH().toString()); + Referenceable ref = new Referenceable(HDFS_PATH.toString()); ref.set("path", pathUri); Path path = new Path(pathUri); ref.set(AtlasClient.NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase()); @@ -586,9 +601,9 @@ public class HiveMetaStoreBridge { colReferenceable.set(AtlasClient.NAME, fs.getName()); colReferenceable.set(AtlasClient.OWNER, tableReference.get(AtlasClient.OWNER)); colReferenceable.set("type", fs.getType()); - colReferenceable.set(HiveDataModelGenerator.POSITION, columnPosition++); - colReferenceable.set(HiveDataModelGenerator.COMMENT, fs.getComment()); - colReferenceable.set(HiveDataModelGenerator.TABLE, tableReference.getId()); + colReferenceable.set(POSITION, columnPosition++); + colReferenceable.set(COMMENT, fs.getComment()); + colReferenceable.set(TABLE, tableReference.getId()); colList.add(colReferenceable); @@ -596,43 +611,6 @@ public class HiveMetaStoreBridge { return colList; } - /** - * Register the Hive DataModel in Atlas, if not already defined. - * - * The method checks for the presence of the type {@link HiveDataTypes#HIVE_PROCESS} with the Atlas server. - * If this type is defined, then we assume the Hive DataModel is registered. - * @throws Exception - */ - public synchronized void registerHiveDataModel() throws Exception { - HiveDataModelGenerator dataModelGenerator = new HiveDataModelGenerator(); - AtlasClient dgiClient = getAtlasClient(); - - try { - dgiClient.getType(FSDataTypes.HDFS_PATH().toString()); - LOG.info("HDFS data model is already registered!"); - } catch(AtlasServiceException ase) { - if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) { - //Trigger val definition - FSDataModel.main(null); - - final String hdfsModelJson = TypesSerialization.toJson(FSDataModel.typesDef()); - //Expected in case types do not exist - LOG.info("Registering HDFS data model : " + hdfsModelJson); - dgiClient.createType(hdfsModelJson); - } - } - - try { - dgiClient.getType(HiveDataTypes.HIVE_PROCESS.getName()); - LOG.info("Hive data model is already registered!"); - } catch(AtlasServiceException ase) { - if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) { - //Expected in case types do not exist - LOG.info("Registering Hive data model"); - dgiClient.createType(dataModelGenerator.getModelAsJson()); - } - } - } public static void main(String[] args) throws Exception { @@ -661,7 +639,6 @@ public class HiveMetaStoreBridge { } HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(atlasConf, new HiveConf(), atlasClient); - hiveMetaStoreBridge.registerHiveDataModel(); hiveMetaStoreBridge.importHiveMetadata(failOnError); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java index eaef337..0c6582b 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java @@ -25,7 +25,6 @@ import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasConstants; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.hive.bridge.ColumnLineageUtils; -import org.apache.atlas.hive.model.HiveDataModelGenerator; import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.hook.AtlasHook; import org.apache.atlas.notification.hook.HookNotification; @@ -411,10 +410,10 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { Referenceable tableEntity = tables.get(Type.TABLE); //Reset regular column QF Name to old Name and create a new partial notification request to replace old column QFName to newName to retain any existing traits - replaceColumnQFName(event, (List) tableEntity.get(HiveDataModelGenerator.COLUMNS), oldQualifiedName, newQualifiedName); + replaceColumnQFName(event, (List) tableEntity.get(HiveMetaStoreBridge.COLUMNS), oldQualifiedName, newQualifiedName); //Reset partition key column QF Name to old Name and create a new partial notification request to replace old column QFName to newName to retain any existing traits - replaceColumnQFName(event, (List) tableEntity.get(HiveDataModelGenerator.PART_COLS), oldQualifiedName, newQualifiedName); + replaceColumnQFName(event, (List) tableEntity.get(HiveMetaStoreBridge.PART_COLS), oldQualifiedName, newQualifiedName); //Reset SD QF Name to old Name and create a new partial notification request to replace old SD QFName to newName to retain any existing traits replaceSDQFName(event, tableEntity, oldQualifiedName, newQualifiedName); @@ -437,7 +436,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { ArrayList alias_list = new ArrayList<>(); alias_list.add(oldTable.getTableName().toLowerCase()); - newEntity.set(HiveDataModelGenerator.TABLE_ALIAS_LIST, alias_list); + newEntity.set(HiveMetaStoreBridge.TABLE_ALIAS_LIST, alias_list); event.addMessage(new HookNotification.EntityPartialUpdateRequest(event.getUser(), HiveDataTypes.HIVE_TABLE.getName(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, oldTableQFName, newEntity)); @@ -466,7 +465,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { private Referenceable replaceSDQFName(final HiveEventContext event, Referenceable tableEntity, final String oldTblQFName, final String newTblQFName) { //Reset storage desc QF Name to old Name - final Referenceable sdRef = ((Referenceable) tableEntity.get(HiveDataModelGenerator.STORAGE_DESC)); + final Referenceable sdRef = ((Referenceable) tableEntity.get(HiveMetaStoreBridge.STORAGE_DESC)); sdRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, HiveMetaStoreBridge.getStorageDescQFName(oldTblQFName)); //Replace SD QF name first to retain tags http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java deleted file mode 100755 index d0fc43f..0000000 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java +++ /dev/null @@ -1,350 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.hive.model; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import org.apache.atlas.AtlasClient; -import org.apache.atlas.AtlasConstants; -import org.apache.atlas.AtlasException; -import org.apache.atlas.addons.ModelDefinitionDump; -import org.apache.atlas.typesystem.TypesDef; -import org.apache.atlas.typesystem.json.TypesSerialization; -import org.apache.atlas.typesystem.types.AttributeDefinition; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.EnumType; -import org.apache.atlas.typesystem.types.EnumTypeDefinition; -import org.apache.atlas.typesystem.types.EnumValue; -import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.StructType; -import org.apache.atlas.typesystem.types.StructTypeDefinition; -import org.apache.atlas.typesystem.types.TraitType; -import org.apache.atlas.typesystem.types.utils.TypesUtil; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -/** - * Utility that generates hive data model for both metastore entities and DDL/DML queries. - */ -public class HiveDataModelGenerator { - - private static final Logger LOG = LoggerFactory.getLogger(HiveDataModelGenerator.class); - - private static final DataTypes.MapType STRING_MAP_TYPE = - new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE); - - private final Map> classTypeDefinitions; - private final Map enumTypeDefinitionMap; - private final Map structTypeDefinitionMap; - - public static final String COMMENT = "comment"; - public static final String PARAMETERS = "parameters"; - public static final String COLUMNS = "columns"; - public static final String POSITION = "position"; - public static final String PART_COLS = "partitionKeys"; - public static final String TABLE_ALIAS_LIST = "aliases"; - - public static final String STORAGE_NUM_BUCKETS = "numBuckets"; - public static final String STORAGE_IS_STORED_AS_SUB_DIRS = "storedAsSubDirectories"; - - public static final String TABLE = "table"; - public static final String DB = "db"; - - public static final String STORAGE_DESC = "sd"; - public static final String STORAGE_DESC_INPUT_FMT = "inputFormat"; - public static final String STORAGE_DESC_OUTPUT_FMT = "outputFormat"; - public static final String LOCATION = "location"; - - public static final String TABLE_TYPE_ATTR = "tableType"; - - public static final String CREATE_TIME = "createTime"; - public static final String LAST_ACCESS_TIME = "lastAccessTime"; - - public HiveDataModelGenerator() { - classTypeDefinitions = new HashMap<>(); - enumTypeDefinitionMap = new HashMap<>(); - structTypeDefinitionMap = new HashMap<>(); - } - - public void createDataModel() throws AtlasException { - LOG.info("Generating the Hive Data Model...."); - - // enums - createHivePrincipalTypeEnum(); - // structs - createSerDeStruct(); - createOrderStruct(); - createStorageDescClass(); - - // classes - createDBClass(); - createColumnClass(); - createTableClass(); - - // DDL/DML Process - createProcessClass(); - createColumnLineageClass(); - } - - public TypesDef getTypesDef() { - return TypesUtil.getTypesDef(getEnumTypeDefinitions(), getStructTypeDefinitions(), getTraitTypeDefinitions(), - getClassTypeDefinitions()); - } - - public String getDataModelAsJSON() { - return TypesSerialization.toJson(getTypesDef()); - } - - public ImmutableList getEnumTypeDefinitions() { - return ImmutableList.copyOf(enumTypeDefinitionMap.values()); - } - - public ImmutableList getStructTypeDefinitions() { - return ImmutableList.copyOf(structTypeDefinitionMap.values()); - } - - public ImmutableList> getClassTypeDefinitions() { - return ImmutableList.copyOf(classTypeDefinitions.values()); - } - - public ImmutableList> getTraitTypeDefinitions() { - return ImmutableList.of(); - } - - private void createHivePrincipalTypeEnum() throws AtlasException { - EnumValue values[] = {new EnumValue("USER", 1), new EnumValue("ROLE", 2), new EnumValue("GROUP", 3),}; - - EnumTypeDefinition definition = new EnumTypeDefinition(HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), values); - - enumTypeDefinitionMap.put(HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), definition); - LOG.debug("Created definition for " + HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName()); - } - - private void createSerDeStruct() throws AtlasException { - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition(AtlasClient.NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), - new AttributeDefinition("serializationLib", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, - false, null), - new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),}; - StructTypeDefinition definition = - new StructTypeDefinition(HiveDataTypes.HIVE_SERDE.getName(), attributeDefinitions); - structTypeDefinitionMap.put(HiveDataTypes.HIVE_SERDE.getName(), definition); - LOG.debug("Created definition for " + HiveDataTypes.HIVE_SERDE.getName()); - } - - private void createOrderStruct() throws AtlasException { - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("col", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition("order", DataTypes.INT_TYPE.getName(), Multiplicity.REQUIRED, false, null),}; - - StructTypeDefinition definition = - new StructTypeDefinition(HiveDataTypes.HIVE_ORDER.getName(), attributeDefinitions); - structTypeDefinitionMap.put(HiveDataTypes.HIVE_ORDER.getName(), definition); - LOG.debug("Created definition for " + HiveDataTypes.HIVE_ORDER.getName()); - } - - private void createStorageDescClass() throws AtlasException { - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - //Optional to keep it backward-compatible - new AttributeDefinition(TABLE, HiveDataTypes.HIVE_TABLE.getName(), Multiplicity.OPTIONAL, false, - STORAGE_DESC), - new AttributeDefinition(LOCATION, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, - null), - new AttributeDefinition("inputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, - null), - new AttributeDefinition("outputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, - null), - new AttributeDefinition("compressed", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.REQUIRED, false, - null), - new AttributeDefinition(STORAGE_NUM_BUCKETS, DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, - null), - new AttributeDefinition("serdeInfo", HiveDataTypes.HIVE_SERDE.getName(), Multiplicity.OPTIONAL, true, - null), - new AttributeDefinition("bucketCols", String.format("array<%s>", DataTypes.STRING_TYPE.getName()), - Multiplicity.OPTIONAL, false, null), - new AttributeDefinition("sortCols", String.format("array<%s>", HiveDataTypes.HIVE_ORDER.getName()), - Multiplicity.OPTIONAL, false, null), - new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null), - //new AttributeDefinition("skewedInfo", DefinedTypes.HIVE_SKEWEDINFO.getName(), - // Multiplicity.OPTIONAL, false, null), - new AttributeDefinition(STORAGE_IS_STORED_AS_SUB_DIRS, DataTypes.BOOLEAN_TYPE.getName(), - Multiplicity.OPTIONAL, false, null),}; - - HierarchicalTypeDefinition definition = - new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_STORAGEDESC.getName(), null, - ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions); - classTypeDefinitions.put(HiveDataTypes.HIVE_STORAGEDESC.getName(), definition); - LOG.debug("Created definition for " + HiveDataTypes.HIVE_STORAGEDESC.getName()); - } - - /** Revisit later after nested array types are handled by the typesystem **/ - - private void createDBClass() throws AtlasException { - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, - false, true, null), - new AttributeDefinition(LOCATION, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, - null), - new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null), - new AttributeDefinition("ownerType", HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), Multiplicity.OPTIONAL, - false, null),}; - - HierarchicalTypeDefinition definition = - new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_DB.getName(), null, - ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE), attributeDefinitions); - classTypeDefinitions.put(HiveDataTypes.HIVE_DB.getName(), definition); - LOG.debug("Created definition for " + HiveDataTypes.HIVE_DB.getName()); - } - - private void createColumnClass() throws AtlasException { - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null), - new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), - //Making this optional since this is an incompatible change - //Reverse attribute to 'columns' in Table - new AttributeDefinition(TABLE, HiveDataTypes.HIVE_TABLE.getName(), Multiplicity.OPTIONAL, false, COLUMNS), - new AttributeDefinition(POSITION, DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, false, false, null)}; - - HierarchicalTypeDefinition definition = - new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_COLUMN.getName(), null, "1.1", - ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions); - classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN.getName(), definition); - LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN.getName()); - } - - private void createTableClass() throws AtlasException { - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition(DB, HiveDataTypes.HIVE_DB.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition(CREATE_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.OPTIONAL, false, - null), - new AttributeDefinition(LAST_ACCESS_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.OPTIONAL, false, - null), - new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), - new AttributeDefinition("retention", DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, null), - new AttributeDefinition(STORAGE_DESC, HiveDataTypes.HIVE_STORAGEDESC.getName(), Multiplicity.OPTIONAL, true, - null), - new AttributeDefinition(PART_COLS, DataTypes.arrayTypeName(HiveDataTypes.HIVE_COLUMN.getName()), - Multiplicity.OPTIONAL, true, null), - new AttributeDefinition(TABLE_ALIAS_LIST, DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()), - Multiplicity.OPTIONAL, true, null), - new AttributeDefinition("columns", DataTypes.arrayTypeName(HiveDataTypes.HIVE_COLUMN.getName()), - Multiplicity.OPTIONAL, true, null), - new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null), - new AttributeDefinition("viewOriginalText", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, - false, null), - new AttributeDefinition("viewExpandedText", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, - false, null), - new AttributeDefinition(HiveDataModelGenerator.TABLE_TYPE_ATTR, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, - null), - new AttributeDefinition("temporary", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, false, - false, true, null),}; - HierarchicalTypeDefinition definition = - new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.getName(), null, - ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions); - classTypeDefinitions.put(HiveDataTypes.HIVE_TABLE.getName(), definition); - LOG.debug("Created definition for " + HiveDataTypes.HIVE_TABLE.getName()); - } - - private void createProcessClass() throws AtlasException { - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("startTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition("endTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition("userName", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, - false, true, null), - new AttributeDefinition("operationType", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, - false, true, null), - new AttributeDefinition("queryText", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, - null), - new AttributeDefinition("queryPlan", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, - null), - new AttributeDefinition("queryId", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition("recentQueries", String.format("array<%s>", DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL, false, null), - new AttributeDefinition(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), - new AttributeDefinition("queryGraph", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, - null),}; - - HierarchicalTypeDefinition definition = - new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_PROCESS.getName(), null, - ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions); - classTypeDefinitions.put(HiveDataTypes.HIVE_PROCESS.getName(), definition); - LOG.debug("Created definition for " + HiveDataTypes.HIVE_PROCESS.getName()); - } - - public String getModelAsJson() throws AtlasException { - createDataModel(); - return getDataModelAsJSON(); - } - - public static void main(String[] args) throws Exception { - HiveDataModelGenerator hiveDataModelGenerator = new HiveDataModelGenerator(); - String modelAsJson = hiveDataModelGenerator.getModelAsJson(); - - if (args.length==1) { - ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson); - return; - } - - System.out.println("hiveDataModelAsJSON = " + modelAsJson); - - TypesDef typesDef = hiveDataModelGenerator.getTypesDef(); - for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) { - System.out.println(String.format("%s(%s) - values %s", enumType.name, EnumType.class.getSimpleName(), - Arrays.toString(enumType.enumValues))); - } - for (StructTypeDefinition structType : typesDef.structTypesAsJavaList()) { - System.out.println(String.format("%s(%s) - attributes %s", structType.typeName, StructType.class.getSimpleName(), - Arrays.toString(structType.attributeDefinitions))); - } - for (HierarchicalTypeDefinition classType : typesDef.classTypesAsJavaList()) { - System.out.println(String.format("%s(%s) - super types [%s] - attributes %s", classType.typeName, ClassType.class.getSimpleName(), - StringUtils.join(classType.superTypes, ","), Arrays.toString(classType.attributeDefinitions))); - } - for (HierarchicalTypeDefinition traitType : typesDef.traitTypesAsJavaList()) { - System.out.println(String.format("%s(%s) - %s", traitType.typeName, TraitType.class.getSimpleName(), - Arrays.toString(traitType.attributeDefinitions))); - } - } - - private void createColumnLineageClass() throws AtlasException { - - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("query", HiveDataTypes.HIVE_PROCESS.getName(), - Multiplicity.REQUIRED, false, null), - new AttributeDefinition("depenendencyType",DataTypes.STRING_TYPE.getName(), - Multiplicity.REQUIRED, false, null), - new AttributeDefinition("expression",DataTypes.STRING_TYPE.getName(), - Multiplicity.OPTIONAL, false, null) - }; - HierarchicalTypeDefinition definition = - new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_COLUMN_LINEAGE.getName(), null, - ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions); - classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN_LINEAGE.getName(), definition); - LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN_LINEAGE.getName()); - - } - -} - http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java index bdf0d2d..5abf2df 100644 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java @@ -20,7 +20,6 @@ package org.apache.atlas.hive; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; -import org.apache.atlas.fs.model.FSDataTypes; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.hive.hook.HiveHookIT; import org.apache.atlas.hive.model.HiveDataTypes; @@ -93,7 +92,6 @@ public class HiveITBase { } hiveMetaStoreBridge = new HiveMetaStoreBridge(configuration, conf, atlasClient); - hiveMetaStoreBridge.registerHiveDataModel(); HiveConf conf = new HiveConf(); conf.set("hive.exec.post.hooks", ""); @@ -232,7 +230,7 @@ public class HiveITBase { private String assertHDFSPathIsRegistered(String path) throws Exception { LOG.debug("Searching for hdfs path {}", path); - return assertEntityIsRegistered(FSDataTypes.HDFS_PATH().toString(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, path, null); + return assertEntityIsRegistered(HiveMetaStoreBridge.HDFS_PATH, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, path, null); } protected String assertDatabaseIsRegistered(String dbName) throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java index e488f93..0cba27e 100644 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java @@ -20,7 +20,6 @@ package org.apache.atlas.hive.bridge; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasServiceException; -import org.apache.atlas.hive.model.HiveDataModelGenerator; import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.typesystem.Referenceable; import org.apache.hadoop.fs.Path; @@ -32,7 +31,6 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.mapred.TextInputFormat; -import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.mockito.ArgumentMatcher; import org.mockito.Mock; @@ -110,7 +108,7 @@ public class HiveMetaStoreBridgeTest { // verify update is called on table verify(atlasClient).updateEntity(eq("82e06b34-9151-4023-aa9d-b82103a50e77"), - (Referenceable) argThat(new MatchesReferenceableProperty(HiveDataModelGenerator.TABLE_TYPE_ATTR, + (Referenceable) argThat(new MatchesReferenceableProperty(HiveMetaStoreBridge.TABLE_TYPE_ATTR, TableType.EXTERNAL_TABLE.name()))); } @@ -228,7 +226,7 @@ public class HiveMetaStoreBridgeTest { private Referenceable createTableReference() { Referenceable tableReference = new Referenceable(HiveDataTypes.HIVE_TABLE.getName()); Referenceable sdReference = new Referenceable(HiveDataTypes.HIVE_STORAGEDESC.getName()); - tableReference.set(HiveDataModelGenerator.STORAGE_DESC, sdReference); + tableReference.set(HiveMetaStoreBridge.STORAGE_DESC, sdReference); return tableReference; }