atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject [4/4] incubator-atlas git commit: ATLAS-1272: updated types bootstrap to load from new format typedef JSON files
Date Tue, 08 Nov 2016 00:00:15 GMT
ATLAS-1272: updated types bootstrap to load from new format typedef JSON files


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/def9e385
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/def9e385
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/def9e385

Branch: refs/heads/master
Commit: def9e385cc17c2bceb9450c82947370fadc09c3c
Parents: 2ea3a45
Author: Madhan Neethiraj <madhan@apache.org>
Authored: Sun Nov 6 19:32:16 2016 -0800
Committer: Madhan Neethiraj <madhan@apache.org>
Committed: Mon Nov 7 14:06:57 2016 -0800

----------------------------------------------------------------------
 addons/falcon-bridge/pom.xml                    |  18 -
 .../atlas/falcon/bridge/FalconBridge.java       |  46 +-
 .../falcon/model/FalconDataModelGenerator.java  | 214 --------
 .../apache/atlas/falcon/hook/FalconHookIT.java  |  22 +-
 addons/hdfs-model/pom.xml                       |  18 -
 .../atlas/fs/model/FSDataModelGenerator.java    |  38 --
 .../org/apache/atlas/fs/model/FSDataModel.scala |  95 ----
 .../apache/atlas/fs/model/HDFSModelTest.java    |  83 ----
 addons/hive-bridge/pom.xml                      |  18 -
 .../atlas/hive/bridge/ColumnLineageUtils.java   |   5 +-
 .../atlas/hive/bridge/HiveMetaStoreBridge.java  | 101 ++--
 .../org/apache/atlas/hive/hook/HiveHook.java    |   9 +-
 .../hive/model/HiveDataModelGenerator.java      | 350 -------------
 .../java/org/apache/atlas/hive/HiveITBase.java  |   4 +-
 .../hive/bridge/HiveMetaStoreBridgeTest.java    |   6 +-
 .../org/apache/atlas/hive/hook/HiveHookIT.java  |  73 ++-
 addons/models/0010-base_model.json              |  97 ++++
 addons/models/0020-fs_model.json                | 191 +++++++
 addons/models/0030-hive_model.json              | 498 +++++++++++++++++++
 addons/models/0040-sqoop_model.json             |  97 ++++
 addons/models/0050-falcon_model.json            | 147 ++++++
 addons/models/0060-hbase_model.json             |  24 +
 addons/models/0070-kafka_model.json             |  57 +++
 addons/models/0080-storm_model.json             | 150 ++++++
 .../patches/001-hive_column_add_position.json   |  21 +
 addons/sqoop-bridge/pom.xml                     |  18 -
 .../org/apache/atlas/sqoop/hook/SqoopHook.java  |  46 +-
 .../sqoop/model/SqoopDataModelGenerator.java    | 187 -------
 .../apache/atlas/sqoop/hook/SqoopHookIT.java    |  26 -
 addons/storm-bridge/pom.xml                     |  18 -
 .../apache/atlas/storm/hook/StormAtlasHook.java |   6 +-
 .../storm/model/StormDataModelGenerator.java    |  41 --
 .../atlas/storm/model/StormDataModel.scala      | 104 ----
 .../atlas/storm/hook/StormAtlasHookIT.java      |  56 ---
 .../src/main/assemblies/standalone-package.xml  |  30 +-
 .../java/org/apache/atlas/AtlasErrorCode.java   |   4 +-
 .../atlas/model/typedef/AtlasStructDef.java     |  26 +-
 .../atlas/model/typedef/AtlasTypesDef.java      |   2 +
 .../org/apache/atlas/type/AtlasStructType.java  |  14 +-
 .../apache/atlas/type/AtlasTypeRegistry.java    |   8 +
 .../apache/atlas/RepositoryMetadataModule.java  |   4 -
 .../graph/GraphBackedSearchIndexer.java         |   4 +-
 .../bootstrap/AtlasTypeDefStoreInitializer.java | 364 ++++++++++++++
 .../store/graph/AtlasTypeDefGraphStore.java     |  13 +
 .../store/graph/v1/AtlasStructDefStoreV1.java   |  35 +-
 .../atlas/services/AtlasPatchHandler.java       | 173 -------
 .../atlas/services/AtlasTypeAttributePatch.java | 296 -----------
 .../apache/atlas/services/AtlasTypePatch.java   | 104 ----
 .../atlas/services/DefaultMetadataService.java  |  82 +--
 .../services/IBootstrapTypesRegistrar.java      |  27 -
 .../atlas/services/ReservedTypesRegistrar.java  | 135 -----
 .../ReservedTypesRegistrationException.java     |  26 -
 .../org/apache/atlas/BaseRepositoryTest.java    |  42 ++
 .../test/java/org/apache/atlas/TestUtils.java   |   2 -
 .../store/graph/AtlasTypeDefGraphStoreTest.java |   4 +-
 .../DefaultMetadataServiceMockTest.java         | 151 ------
 .../services/ReservedTypesRegistrarTest.java    | 103 ----
 57 files changed, 1917 insertions(+), 2616 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/falcon-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/pom.xml b/addons/falcon-bridge/pom.xml
index de3e810..96743e7 100644
--- a/addons/falcon-bridge/pom.xml
+++ b/addons/falcon-bridge/pom.xml
@@ -355,24 +355,6 @@
                 <version>1.2.1</version>
                 <inherited>false</inherited>
                 <executions>
-                    <execution>
-                        <configuration>
-                            <mainClass>org.apache.atlas.falcon.model.FalconDataModelGenerator</mainClass>
-                            <systemProperties>
-                                <systemProperty>
-                                    <key>atlas.conf</key>
-                                    <value>${project.build.directory}/../../../typesystem/target/test-classes</value>
-                                </systemProperty>
-                            </systemProperties>
-                            <arguments>
-                                <argument>${project.build.directory}/models/falcon_model.json</argument>
-                            </arguments>
-                        </configuration>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>java</goal>
-                        </goals>
-                    </execution>
                 </executions>
             </plugin>
         </plugins>

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
index 05d072f..d1a7e87 100644
--- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
+++ b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
@@ -21,11 +21,8 @@ package org.apache.atlas.falcon.bridge;
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasConstants;
 import org.apache.atlas.falcon.Util.EventUtil;
-import org.apache.atlas.falcon.model.FalconDataModelGenerator;
 import org.apache.atlas.falcon.model.FalconDataTypes;
-import org.apache.atlas.fs.model.FSDataTypes;
 import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
 import org.apache.atlas.hive.model.HiveDataTypes;
 import org.apache.atlas.typesystem.Referenceable;
 import org.apache.commons.collections.CollectionUtils;
@@ -61,6 +58,15 @@ import java.util.Map;
 public class FalconBridge {
     private static final Logger LOG = LoggerFactory.getLogger(FalconBridge.class);
 
+    public static final String COLO = "colo";
+    public static final String TAGS = "tags";
+    public static final String GROUPS = "groups";
+    public static final String PIPELINES = "pipelines";
+    public static final String WFPROPERTIES = "workflow-properties";
+    public static final String RUNSON = "runs-on";
+    public static final String STOREDIN = "stored-in";
+    public static final String FREQUENCY = "frequency";
+
     /**
      * Creates cluster entity
      *
@@ -77,14 +83,14 @@ public class FalconBridge {
         clusterRef.set(AtlasClient.DESCRIPTION, cluster.getDescription());
         clusterRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, cluster.getName());
 
-        clusterRef.set(FalconDataModelGenerator.COLO, cluster.getColo());
+        clusterRef.set(FalconBridge.COLO, cluster.getColo());
 
         if (cluster.getACL() != null) {
             clusterRef.set(AtlasClient.OWNER, cluster.getACL().getGroup());
         }
 
         if (StringUtils.isNotEmpty(cluster.getTags())) {
-            clusterRef.set(FalconDataModelGenerator.TAGS,
+            clusterRef.set(FalconBridge.TAGS,
                     EventUtil.convertKeyValueStringToMap(cluster.getTags()));
         }
 
@@ -100,19 +106,19 @@ public class FalconBridge {
         String feedQualifiedName =
                 getFeedQualifiedName(feed.getName(), (String) clusterReferenceable.get(AtlasClient.NAME));
         feedEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, feedQualifiedName);
-        feedEntity.set(FalconDataModelGenerator.FREQUENCY, feed.getFrequency().toString());
-        feedEntity.set(FalconDataModelGenerator.STOREDIN, clusterReferenceable);
+        feedEntity.set(FalconBridge.FREQUENCY, feed.getFrequency().toString());
+        feedEntity.set(FalconBridge.STOREDIN, clusterReferenceable);
         if (feed.getACL() != null) {
             feedEntity.set(AtlasClient.OWNER, feed.getACL().getOwner());
         }
 
         if (StringUtils.isNotEmpty(feed.getTags())) {
-            feedEntity.set(FalconDataModelGenerator.TAGS,
+            feedEntity.set(FalconBridge.TAGS,
                     EventUtil.convertKeyValueStringToMap(feed.getTags()));
         }
 
         if (feed.getGroups() != null) {
-            feedEntity.set(FalconDataModelGenerator.GROUPS, feed.getGroups());
+            feedEntity.set(FalconBridge.GROUPS, feed.getGroups());
         }
 
         return feedEntity;
@@ -165,7 +171,7 @@ public class FalconBridge {
                         feedCreateEntity.set(AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS, outputs);
                     }
 
-                    feedCreateEntity.set(FalconDataModelGenerator.STOREDIN, clusterReferenceable);
+                    feedCreateEntity.set(FalconBridge.STOREDIN, clusterReferenceable);
                     entities.add(feedCreateEntity);
                 }
 
@@ -244,7 +250,7 @@ public class FalconBridge {
                     processEntity.set(AtlasClient.NAME, process.getName());
                     processEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
                             getProcessQualifiedName(process.getName(), cluster.getName()));
-                    processEntity.set(FalconDataModelGenerator.FREQUENCY, process.getFrequency().toString());
+                    processEntity.set(FalconBridge.FREQUENCY, process.getFrequency().toString());
 
                     if (!inputs.isEmpty()) {
                         processEntity.set(AtlasClient.PROCESS_ATTRIBUTE_INPUTS, inputs);
@@ -254,7 +260,7 @@ public class FalconBridge {
                     }
 
                     // set cluster
-                    processEntity.set(FalconDataModelGenerator.RUNSON, clusterReferenceable);
+                    processEntity.set(FalconBridge.RUNSON, clusterReferenceable);
 
                     // Set user
                     if (process.getACL() != null) {
@@ -262,15 +268,15 @@ public class FalconBridge {
                     }
 
                     if (StringUtils.isNotEmpty(process.getTags())) {
-                        processEntity.set(FalconDataModelGenerator.TAGS,
+                        processEntity.set(FalconBridge.TAGS,
                                 EventUtil.convertKeyValueStringToMap(process.getTags()));
                     }
 
                     if (process.getPipelines() != null) {
-                        processEntity.set(FalconDataModelGenerator.PIPELINES, process.getPipelines());
+                        processEntity.set(FalconBridge.PIPELINES, process.getPipelines());
                     }
 
-                    processEntity.set(FalconDataModelGenerator.WFPROPERTIES,
+                    processEntity.set(FalconBridge.WFPROPERTIES,
                             getProcessEntityWFProperties(process.getWorkflow(),
                                     process.getName()));
 
@@ -319,7 +325,7 @@ public class FalconBridge {
 
     private static List<Referenceable> fillHDFSDataSet(final String pathUri, final String clusterName) {
         List<Referenceable> entities = new ArrayList<>();
-        Referenceable ref = new Referenceable(FSDataTypes.HDFS_PATH().toString());
+        Referenceable ref = new Referenceable(HiveMetaStoreBridge.HDFS_PATH);
         ref.set("path", pathUri);
         //        Path path = new Path(pathUri);
         //        ref.set("name", path.getName());
@@ -352,7 +358,7 @@ public class FalconBridge {
         tableRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
                 HiveMetaStoreBridge.getTableQualifiedName(clusterName, dbName, tableName));
         tableRef.set(AtlasClient.NAME, tableName.toLowerCase());
-        tableRef.set(HiveDataModelGenerator.DB, dbRef);
+        tableRef.set(HiveMetaStoreBridge.DB, dbRef);
         entities.add(tableRef);
 
         return entities;
@@ -364,7 +370,7 @@ public class FalconBridge {
         Referenceable clusterRef = new Referenceable(FalconDataTypes.FALCON_CLUSTER.getName());
         clusterRef.set(AtlasClient.NAME, String.format("%s", clusterName));
         clusterRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, clusterName);
-        clusterRef.set(FalconDataModelGenerator.COLO, colo);
+        clusterRef.set(FalconBridge.COLO, colo);
         return clusterRef;
     }
 
@@ -375,8 +381,8 @@ public class FalconBridge {
         feedDatasetRef.set(AtlasClient.NAME, feed.getName());
         feedDatasetRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, getFeedQualifiedName(feed.getName(),
                 (String) clusterReference.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME)));
-        feedDatasetRef.set(FalconDataModelGenerator.STOREDIN, clusterReference);
-        feedDatasetRef.set(FalconDataModelGenerator.FREQUENCY, feed.getFrequency());
+        feedDatasetRef.set(FalconBridge.STOREDIN, clusterReference);
+        feedDatasetRef.set(FalconBridge.FREQUENCY, feed.getFrequency());
         return feedDatasetRef;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java
deleted file mode 100644
index fc0a9b2..0000000
--- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.falcon.model;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import org.apache.atlas.AtlasClient;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.addons.ModelDefinitionDump;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.AttributeDefinition;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.EnumType;
-import org.apache.atlas.typesystem.types.EnumTypeDefinition;
-import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.StructType;
-import org.apache.atlas.typesystem.types.StructTypeDefinition;
-import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Utility that generates falcon data model.
- */
-public class FalconDataModelGenerator {
-
-    private static final Logger LOG = LoggerFactory.getLogger(FalconDataModelGenerator.class);
-    public static final String FREQUENCY = "frequency";
-
-    private final Map<String, HierarchicalTypeDefinition<ClassType>> classTypeDefinitions;
-
-    public static final String COLO = "colo";
-    public static final String TAGS = "tags";
-    public static final String GROUPS = "groups";
-    public static final String PIPELINES = "pipelines";
-    public static final String WFPROPERTIES = "workflow-properties";
-    public static final String RUNSON = "runs-on";
-    public static final String STOREDIN = "stored-in";
-
-    public FalconDataModelGenerator() {
-        classTypeDefinitions = new HashMap<>();
-    }
-
-    public void createDataModel() throws AtlasException {
-        LOG.info("Generating the Falcon Data Model");
-
-        // classes
-        createClusterEntityClass();
-        createProcessEntityClass();
-        createFeedCreationEntityClass();
-        createFeedEntityClass();
-        createReplicationFeedEntityClass();
-    }
-
-    private TypesDef getTypesDef() {
-        return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
-                getTraitTypeDefinitions(), getClassTypeDefinitions());
-    }
-
-    public String getDataModelAsJSON() {
-        return TypesSerialization.toJson(getTypesDef());
-    }
-
-    private ImmutableList<HierarchicalTypeDefinition<ClassType>> getClassTypeDefinitions() {
-        return ImmutableList.copyOf(classTypeDefinitions.values());
-    }
-
-    private ImmutableList<HierarchicalTypeDefinition<TraitType>> getTraitTypeDefinitions() {
-        return ImmutableList.of();
-    }
-
-    private void createClusterEntityClass() throws AtlasException {
-        AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
-                new AttributeDefinition(COLO, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
-                        false, true, null),
-                // map of tags
-                new AttributeDefinition(TAGS,
-                        DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
-                        Multiplicity.OPTIONAL, false, null),};
-
-        HierarchicalTypeDefinition<ClassType> definition =
-                new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_CLUSTER.getName(), null,
-                        ImmutableSet.of(AtlasClient.INFRASTRUCTURE_SUPER_TYPE), attributeDefinitions);
-        classTypeDefinitions.put(FalconDataTypes.FALCON_CLUSTER.getName(), definition);
-        LOG.debug("Created definition for {}", FalconDataTypes.FALCON_CLUSTER.getName());
-    }
-
-    private void createFeedCreationEntityClass() throws AtlasException {
-        AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
-                new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
-                        false, false, true, null)
-        };
-
-        HierarchicalTypeDefinition<ClassType> definition =
-                new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_FEED_CREATION.getName(), null,
-                        ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
-        classTypeDefinitions.put(FalconDataTypes.FALCON_FEED_CREATION.getName(), definition);
-        LOG.debug("Created definition for {}", FalconDataTypes.FALCON_FEED_CREATION.getName());
-    }
-
-    private void createFeedEntityClass() throws AtlasException {
-        AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
-                TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE),
-                new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
-                        false, false, true, null),
-                new AttributeDefinition(GROUPS, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
-                // map of tags
-                new AttributeDefinition(TAGS,
-                        DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
-                        Multiplicity.OPTIONAL, false, null)
-        };
-
-        HierarchicalTypeDefinition<ClassType> definition =
-                new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_FEED.getName(), null,
-                        ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
-        classTypeDefinitions.put(FalconDataTypes.FALCON_FEED.getName(), definition);
-        LOG.debug("Created definition for {}", FalconDataTypes.FALCON_FEED.getName());
-    }
-
-
-    private void createReplicationFeedEntityClass() throws AtlasException {
-        HierarchicalTypeDefinition<ClassType> definition =
-                new HierarchicalTypeDefinition<>(ClassType.class,
-                        FalconDataTypes.FALCON_FEED_REPLICATION.getName(), null,
-                        ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), null);
-        classTypeDefinitions.put(FalconDataTypes.FALCON_FEED_REPLICATION.getName(), definition);
-        LOG.debug("Created definition for {}", FalconDataTypes.FALCON_FEED_REPLICATION.getName());
-    }
-
-    private void createProcessEntityClass() throws AtlasException {
-        AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
-                TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE),
-                new AttributeDefinition(RUNSON, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
-                        false, false, true, null),
-                // map of tags
-                new AttributeDefinition(TAGS,
-                        DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
-                        Multiplicity.OPTIONAL, false, null),
-                new AttributeDefinition(PIPELINES, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null),
-                // wf properties
-                new AttributeDefinition(WFPROPERTIES,
-                        DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
-                        Multiplicity.OPTIONAL, false, null),};
-
-        HierarchicalTypeDefinition<ClassType> definition =
-                new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_PROCESS.getName(), null,
-                        ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
-        classTypeDefinitions.put(FalconDataTypes.FALCON_PROCESS.getName(), definition);
-        LOG.debug("Created definition for {}", FalconDataTypes.FALCON_PROCESS.getName());
-    }
-
-
-    public String getModelAsJson() throws AtlasException {
-        createDataModel();
-        return getDataModelAsJSON();
-    }
-
-    public static void main(String[] args) throws Exception {
-        FalconDataModelGenerator falconDataModelGenerator = new FalconDataModelGenerator();
-        String modelAsJson = falconDataModelGenerator.getModelAsJson();
-
-        if (args.length == 1) {
-            ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson);
-            return;
-        }
-
-        System.out.println("falconDataModelAsJSON = " + modelAsJson);
-
-        TypesDef typesDef = falconDataModelGenerator.getTypesDef();
-        for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) {
-            System.out.println(String.format("%s(%s) - values %s", enumType.name, EnumType.class.getSimpleName(),
-                    Arrays.toString(enumType.enumValues)));
-        }
-        for (StructTypeDefinition structType : typesDef.structTypesAsJavaList()) {
-            System.out.println(
-                    String.format("%s(%s) - attributes %s", structType.typeName, StructType.class.getSimpleName(),
-                            Arrays.toString(structType.attributeDefinitions)));
-        }
-        for (HierarchicalTypeDefinition<ClassType> classType : typesDef.classTypesAsJavaList()) {
-            System.out.println(String.format("%s(%s) - super types [%s] - attributes %s", classType.typeName,
-                    ClassType.class.getSimpleName(),
-                    StringUtils.join(classType.superTypes, ","), Arrays.toString(classType.attributeDefinitions)));
-        }
-        for (HierarchicalTypeDefinition<TraitType> traitType : typesDef.traitTypesAsJavaList()) {
-            System.out.println(String.format("%s(%s) - %s", traitType.typeName, TraitType.class.getSimpleName(),
-                    Arrays.toString(traitType.attributeDefinitions)));
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
index 31b70fd..8d0a47a 100644
--- a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
+++ b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
@@ -23,9 +23,7 @@ import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasServiceException;
 import org.apache.atlas.falcon.bridge.FalconBridge;
-import org.apache.atlas.falcon.model.FalconDataModelGenerator;
 import org.apache.atlas.falcon.model.FalconDataTypes;
-import org.apache.atlas.fs.model.FSDataTypes;
 import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
 import org.apache.atlas.hive.model.HiveDataTypes;
 import org.apache.atlas.typesystem.Referenceable;
@@ -47,7 +45,6 @@ import org.apache.falcon.entity.v0.feed.Location;
 import org.apache.falcon.entity.v0.feed.LocationType;
 import org.apache.falcon.entity.v0.process.Process;
 import org.apache.falcon.security.CurrentUser;
-import org.apache.hadoop.hive.conf.HiveConf;
 import org.slf4j.Logger;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
@@ -84,24 +81,9 @@ public class FalconHookIT {
         AtlasService service = new AtlasService();
         service.init();
         STORE.registerListener(service);
-        registerFalconDataModel();
         CurrentUser.authenticate(System.getProperty("user.name"));
     }
 
-    private void registerFalconDataModel() throws Exception {
-        if (isDataModelAlreadyRegistered()) {
-            LOG.info("Falcon data model is already registered!");
-            return;
-        }
-
-        HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(ApplicationProperties.get(), new HiveConf(), atlasClient);
-        hiveMetaStoreBridge.registerHiveDataModel();
-
-        FalconDataModelGenerator dataModelGenerator = new FalconDataModelGenerator();
-        LOG.info("Registering Falcon data model");
-        atlasClient.createType(dataModelGenerator.getModelAsJson());
-    }
-
     private boolean isDataModelAlreadyRegistered() throws Exception {
         try {
             atlasClient.getType(FalconDataTypes.FALCON_PROCESS.getName());
@@ -196,7 +178,7 @@ public class FalconHookIT {
 
         String inputId = ((List<Id>) processEntity.get("inputs")).get(0).getId()._getId();
         Referenceable pathEntity = atlasClient.getEntity(inputId);
-        assertEquals(pathEntity.getTypeName(), FSDataTypes.HDFS_PATH().toString());
+        assertEquals(pathEntity.getTypeName(), HiveMetaStoreBridge.HDFS_PATH.toString());
 
         List<Location> locations = FeedHelper.getLocations(feedCluster, feed);
         Location dataLocation = FileSystemStorage.getLocation(locations, LocationType.DATA);
@@ -243,7 +225,7 @@ public class FalconHookIT {
     private void assertFeedAttributes(String feedId) throws Exception {
         Referenceable feedEntity = atlasClient.getEntity(feedId);
         assertEquals(feedEntity.get(AtlasClient.OWNER), "testuser");
-        assertEquals(feedEntity.get(FalconDataModelGenerator.FREQUENCY), "hours(1)");
+        assertEquals(feedEntity.get(FalconBridge.FREQUENCY), "hours(1)");
         assertEquals(feedEntity.get(AtlasClient.DESCRIPTION), "test input");
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hdfs-model/pom.xml
----------------------------------------------------------------------
diff --git a/addons/hdfs-model/pom.xml b/addons/hdfs-model/pom.xml
index e2ab029..1376fdc 100644
--- a/addons/hdfs-model/pom.xml
+++ b/addons/hdfs-model/pom.xml
@@ -180,24 +180,6 @@
                 <version>1.2.1</version>
                 <inherited>false</inherited>
                 <executions>
-                    <execution>
-                        <configuration>
-                            <mainClass>org.apache.atlas.fs.model.FSDataModelGenerator</mainClass>
-                            <systemProperties>
-                                <systemProperty>
-                                    <key>atlas.conf</key>
-                                    <value>${project.build.directory}/../../../typesystem/src/test/resources/</value>
-                                </systemProperty>
-                            </systemProperties>
-                            <arguments>
-                                <argument>${project.build.directory}/models/fs_model.json</argument>
-                            </arguments>
-                        </configuration>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>java</goal>
-                        </goals>
-                    </execution>
                 </executions>
             </plugin>
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hdfs-model/src/main/java/org/apache/atlas/fs/model/FSDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/hdfs-model/src/main/java/org/apache/atlas/fs/model/FSDataModelGenerator.java b/addons/hdfs-model/src/main/java/org/apache/atlas/fs/model/FSDataModelGenerator.java
deleted file mode 100644
index 555d565..0000000
--- a/addons/hdfs-model/src/main/java/org/apache/atlas/fs/model/FSDataModelGenerator.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.fs.model;
-
-import org.apache.atlas.addons.ModelDefinitionDump;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-
-import java.io.IOException;
-
-public class FSDataModelGenerator {
-
-    public static void main(String[] args) throws IOException {
-      FSDataModel.main(args);
-      TypesDef typesDef = FSDataModel.typesDef();
-      String fsTypesAsJSON = TypesSerialization.toJson(typesDef);
-      if (args.length == 1) {
-        ModelDefinitionDump.dumpModelToFile(args[0], fsTypesAsJSON);
-        return;
-      }
-      System.out.println("FS Data Model as JSON = " + fsTypesAsJSON);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala
----------------------------------------------------------------------
diff --git a/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala b/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala
deleted file mode 100644
index ad40340..0000000
--- a/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.fs.model
-
-import org.apache.atlas.{AtlasConstants, AtlasClient}
-import org.apache.atlas.typesystem.TypesDef
-import org.apache.atlas.typesystem.builders.TypesBuilder
-import org.apache.atlas.typesystem.json.TypesSerialization
-import org.apache.atlas.typesystem.types.DataTypes.MapType
-import org.apache.hadoop.fs.permission.FsAction
-
-import scala.tools.scalap.scalax.rules.scalasig.ClassFileParser.EnumConstValue
-
-/**
- * This represents the data model for a HDFS Path
- */
-object FSDataModel extends App {
-
-    val typesBuilder = new TypesBuilder
-    import typesBuilder._
-
-    val typesDef : TypesDef = types {
-
-        // FS DataSet
-        _class(FSDataTypes.FS_PATH.toString, List(AtlasClient.DATA_SET_SUPER_TYPE)) {
-            //fully qualified path/URI to the filesystem path is stored in 'qualifiedName' and 'path'.
-            "path" ~ (string, required, indexed)
-            "createTime" ~ (date, optional, indexed)
-            "modifiedTime" ~ (date, optional, indexed)
-            //Is a regular file or a directory. If true, it is a file else a directory
-            "isFile" ~ (boolean, optional, indexed)
-            //Is a symlink or not
-            "isSymlink" ~ (boolean, optional)
-            //Optional and may not be set for a directory
-            "fileSize" ~ (long, optional)
-            "group" ~ (string, optional, indexed)
-            "posixPermissions" ~ (FSDataTypes.FS_PERMISSIONS.toString, optional, indexed)
-        }
-
-        enum(FSDataTypes.FS_ACTION.toString,  FsAction.values().map(x => x.name()) : _*)
-
-        struct(FSDataTypes.FS_PERMISSIONS.toString) {
-            PosixPermissions.PERM_USER.toString ~ (FSDataTypes.FS_ACTION.toString, required, indexed)
-            PosixPermissions.PERM_GROUP.toString ~ (FSDataTypes.FS_ACTION.toString, required, indexed)
-            PosixPermissions.PERM_OTHER.toString ~ (FSDataTypes.FS_ACTION.toString, required, indexed)
-            PosixPermissions.STICKY_BIT.toString ~ (boolean, required, indexed)
-        }
-
-        //HDFS DataSet
-        _class(FSDataTypes.HDFS_PATH.toString, List(FSDataTypes.FS_PATH.toString)) {
-            //Making cluster optional since path is already unique containing the namenode URI
-            AtlasConstants.CLUSTER_NAME_ATTRIBUTE ~ (string, optional, indexed)
-            "numberOfReplicas" ~ (int, optional, indexed)
-            "extendedAttributes" ~ (map(string, string), optional)
-        }
-        //TODO - ACLs - https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html#ACLs_Access_Control_Lists
-    }
-
-    // add the types to atlas
-    val typesAsJSON = TypesSerialization.toJson(typesDef)
-    println("FS Data Model as JSON: ")
-    println(typesAsJSON)
-
-}
-
-object FSDataTypes extends Enumeration {
-    type FSDataTypes = Value
-    val FS_ACTION = Value("file_action")
-    val FS_PATH = Value("fs_path")
-    val HDFS_PATH = Value("hdfs_path")
-    val FS_PERMISSIONS = Value("fs_permissions")
-}
-
-object PosixPermissions extends Enumeration {
-    type PosixPermissions = Value
-    val PERM_USER = Value("user")
-    val PERM_GROUP = Value("group")
-    val PERM_OTHER = Value("others")
-    val STICKY_BIT = Value("sticky")
-}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java
----------------------------------------------------------------------
diff --git a/addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java b/addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java
deleted file mode 100644
index 5944da8..0000000
--- a/addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.fs.model;
-
-import javax.inject.Inject;
-
-import org.apache.atlas.RepositoryMetadataModule;
-import org.apache.atlas.repository.graph.AtlasGraphProvider;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.services.MetadataService;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.TypeSystem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import scala.Enumeration;
-import scala.collection.Iterator;
-
-@Test
-@Guice(modules = RepositoryMetadataModule.class)
-public class HDFSModelTest {
-
-    public static final Logger LOG = LoggerFactory.getLogger(HDFSModelTest.class);
-    private static final String ATLAS_URL = "http://localhost:21000/";
-
-    @Inject
-    private MetadataService metadataService;
-
-    @BeforeClass
-    public void setUp() throws Exception {
-    }
-
-    @AfterClass
-    public void tearDown() throws Exception {
-        TypeSystem.getInstance().reset();
-        AtlasGraphProvider.cleanup();
-    }
-
-    @Test
-    public void testCreateDataModel() throws Exception {
-        FSDataModel.main(new String[]{});
-        TypesDef fsTypesDef = FSDataModel.typesDef();
-
-        String fsTypesAsJSON = TypesSerialization.toJson(fsTypesDef);
-        LOG.info("fsTypesAsJSON = {}", fsTypesAsJSON);
-
-        metadataService.createType(fsTypesAsJSON);
-
-        // verify types are registered
-        final Iterator<Enumeration.Value> valueIterator = FSDataTypes.values().iterator();
-        while (valueIterator.hasNext() ) {
-            final Enumeration.Value typeEnum = valueIterator.next();
-            String typeDefStr = metadataService.getTypeDefinition(typeEnum.toString());
-            Assert.assertNotNull(typeDefStr);
-
-            TypesDef typesDef = TypesSerialization.fromJson(typeDefStr);
-            Assert.assertNotNull(typesDef);
-        }
-    }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/pom.xml b/addons/hive-bridge/pom.xml
index 884791d..7ab47f1 100755
--- a/addons/hive-bridge/pom.xml
+++ b/addons/hive-bridge/pom.xml
@@ -399,24 +399,6 @@
                 <version>1.2.1</version>
                 <inherited>false</inherited>
                 <executions>
-                    <execution>
-                        <configuration>
-                            <mainClass>org.apache.atlas.hive.model.HiveDataModelGenerator</mainClass>
-                            <systemProperties>
-                                <systemProperty>
-                                    <key>atlas.conf</key>
-                                    <value>${project.build.directory}/../../../typesystem/target/test-classes</value>
-                                </systemProperty>
-                            </systemProperties>
-                            <arguments>
-                                <argument>${project.build.directory}/models/hive_model.json</argument>
-                            </arguments>
-                        </configuration>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>java</goal>
-                        </goals>
-                    </execution>
                 </executions>
             </plugin>
         </plugins>

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java
index e4a20e1..c54fdb3 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java
@@ -19,8 +19,6 @@
 package org.apache.atlas.hive.bridge;
 
 import org.apache.atlas.AtlasClient;
-import org.apache.atlas.hive.hook.HiveHook;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
 import org.apache.atlas.hive.model.HiveDataTypes;
 import org.apache.atlas.typesystem.Referenceable;
 import org.apache.hadoop.hive.ql.hooks.LineageInfo;
@@ -31,7 +29,6 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 
 public class ColumnLineageUtils {
     public static final Logger LOG = LoggerFactory.getLogger(ColumnLineageUtils.class);
@@ -92,7 +89,7 @@ public class ColumnLineageUtils {
         if (r.getTypeName().equals(HiveDataTypes.HIVE_TABLE.getName())) {
             String qName = (String) r.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME);
             String[] qNameComps = extractComponents(qName);
-            for (Referenceable col : (List<Referenceable>) r.get(HiveDataModelGenerator.COLUMNS)) {
+            for (Referenceable col : (List<Referenceable>) r.get(HiveMetaStoreBridge.COLUMNS)) {
                 String cName = (String) col.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME);
                 String[] colQNameComps = extractComponents(cName);
                 String colQName = colQNameComps[0] + "." + colQNameComps[1] + "." + colQNameComps[2];

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index 2d2039b..cbc51cc 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -26,15 +26,11 @@ import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasConstants;
 import org.apache.atlas.AtlasServiceException;
-import org.apache.atlas.fs.model.FSDataModel;
-import org.apache.atlas.fs.model.FSDataTypes;
 import org.apache.atlas.hive.hook.HiveHook;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
 import org.apache.atlas.hive.model.HiveDataTypes;
 import org.apache.atlas.typesystem.Referenceable;
 import org.apache.atlas.typesystem.Struct;
 import org.apache.atlas.typesystem.json.InstanceSerialization;
-import org.apache.atlas.typesystem.json.TypesSerialization;
 import org.apache.atlas.typesystem.persistence.Id;
 import org.apache.atlas.utils.AuthenticationUtil;
 import org.apache.commons.cli.BasicParser;
@@ -79,6 +75,25 @@ public class HiveMetaStoreBridge {
 
     public static final String ATLAS_ENDPOINT = "atlas.rest.address";
 
+    public static final String COMMENT = "comment";
+    public static final String PARAMETERS = "parameters";
+    public static final String COLUMNS = "columns";
+    public static final String POSITION = "position";
+    public static final String PART_COLS = "partitionKeys";
+    public static final String TABLE_ALIAS_LIST = "aliases";
+    public static final String STORAGE_NUM_BUCKETS = "numBuckets";
+    public static final String STORAGE_IS_STORED_AS_SUB_DIRS = "storedAsSubDirectories";
+    public static final String TABLE = "table";
+    public static final String DB = "db";
+    public static final String STORAGE_DESC = "sd";
+    public static final String STORAGE_DESC_INPUT_FMT = "inputFormat";
+    public static final String STORAGE_DESC_OUTPUT_FMT = "outputFormat";
+    public static final String LOCATION = "location";
+    public static final String TABLE_TYPE_ATTR = "tableType";
+    public static final String CREATE_TIME = "createTime";
+    public static final String LAST_ACCESS_TIME = "lastAccessTime";
+    public static final String HDFS_PATH = "hdfs_path";
+
     private static final Logger LOG = LoggerFactory.getLogger(HiveMetaStoreBridge.class);
 
     public final Hive hiveClient;
@@ -174,8 +189,8 @@ public class HiveMetaStoreBridge {
         dbRef.set(AtlasClient.NAME, dbName);
         dbRef.set(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, clusterName);
         dbRef.set(DESCRIPTION_ATTR, hiveDB.getDescription());
-        dbRef.set(HiveDataModelGenerator.LOCATION, hiveDB.getLocationUri());
-        dbRef.set(HiveDataModelGenerator.PARAMETERS, hiveDB.getParameters());
+        dbRef.set(LOCATION, hiveDB.getLocationUri());
+        dbRef.set(PARAMETERS, hiveDB.getParameters());
         dbRef.set(AtlasClient.OWNER, hiveDB.getOwnerName());
         if (hiveDB.getOwnerType() != null) {
             dbRef.set("ownerType", hiveDB.getOwnerType().getValue());
@@ -431,7 +446,7 @@ public class HiveMetaStoreBridge {
             try {
                 createDate = getTableCreatedTime(hiveTable);
                 LOG.debug("Setting create time to {} ", createDate);
-                tableReference.set(HiveDataModelGenerator.CREATE_TIME, createDate);
+                tableReference.set(CREATE_TIME, createDate);
             } catch(Exception ne) {
                 LOG.error("Error while setting createTime for the table {} ", hiveTable.getCompleteName(), ne);
             }
@@ -441,19 +456,19 @@ public class HiveMetaStoreBridge {
         if ( hiveTable.getLastAccessTime() > 0) {
             lastAccessTime = new Date(hiveTable.getLastAccessTime() * MILLIS_CONVERT_FACTOR);
         }
-        tableReference.set(HiveDataModelGenerator.LAST_ACCESS_TIME, lastAccessTime);
+        tableReference.set(LAST_ACCESS_TIME, lastAccessTime);
         tableReference.set("retention", hiveTable.getRetention());
 
-        tableReference.set(HiveDataModelGenerator.COMMENT, hiveTable.getParameters().get(HiveDataModelGenerator.COMMENT));
+        tableReference.set(COMMENT, hiveTable.getParameters().get(COMMENT));
 
         // add reference to the database
-        tableReference.set(HiveDataModelGenerator.DB, dbReference);
+        tableReference.set(DB, dbReference);
 
         // add reference to the StorageDescriptor
         Referenceable sdReferenceable = fillStorageDesc(hiveTable.getSd(), tableQualifiedName, getStorageDescQFName(tableQualifiedName), tableReference.getId());
-        tableReference.set(HiveDataModelGenerator.STORAGE_DESC, sdReferenceable);
+        tableReference.set(STORAGE_DESC, sdReferenceable);
 
-        tableReference.set(HiveDataModelGenerator.PARAMETERS, hiveTable.getParameters());
+        tableReference.set(PARAMETERS, hiveTable.getParameters());
 
         if (hiveTable.getViewOriginalText() != null) {
             tableReference.set("viewOriginalText", hiveTable.getViewOriginalText());
@@ -463,14 +478,14 @@ public class HiveMetaStoreBridge {
             tableReference.set("viewExpandedText", hiveTable.getViewExpandedText());
         }
 
-        tableReference.set(HiveDataModelGenerator.TABLE_TYPE_ATTR, hiveTable.getTableType().name());
+        tableReference.set(TABLE_TYPE_ATTR, hiveTable.getTableType().name());
         tableReference.set("temporary", hiveTable.isTemporary());
 
         // add reference to the Partition Keys
         List<Referenceable> partKeys = getColumns(hiveTable.getPartitionKeys(), tableReference);
         tableReference.set("partitionKeys", partKeys);
 
-        tableReference.set(HiveDataModelGenerator.COLUMNS, getColumns(hiveTable.getCols(), tableReference));
+        tableReference.set(COLUMNS, getColumns(hiveTable.getCols(), tableReference));
 
         return tableReference;
     }
@@ -523,12 +538,12 @@ public class HiveMetaStoreBridge {
 
         serdeInfoStruct.set(AtlasClient.NAME, serdeInfo.getName());
         serdeInfoStruct.set("serializationLib", serdeInfo.getSerializationLib());
-        serdeInfoStruct.set(HiveDataModelGenerator.PARAMETERS, serdeInfo.getParameters());
+        serdeInfoStruct.set(PARAMETERS, serdeInfo.getParameters());
 
         sdReferenceable.set("serdeInfo", serdeInfoStruct);
-        sdReferenceable.set(HiveDataModelGenerator.STORAGE_NUM_BUCKETS, storageDesc.getNumBuckets());
+        sdReferenceable.set(STORAGE_NUM_BUCKETS, storageDesc.getNumBuckets());
         sdReferenceable
-                .set(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS, storageDesc.isStoredAsSubDirectories());
+                .set(STORAGE_IS_STORED_AS_SUB_DIRS, storageDesc.isStoredAsSubDirectories());
 
         List<Struct> sortColsStruct = new ArrayList<>();
         for (Order sortcol : storageDesc.getSortCols()) {
@@ -543,7 +558,7 @@ public class HiveMetaStoreBridge {
             sdReferenceable.set("sortCols", sortColsStruct);
         }
 
-        sdReferenceable.set(HiveDataModelGenerator.LOCATION, storageDesc.getLocation());
+        sdReferenceable.set(LOCATION, storageDesc.getLocation());
         sdReferenceable.set("inputFormat", storageDesc.getInputFormat());
         sdReferenceable.set("outputFormat", storageDesc.getOutputFormat());
         sdReferenceable.set("compressed", storageDesc.isCompressed());
@@ -552,15 +567,15 @@ public class HiveMetaStoreBridge {
             sdReferenceable.set("bucketCols", storageDesc.getBucketCols());
         }
 
-        sdReferenceable.set(HiveDataModelGenerator.PARAMETERS, storageDesc.getParameters());
+        sdReferenceable.set(PARAMETERS, storageDesc.getParameters());
         sdReferenceable.set("storedAsSubDirectories", storageDesc.isStoredAsSubDirectories());
-        sdReferenceable.set(HiveDataModelGenerator.TABLE, tableId);
+        sdReferenceable.set(TABLE, tableId);
 
         return sdReferenceable;
     }
 
     public Referenceable fillHDFSDataSet(String pathUri) {
-        Referenceable ref = new Referenceable(FSDataTypes.HDFS_PATH().toString());
+        Referenceable ref = new Referenceable(HDFS_PATH.toString());
         ref.set("path", pathUri);
         Path path = new Path(pathUri);
         ref.set(AtlasClient.NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
@@ -586,9 +601,9 @@ public class HiveMetaStoreBridge {
             colReferenceable.set(AtlasClient.NAME, fs.getName());
             colReferenceable.set(AtlasClient.OWNER, tableReference.get(AtlasClient.OWNER));
             colReferenceable.set("type", fs.getType());
-            colReferenceable.set(HiveDataModelGenerator.POSITION, columnPosition++);
-            colReferenceable.set(HiveDataModelGenerator.COMMENT, fs.getComment());
-            colReferenceable.set(HiveDataModelGenerator.TABLE, tableReference.getId());
+            colReferenceable.set(POSITION, columnPosition++);
+            colReferenceable.set(COMMENT, fs.getComment());
+            colReferenceable.set(TABLE, tableReference.getId());
 
 
             colList.add(colReferenceable);
@@ -596,43 +611,6 @@ public class HiveMetaStoreBridge {
         return colList;
     }
 
-    /**
-     * Register the Hive DataModel in Atlas, if not already defined.
-     *
-     * The method checks for the presence of the type {@link HiveDataTypes#HIVE_PROCESS} with the Atlas server.
-     * If this type is defined, then we assume the Hive DataModel is registered.
-     * @throws Exception
-     */
-    public synchronized void registerHiveDataModel() throws Exception {
-        HiveDataModelGenerator dataModelGenerator = new HiveDataModelGenerator();
-        AtlasClient dgiClient = getAtlasClient();
-
-        try {
-            dgiClient.getType(FSDataTypes.HDFS_PATH().toString());
-            LOG.info("HDFS data model is already registered!");
-        } catch(AtlasServiceException ase) {
-            if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) {
-                //Trigger val definition
-                FSDataModel.main(null);
-
-                final String hdfsModelJson = TypesSerialization.toJson(FSDataModel.typesDef());
-                //Expected in case types do not exist
-                LOG.info("Registering HDFS data model : " + hdfsModelJson);
-                dgiClient.createType(hdfsModelJson);
-            }
-        }
-
-        try {
-            dgiClient.getType(HiveDataTypes.HIVE_PROCESS.getName());
-            LOG.info("Hive data model is already registered!");
-        } catch(AtlasServiceException ase) {
-            if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) {
-                //Expected in case types do not exist
-                LOG.info("Registering Hive data model");
-                dgiClient.createType(dataModelGenerator.getModelAsJson());
-            }
-        }
-    }
 
     public static void main(String[] args) throws Exception {
 
@@ -661,7 +639,6 @@ public class HiveMetaStoreBridge {
         }
 
         HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(atlasConf, new HiveConf(), atlasClient);
-        hiveMetaStoreBridge.registerHiveDataModel();
         hiveMetaStoreBridge.importHiveMetadata(failOnError);
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index eaef337..0c6582b 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -25,7 +25,6 @@ import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasConstants;
 import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
 import org.apache.atlas.hive.bridge.ColumnLineageUtils;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
 import org.apache.atlas.hive.model.HiveDataTypes;
 import org.apache.atlas.hook.AtlasHook;
 import org.apache.atlas.notification.hook.HookNotification;
@@ -411,10 +410,10 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
                     Referenceable tableEntity = tables.get(Type.TABLE);
 
                     //Reset regular column QF Name to old Name and create a new partial notification request to replace old column QFName to newName to retain any existing traits
-                    replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveDataModelGenerator.COLUMNS), oldQualifiedName, newQualifiedName);
+                    replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveMetaStoreBridge.COLUMNS), oldQualifiedName, newQualifiedName);
 
                     //Reset partition key column QF Name to old Name and create a new partial notification request to replace old column QFName to newName to retain any existing traits
-                    replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveDataModelGenerator.PART_COLS), oldQualifiedName, newQualifiedName);
+                    replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveMetaStoreBridge.PART_COLS), oldQualifiedName, newQualifiedName);
 
                     //Reset SD QF Name to old Name and create a new partial notification request to replace old SD QFName to newName to retain any existing traits
                     replaceSDQFName(event, tableEntity, oldQualifiedName, newQualifiedName);
@@ -437,7 +436,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
 
         ArrayList<String> alias_list = new ArrayList<>();
         alias_list.add(oldTable.getTableName().toLowerCase());
-        newEntity.set(HiveDataModelGenerator.TABLE_ALIAS_LIST, alias_list);
+        newEntity.set(HiveMetaStoreBridge.TABLE_ALIAS_LIST, alias_list);
         event.addMessage(new HookNotification.EntityPartialUpdateRequest(event.getUser(),
             HiveDataTypes.HIVE_TABLE.getName(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
             oldTableQFName, newEntity));
@@ -466,7 +465,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
 
     private Referenceable replaceSDQFName(final HiveEventContext event, Referenceable tableEntity, final String oldTblQFName, final String newTblQFName) {
         //Reset storage desc QF Name to old Name
-        final Referenceable sdRef = ((Referenceable) tableEntity.get(HiveDataModelGenerator.STORAGE_DESC));
+        final Referenceable sdRef = ((Referenceable) tableEntity.get(HiveMetaStoreBridge.STORAGE_DESC));
         sdRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, HiveMetaStoreBridge.getStorageDescQFName(oldTblQFName));
 
         //Replace SD QF name first to retain tags

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
deleted file mode 100755
index d0fc43f..0000000
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
+++ /dev/null
@@ -1,350 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.hive.model;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import org.apache.atlas.AtlasClient;
-import org.apache.atlas.AtlasConstants;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.addons.ModelDefinitionDump;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.AttributeDefinition;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.EnumType;
-import org.apache.atlas.typesystem.types.EnumTypeDefinition;
-import org.apache.atlas.typesystem.types.EnumValue;
-import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.StructType;
-import org.apache.atlas.typesystem.types.StructTypeDefinition;
-import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Utility that generates hive data model for both metastore entities and DDL/DML queries.
- */
-public class HiveDataModelGenerator {
-
-    private static final Logger LOG = LoggerFactory.getLogger(HiveDataModelGenerator.class);
-
-    private static final DataTypes.MapType STRING_MAP_TYPE =
-            new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE);
-
-    private final Map<String, HierarchicalTypeDefinition<ClassType>> classTypeDefinitions;
-    private final Map<String, EnumTypeDefinition> enumTypeDefinitionMap;
-    private final Map<String, StructTypeDefinition> structTypeDefinitionMap;
-
-    public static final String COMMENT = "comment";
-    public static final String PARAMETERS = "parameters";
-    public static final String COLUMNS = "columns";
-    public static final String POSITION = "position";
-    public static final String PART_COLS = "partitionKeys";
-    public static final String TABLE_ALIAS_LIST = "aliases";
-
-    public static final String STORAGE_NUM_BUCKETS = "numBuckets";
-    public static final String STORAGE_IS_STORED_AS_SUB_DIRS = "storedAsSubDirectories";
-
-    public static final String TABLE = "table";
-    public static final String DB = "db";
-
-    public static final String STORAGE_DESC = "sd";
-    public static final String STORAGE_DESC_INPUT_FMT = "inputFormat";
-    public static final String STORAGE_DESC_OUTPUT_FMT = "outputFormat";
-    public static final String LOCATION = "location";
-
-    public static final String TABLE_TYPE_ATTR = "tableType";
-
-    public static final String CREATE_TIME = "createTime";
-    public static final String LAST_ACCESS_TIME = "lastAccessTime";
-
-    public HiveDataModelGenerator() {
-        classTypeDefinitions = new HashMap<>();
-        enumTypeDefinitionMap = new HashMap<>();
-        structTypeDefinitionMap = new HashMap<>();
-    }
-
-    public void createDataModel() throws AtlasException {
-        LOG.info("Generating the Hive Data Model....");
-
-        // enums
-        createHivePrincipalTypeEnum();
-        // structs
-        createSerDeStruct();
-        createOrderStruct();
-        createStorageDescClass();
-
-        // classes
-        createDBClass();
-        createColumnClass();
-        createTableClass();
-
-        // DDL/DML Process
-        createProcessClass();
-        createColumnLineageClass();
-    }
-
-    public TypesDef getTypesDef() {
-        return TypesUtil.getTypesDef(getEnumTypeDefinitions(), getStructTypeDefinitions(), getTraitTypeDefinitions(),
-                getClassTypeDefinitions());
-    }
-
-    public String getDataModelAsJSON() {
-        return TypesSerialization.toJson(getTypesDef());
-    }
-
-    public ImmutableList<EnumTypeDefinition> getEnumTypeDefinitions() {
-        return ImmutableList.copyOf(enumTypeDefinitionMap.values());
-    }
-
-    public ImmutableList<StructTypeDefinition> getStructTypeDefinitions() {
-        return ImmutableList.copyOf(structTypeDefinitionMap.values());
-    }
-
-    public ImmutableList<HierarchicalTypeDefinition<ClassType>> getClassTypeDefinitions() {
-        return ImmutableList.copyOf(classTypeDefinitions.values());
-    }
-
-    public ImmutableList<HierarchicalTypeDefinition<TraitType>> getTraitTypeDefinitions() {
-        return ImmutableList.of();
-    }
-
-    private void createHivePrincipalTypeEnum() throws AtlasException {
-        EnumValue values[] = {new EnumValue("USER", 1), new EnumValue("ROLE", 2), new EnumValue("GROUP", 3),};
-
-        EnumTypeDefinition definition = new EnumTypeDefinition(HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), values);
-
-        enumTypeDefinitionMap.put(HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), definition);
-        LOG.debug("Created definition for " + HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName());
-    }
-
-    private void createSerDeStruct() throws AtlasException {
-        AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
-                new AttributeDefinition(AtlasClient.NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
-                new AttributeDefinition("serializationLib", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
-                        false, null),
-                new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),};
-        StructTypeDefinition definition =
-                new StructTypeDefinition(HiveDataTypes.HIVE_SERDE.getName(), attributeDefinitions);
-        structTypeDefinitionMap.put(HiveDataTypes.HIVE_SERDE.getName(), definition);
-        LOG.debug("Created definition for " + HiveDataTypes.HIVE_SERDE.getName());
-    }
-
-    private void createOrderStruct() throws AtlasException {
-        AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
-                new AttributeDefinition("col", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
-                new AttributeDefinition("order", DataTypes.INT_TYPE.getName(), Multiplicity.REQUIRED, false, null),};
-
-        StructTypeDefinition definition =
-                new StructTypeDefinition(HiveDataTypes.HIVE_ORDER.getName(), attributeDefinitions);
-        structTypeDefinitionMap.put(HiveDataTypes.HIVE_ORDER.getName(), definition);
-        LOG.debug("Created definition for " + HiveDataTypes.HIVE_ORDER.getName());
-    }
-
-    private void createStorageDescClass() throws AtlasException {
-        AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
-                //Optional to keep it backward-compatible
-                new AttributeDefinition(TABLE, HiveDataTypes.HIVE_TABLE.getName(), Multiplicity.OPTIONAL, false,
-                        STORAGE_DESC),
-                new AttributeDefinition(LOCATION, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
-                        null),
-                new AttributeDefinition("inputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
-                        null),
-                new AttributeDefinition("outputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
-                        null),
-                new AttributeDefinition("compressed", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.REQUIRED, false,
-                        null),
-                new AttributeDefinition(STORAGE_NUM_BUCKETS, DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false,
-                        null),
-                new AttributeDefinition("serdeInfo", HiveDataTypes.HIVE_SERDE.getName(), Multiplicity.OPTIONAL, true,
-                        null),
-                new AttributeDefinition("bucketCols", String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
-                        Multiplicity.OPTIONAL, false, null),
-                new AttributeDefinition("sortCols", String.format("array<%s>", HiveDataTypes.HIVE_ORDER.getName()),
-                        Multiplicity.OPTIONAL, false, null),
-                new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
-                //new AttributeDefinition("skewedInfo", DefinedTypes.HIVE_SKEWEDINFO.getName(),
-                // Multiplicity.OPTIONAL, false, null),
-                new AttributeDefinition(STORAGE_IS_STORED_AS_SUB_DIRS, DataTypes.BOOLEAN_TYPE.getName(),
-                        Multiplicity.OPTIONAL, false, null),};
-
-        HierarchicalTypeDefinition<ClassType> definition =
-                new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_STORAGEDESC.getName(), null,
-                        ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions);
-        classTypeDefinitions.put(HiveDataTypes.HIVE_STORAGEDESC.getName(), definition);
-        LOG.debug("Created definition for " + HiveDataTypes.HIVE_STORAGEDESC.getName());
-    }
-
-    /** Revisit later after nested array types are handled by the typesystem **/
-
-    private void createDBClass() throws AtlasException {
-        AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
-                new AttributeDefinition(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
-                        false, true, null),
-                new AttributeDefinition(LOCATION, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
-                        null),
-                new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
-                new AttributeDefinition("ownerType", HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), Multiplicity.OPTIONAL,
-                        false, null),};
-
-        HierarchicalTypeDefinition<ClassType> definition =
-                new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_DB.getName(), null,
-                    ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE), attributeDefinitions);
-        classTypeDefinitions.put(HiveDataTypes.HIVE_DB.getName(), definition);
-        LOG.debug("Created definition for " + HiveDataTypes.HIVE_DB.getName());
-    }
-
-    private void createColumnClass() throws AtlasException {
-        AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
-                new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
-                new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
-                //Making this optional since this is an incompatible change
-                //Reverse attribute to 'columns' in Table
-                new AttributeDefinition(TABLE, HiveDataTypes.HIVE_TABLE.getName(), Multiplicity.OPTIONAL, false, COLUMNS),
-                new AttributeDefinition(POSITION, DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, false, false, null)};
-
-        HierarchicalTypeDefinition<ClassType> definition =
-                new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_COLUMN.getName(), null, "1.1",
-                    ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
-        classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN.getName(), definition);
-        LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN.getName());
-    }
-
-    private void createTableClass() throws AtlasException {
-        AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
-                new AttributeDefinition(DB, HiveDataTypes.HIVE_DB.getName(), Multiplicity.REQUIRED, false, null),
-                new AttributeDefinition(CREATE_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.OPTIONAL, false,
-                        null),
-                new AttributeDefinition(LAST_ACCESS_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.OPTIONAL, false,
-                        null),
-                new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
-                new AttributeDefinition("retention", DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
-                new AttributeDefinition(STORAGE_DESC, HiveDataTypes.HIVE_STORAGEDESC.getName(), Multiplicity.OPTIONAL, true,
-                        null),
-                new AttributeDefinition(PART_COLS, DataTypes.arrayTypeName(HiveDataTypes.HIVE_COLUMN.getName()),
-                        Multiplicity.OPTIONAL, true, null),
-                new AttributeDefinition(TABLE_ALIAS_LIST, DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()),
-                        Multiplicity.OPTIONAL, true, null),
-                new AttributeDefinition("columns", DataTypes.arrayTypeName(HiveDataTypes.HIVE_COLUMN.getName()),
-                        Multiplicity.OPTIONAL, true, null),
-                new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
-                new AttributeDefinition("viewOriginalText", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
-                        false, null),
-                new AttributeDefinition("viewExpandedText", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
-                        false, null),
-                new AttributeDefinition(HiveDataModelGenerator.TABLE_TYPE_ATTR, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
-                        null),
-                new AttributeDefinition("temporary", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, false,
-                        false, true, null),};
-        HierarchicalTypeDefinition<ClassType> definition =
-                new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.getName(), null,
-                    ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
-        classTypeDefinitions.put(HiveDataTypes.HIVE_TABLE.getName(), definition);
-        LOG.debug("Created definition for " + HiveDataTypes.HIVE_TABLE.getName());
-    }
-
-    private void createProcessClass() throws AtlasException {
-        AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
-                new AttributeDefinition("startTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
-                new AttributeDefinition("endTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
-                new AttributeDefinition("userName", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
-                        false, true, null),
-                new AttributeDefinition("operationType", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
-                        false, true, null),
-                new AttributeDefinition("queryText", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
-                        null),
-                new AttributeDefinition("queryPlan", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
-                        null),
-                new AttributeDefinition("queryId", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
-                new AttributeDefinition("recentQueries", String.format("array<%s>", DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL, false, null),
-                new AttributeDefinition(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
-                new AttributeDefinition("queryGraph", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
-                        null),};
-
-        HierarchicalTypeDefinition<ClassType> definition =
-                new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_PROCESS.getName(), null,
-                    ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
-        classTypeDefinitions.put(HiveDataTypes.HIVE_PROCESS.getName(), definition);
-        LOG.debug("Created definition for " + HiveDataTypes.HIVE_PROCESS.getName());
-    }
-
-    public String getModelAsJson() throws AtlasException {
-        createDataModel();
-        return getDataModelAsJSON();
-    }
-
-    public static void main(String[] args) throws Exception {
-        HiveDataModelGenerator hiveDataModelGenerator = new HiveDataModelGenerator();
-        String modelAsJson = hiveDataModelGenerator.getModelAsJson();
-
-        if (args.length==1) {
-            ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson);
-            return;
-        }
-
-        System.out.println("hiveDataModelAsJSON = " + modelAsJson);
-
-        TypesDef typesDef = hiveDataModelGenerator.getTypesDef();
-        for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) {
-            System.out.println(String.format("%s(%s) - values %s", enumType.name, EnumType.class.getSimpleName(),
-                    Arrays.toString(enumType.enumValues)));
-        }
-        for (StructTypeDefinition structType : typesDef.structTypesAsJavaList()) {
-            System.out.println(String.format("%s(%s) - attributes %s", structType.typeName, StructType.class.getSimpleName(),
-                    Arrays.toString(structType.attributeDefinitions)));
-        }
-        for (HierarchicalTypeDefinition<ClassType> classType : typesDef.classTypesAsJavaList()) {
-            System.out.println(String.format("%s(%s) - super types [%s] - attributes %s", classType.typeName, ClassType.class.getSimpleName(),
-                    StringUtils.join(classType.superTypes, ","), Arrays.toString(classType.attributeDefinitions)));
-        }
-        for (HierarchicalTypeDefinition<TraitType> traitType : typesDef.traitTypesAsJavaList()) {
-            System.out.println(String.format("%s(%s) - %s", traitType.typeName, TraitType.class.getSimpleName(),
-                    Arrays.toString(traitType.attributeDefinitions)));
-        }
-    }
-
-    private void createColumnLineageClass() throws AtlasException {
-
-        AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
-                new AttributeDefinition("query", HiveDataTypes.HIVE_PROCESS.getName(),
-                        Multiplicity.REQUIRED, false, null),
-                new AttributeDefinition("depenendencyType",DataTypes.STRING_TYPE.getName(),
-                        Multiplicity.REQUIRED, false, null),
-                new AttributeDefinition("expression",DataTypes.STRING_TYPE.getName(),
-                        Multiplicity.OPTIONAL, false, null)
-        };
-        HierarchicalTypeDefinition<ClassType> definition =
-                new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_COLUMN_LINEAGE.getName(), null,
-                        ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
-        classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN_LINEAGE.getName(), definition);
-        LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN_LINEAGE.getName());
-
-    }
-
-}
-

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
index bdf0d2d..5abf2df 100644
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
@@ -20,7 +20,6 @@ package org.apache.atlas.hive;
 
 import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasClient;
-import org.apache.atlas.fs.model.FSDataTypes;
 import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
 import org.apache.atlas.hive.hook.HiveHookIT;
 import org.apache.atlas.hive.model.HiveDataTypes;
@@ -93,7 +92,6 @@ public class HiveITBase {
         }
 
         hiveMetaStoreBridge = new HiveMetaStoreBridge(configuration, conf, atlasClient);
-        hiveMetaStoreBridge.registerHiveDataModel();
 
         HiveConf conf = new HiveConf();
         conf.set("hive.exec.post.hooks", "");
@@ -232,7 +230,7 @@ public class HiveITBase {
 
     private String assertHDFSPathIsRegistered(String path) throws Exception {
         LOG.debug("Searching for hdfs path {}", path);
-        return assertEntityIsRegistered(FSDataTypes.HDFS_PATH().toString(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, path, null);
+        return assertEntityIsRegistered(HiveMetaStoreBridge.HDFS_PATH, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, path, null);
     }
 
     protected String assertDatabaseIsRegistered(String dbName) throws Exception {

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
index e488f93..0cba27e 100644
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
@@ -20,7 +20,6 @@ package org.apache.atlas.hive.bridge;
 
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasServiceException;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
 import org.apache.atlas.hive.model.HiveDataTypes;
 import org.apache.atlas.typesystem.Referenceable;
 import org.apache.hadoop.fs.Path;
@@ -32,7 +31,6 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.Partition;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.mapred.TextInputFormat;
-import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
 import org.mockito.ArgumentMatcher;
 import org.mockito.Mock;
@@ -110,7 +108,7 @@ public class HiveMetaStoreBridgeTest {
 
         // verify update is called on table
         verify(atlasClient).updateEntity(eq("82e06b34-9151-4023-aa9d-b82103a50e77"),
-                (Referenceable) argThat(new MatchesReferenceableProperty(HiveDataModelGenerator.TABLE_TYPE_ATTR,
+                (Referenceable) argThat(new MatchesReferenceableProperty(HiveMetaStoreBridge.TABLE_TYPE_ATTR,
                         TableType.EXTERNAL_TABLE.name())));
     }
 
@@ -228,7 +226,7 @@ public class HiveMetaStoreBridgeTest {
     private Referenceable createTableReference() {
         Referenceable tableReference = new Referenceable(HiveDataTypes.HIVE_TABLE.getName());
         Referenceable sdReference = new Referenceable(HiveDataTypes.HIVE_STORAGEDESC.getName());
-        tableReference.set(HiveDataModelGenerator.STORAGE_DESC, sdReference);
+        tableReference.set(HiveMetaStoreBridge.STORAGE_DESC, sdReference);
         return tableReference;
     }
 


Mime
View raw message