atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject [3/4] incubator-atlas git commit: ATLAS-1272: updated types bootstrap to load from new format typedef JSON files
Date Tue, 08 Nov 2016 00:00:14 GMT
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index b6f55a1..40866fe 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -23,10 +23,8 @@ import com.google.common.collect.ImmutableList;
 import com.sun.jersey.api.client.ClientResponse;
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasServiceException;
-import org.apache.atlas.fs.model.FSDataTypes;
 import org.apache.atlas.hive.HiveITBase;
 import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
 import org.apache.atlas.hive.model.HiveDataTypes;
 import org.apache.atlas.typesystem.Referenceable;
 import org.apache.atlas.typesystem.Struct;
@@ -45,7 +43,6 @@ import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.plan.HiveOperation;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 import org.slf4j.Logger;
@@ -80,7 +77,7 @@ public class HiveHookIT extends HiveITBase {
         String dbId = assertDatabaseIsRegistered(dbName);
 
         Referenceable definition = atlasClient.getEntity(dbId);
-        Map params = (Map) definition.get(HiveDataModelGenerator.PARAMETERS);
+        Map params = (Map) definition.get(HiveMetaStoreBridge.PARAMETERS);
         Assert.assertNotNull(params);
         Assert.assertEquals(params.size(), 2);
         Assert.assertEquals(params.get("p1"), "v1");
@@ -150,8 +147,8 @@ public class HiveHookIT extends HiveITBase {
         Referenceable colEntity = atlasClient.getEntity(colId);
         Assert.assertEquals(colEntity.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), String.format("%s.%s.%s@%s", dbName.toLowerCase(),
                 tableName.toLowerCase(), colName.toLowerCase(), CLUSTER_NAME));
-        Assert.assertNotNull(colEntity.get(HiveDataModelGenerator.TABLE));
-        Assert.assertEquals(((Id) colEntity.get(HiveDataModelGenerator.TABLE))._getId(), tableId);
+        Assert.assertNotNull(colEntity.get(HiveMetaStoreBridge.TABLE));
+        Assert.assertEquals(((Id) colEntity.get(HiveMetaStoreBridge.TABLE))._getId(), tableId);
 
         //assert that column.owner = table.owner
         Referenceable tableRef = atlasClient.getEntity(tableId);
@@ -161,8 +158,8 @@ public class HiveHookIT extends HiveITBase {
         tableName = createTable();
         tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
         tableRef = atlasClient.getEntity(tableId);
-        Assert.assertEquals(tableRef.get(HiveDataModelGenerator.TABLE_TYPE_ATTR), TableType.MANAGED_TABLE.name());
-        Assert.assertEquals(tableRef.get(HiveDataModelGenerator.COMMENT), "table comment");
+        Assert.assertEquals(tableRef.get(HiveMetaStoreBridge.TABLE_TYPE_ATTR), TableType.MANAGED_TABLE.name());
+        Assert.assertEquals(tableRef.get(HiveMetaStoreBridge.COMMENT), "table comment");
         String entityName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName);
         Assert.assertEquals(tableRef.get(AtlasClient.NAME), tableName.toLowerCase());
         Assert.assertEquals(tableRef.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), entityName);
@@ -170,13 +167,13 @@ public class HiveHookIT extends HiveITBase {
         Table t = hiveMetaStoreBridge.hiveClient.getTable(DEFAULT_DB, tableName);
         long createTime = Long.parseLong(t.getMetadata().getProperty(hive_metastoreConstants.DDL_TIME)) * HiveMetaStoreBridge.MILLIS_CONVERT_FACTOR;
 
-        verifyTimestamps(tableRef, HiveDataModelGenerator.CREATE_TIME, createTime);
-        verifyTimestamps(tableRef, HiveDataModelGenerator.LAST_ACCESS_TIME, createTime);
+        verifyTimestamps(tableRef, HiveMetaStoreBridge.CREATE_TIME, createTime);
+        verifyTimestamps(tableRef, HiveMetaStoreBridge.LAST_ACCESS_TIME, createTime);
 
-        final Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC);
-        Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS), false);
-        Assert.assertNotNull(sdRef.get(HiveDataModelGenerator.TABLE));
-        Assert.assertEquals(((Id) sdRef.get(HiveDataModelGenerator.TABLE))._getId(), tableId);
+        final Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
+        Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_IS_STORED_AS_SUB_DIRS), false);
+        Assert.assertNotNull(sdRef.get(HiveMetaStoreBridge.TABLE));
+        Assert.assertEquals(((Id) sdRef.get(HiveMetaStoreBridge.TABLE))._getId(), tableId);
 
         //Create table where database doesn't exist, will create database instance as well
         assertDatabaseIsRegistered(DEFAULT_DB);
@@ -878,7 +875,7 @@ public class HiveHookIT extends HiveITBase {
 
         String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
         Referenceable tableEntity = atlasClient.getEntity(tableId);
-        final String createTime = (String)tableEntity.get(HiveDataModelGenerator.CREATE_TIME);
+        final String createTime = (String)tableEntity.get(HiveMetaStoreBridge.CREATE_TIME);
         Assert.assertNotNull(createTime);
 
         String columnGuid = assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), NAME));
@@ -916,10 +913,10 @@ public class HiveHookIT extends HiveITBase {
         assertTableIsRegistered(newDBName, newTableName, new AssertPredicate() {
             @Override
             public void assertOnEntity(final Referenceable entity) throws Exception {
-                Referenceable sd = ((Referenceable) entity.get(HiveDataModelGenerator.STORAGE_DESC));
-                String location = (String) sd.get(HiveDataModelGenerator.LOCATION);
+                Referenceable sd = ((Referenceable) entity.get(HiveMetaStoreBridge.STORAGE_DESC));
+                String location = (String) sd.get(HiveMetaStoreBridge.LOCATION);
                 assertTrue(location.contains(newTableName));
-                Assert.assertEquals(entity.get(HiveDataModelGenerator.CREATE_TIME), createTime);
+                Assert.assertEquals(entity.get(HiveMetaStoreBridge.CREATE_TIME), createTime);
             }
         });
     }
@@ -929,7 +926,7 @@ public class HiveHookIT extends HiveITBase {
         Referenceable tableRef = atlasClient.getEntity(tableId);
 
         //with soft delete, the deleted columns are returned as well. So, filter the deleted ones
-        List<Referenceable> columns = ((List<Referenceable>) tableRef.get(HiveDataModelGenerator.COLUMNS));
+        List<Referenceable> columns = ((List<Referenceable>) tableRef.get(HiveMetaStoreBridge.COLUMNS));
         List<Referenceable> activeColumns = new ArrayList<>();
         for (Referenceable col : columns) {
             if (col.getId().getState() == Id.EntityState.ACTIVE) {
@@ -1046,7 +1043,7 @@ public class HiveHookIT extends HiveITBase {
         assertColumnIsRegistered(newColQualifiedName, new AssertPredicate() {
             @Override
             public void assertOnEntity(Referenceable entity) throws Exception {
-                assertEquals(entity.get(HiveDataModelGenerator.COMMENT), comment);
+                assertEquals(entity.get(HiveMetaStoreBridge.COMMENT), comment);
             }
         });
 
@@ -1071,7 +1068,7 @@ public class HiveHookIT extends HiveITBase {
         assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() {
                 @Override
                 public void assertOnEntity(Referenceable entity) throws Exception {
-                    List<Referenceable> columns = (List<Referenceable>) entity.get(HiveDataModelGenerator.COLUMNS);
+                    List<Referenceable> columns = (List<Referenceable>) entity.get(HiveMetaStoreBridge.COLUMNS);
                     assertEquals(columns.get(0).get(NAME), finalNewColName);
                     assertEquals(columns.get(1).get(NAME), "id");
                 }
@@ -1099,7 +1096,7 @@ public class HiveHookIT extends HiveITBase {
         assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() {
                 @Override
                 public void assertOnEntity(Referenceable entity) throws Exception {
-                    List<Referenceable> columns = (List<Referenceable>) entity.get(HiveDataModelGenerator.COLUMNS);
+                    List<Referenceable> columns = (List<Referenceable>) entity.get(HiveMetaStoreBridge.COLUMNS);
                     assertEquals(columns.get(1).get(NAME), finalNewColName2);
                     assertEquals(columns.get(0).get(NAME), "id");
                 }
@@ -1289,8 +1286,8 @@ public class HiveHookIT extends HiveITBase {
         assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() {
             @Override
             public void assertOnEntity(Referenceable tableRef) throws Exception {
-                Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC);
-                Assert.assertEquals(new Path((String)sdRef.get(HiveDataModelGenerator.LOCATION)).toString(), new Path(testPath).toString());
+                Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
+                Assert.assertEquals(new Path((String)sdRef.get(HiveMetaStoreBridge.LOCATION)).toString(), new Path(testPath).toString());
             }
         });
 
@@ -1313,18 +1310,18 @@ public class HiveHookIT extends HiveITBase {
         assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() {
             @Override
             public void assertOnEntity(Referenceable tableRef) throws Exception {
-                Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC);
-                Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_DESC_INPUT_FMT),
+                Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
+                Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_DESC_INPUT_FMT),
                     "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
-                Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_DESC_OUTPUT_FMT),
+                Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_DESC_OUTPUT_FMT),
                     "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat");
                 Assert.assertNotNull(sdRef.get("serdeInfo"));
 
                 Struct serdeInfo = (Struct) sdRef.get("serdeInfo");
                 Assert.assertEquals(serdeInfo.get("serializationLib"), "org.apache.hadoop.hive.ql.io.orc.OrcSerde");
-                Assert.assertNotNull(serdeInfo.get(HiveDataModelGenerator.PARAMETERS));
+                Assert.assertNotNull(serdeInfo.get(HiveMetaStoreBridge.PARAMETERS));
                 Assert.assertEquals(
-                    ((Map<String, String>) serdeInfo.get(HiveDataModelGenerator.PARAMETERS))
+                    ((Map<String, String>) serdeInfo.get(HiveMetaStoreBridge.PARAMETERS))
                         .get("serialization.format"),
                     "1");
             }
@@ -1337,10 +1334,10 @@ public class HiveHookIT extends HiveITBase {
          * runCommand(query);
 
          * tableRef = atlasClient.getEntity(tableId);
-         * sdRef = (Referenceable)tableRef.get(HiveDataModelGenerator.STORAGE_DESC);
-         * Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_DESC_INPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
-         * Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_DESC_OUTPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat");
-         * Assert.assertEquals(((Map) sdRef.get(HiveDataModelGenerator.PARAMETERS)).get("orc.compress"), "ZLIB");
+         * sdRef = (Referenceable)tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
+         * Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_DESC_INPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
+         * Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_DESC_OUTPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat");
+         * Assert.assertEquals(((Map) sdRef.get(HiveMetaStoreBridge.PARAMETERS)).get("orc.compress"), "ZLIB");
          */
     }
 
@@ -1375,8 +1372,8 @@ public class HiveHookIT extends HiveITBase {
     private void verifyBucketSortingProperties(Referenceable tableRef, int numBuckets,
                                                ImmutableList<String> bucketColNames,
                                                ImmutableList<String>  sortcolNames) throws Exception {
-        Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC);
-        Assert.assertEquals(((scala.math.BigInt) sdRef.get(HiveDataModelGenerator.STORAGE_NUM_BUCKETS)).intValue(),
+        Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
+        Assert.assertEquals(((scala.math.BigInt) sdRef.get(HiveMetaStoreBridge.STORAGE_NUM_BUCKETS)).intValue(),
             numBuckets);
         Assert.assertEquals(sdRef.get("bucketCols"), bucketColNames);
 
@@ -1661,7 +1658,7 @@ public class HiveHookIT extends HiveITBase {
         assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() {
             @Override
             public void assertOnEntity(Referenceable tableRef) throws Exception {
-                Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC);
+                Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
                 Struct serdeInfo = (Struct) sdRef.get("serdeInfo");
                 Assert.assertEquals(serdeInfo.get("serializationLib"), serdeLib);
                 verifyProperties(serdeInfo, expectedProps, false);
@@ -1670,7 +1667,7 @@ public class HiveHookIT extends HiveITBase {
     }
 
     private void verifyProperties(Struct referenceable, Map<String, String> expectedProps, boolean checkIfNotExists) {
-        Map<String, String> parameters = (Map<String, String>) referenceable.get(HiveDataModelGenerator.PARAMETERS);
+        Map<String, String> parameters = (Map<String, String>) referenceable.get(HiveMetaStoreBridge.PARAMETERS);
 
         if (checkIfNotExists == false) {
             //Check if properties exist
@@ -1744,7 +1741,7 @@ public class HiveHookIT extends HiveITBase {
     }
 
     private String getDSTypeName(Entity entity) {
-        return Entity.Type.TABLE.equals(entity.getType()) ? HiveDataTypes.HIVE_TABLE.name() : FSDataTypes.HDFS_PATH().toString();
+        return Entity.Type.TABLE.equals(entity.getType()) ? HiveDataTypes.HIVE_TABLE.name() : HiveMetaStoreBridge.HDFS_PATH.toString();
     }
 
     private <T extends Entity> SortedMap<T, Referenceable> getSortedProcessDataSets(Set<T> inputTbls) {

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0010-base_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0010-base_model.json b/addons/models/0010-base_model.json
new file mode 100644
index 0000000..7f64d85
--- /dev/null
+++ b/addons/models/0010-base_model.json
@@ -0,0 +1,97 @@
+{
+    "enumDefs": [],
+    "structDefs": [],
+    "classificationDefs": [],
+    "entityDefs": [
+        {
+            "name": "Referenceable",
+            "superTypes": [],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "qualifiedName",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": true
+                }
+            ]
+        },
+        {
+            "name": "Asset",
+            "superTypes": [],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "name",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "description",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "owner",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": true,
+                    "isUnique": false
+                }
+            ]
+        },
+        {
+            "name": "DataSet",
+            "superTypes": [
+                "Referenceable",
+                "Asset"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": []
+        },
+        {
+            "name": "Infrastructure",
+            "superTypes": [
+                "Referenceable",
+                "Asset"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": []
+        },
+        {
+            "name": "Process",
+            "superTypes": [
+                "Referenceable",
+                "Asset"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "inputs",
+                    "typeName": "array<DataSet>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "outputs",
+                    "typeName": "array<DataSet>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                }
+            ]
+        }
+    ]
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0020-fs_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0020-fs_model.json b/addons/models/0020-fs_model.json
new file mode 100644
index 0000000..ff17261
--- /dev/null
+++ b/addons/models/0020-fs_model.json
@@ -0,0 +1,191 @@
+{
+    "enumDefs": [
+        {
+            "name": "file_action",
+            "typeVersion": "1.0",
+            "elementDefs": [
+                {
+                    "ordinal": 0,
+                    "value": "NONE"
+                },
+                {
+                    "ordinal": 1,
+                    "value": "EXECUTE"
+                },
+                {
+                    "ordinal": 2,
+                    "value": "WRITE"
+                },
+                {
+                    "ordinal": 3,
+                    "value": "WRITE_EXECUTE"
+                },
+                {
+                    "ordinal": 4,
+                    "value": "READ"
+                },
+                {
+                    "ordinal": 5,
+                    "value": "READ_EXECUTE"
+                },
+                {
+                    "ordinal": 6,
+                    "value": "READ_WRITE"
+                },
+                {
+                    "ordinal": 7,
+                    "value": "ALL"
+                }
+            ]
+        }
+    ],
+    "structDefs": [
+        {
+            "name": "fs_permissions",
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "group",
+                    "typeName": "file_action",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "user",
+                    "typeName": "file_action",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "sticky",
+                    "typeName": "boolean",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "others",
+                    "typeName": "file_action",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                }
+            ]
+        }
+    ],
+    "classificationDefs": [],
+    "entityDefs": [
+        {
+            "name": "fs_path",
+            "superTypes": [
+                "DataSet"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "path",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "createTime",
+                    "typeName": "date",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "modifiedTime",
+                    "typeName": "date",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "isFile",
+                    "typeName": "boolean",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "isSymlink",
+                    "typeName": "boolean",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "fileSize",
+                    "typeName": "long",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "group",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "posixPermissions",
+                    "typeName": "fs_permissions",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": true,
+                    "isUnique": false
+                }
+            ]
+        },
+        {
+            "name": "hdfs_path",
+            "superTypes": [
+                "fs_path"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "clusterName",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "numberOfReplicas",
+                    "typeName": "int",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "extendedAttributes",
+                    "typeName": "map<string,string>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                }
+            ]
+        }
+    ]
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0030-hive_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0030-hive_model.json b/addons/models/0030-hive_model.json
new file mode 100644
index 0000000..2c2e9ed
--- /dev/null
+++ b/addons/models/0030-hive_model.json
@@ -0,0 +1,498 @@
+{
+    "enumDefs": [
+        {
+            "name": "hive_principal_type",
+            "typeVersion": "1.0",
+            "elementDefs": [
+                {
+                    "ordinal": 1,
+                    "value": "USER"
+                },
+                {
+                    "ordinal": 2,
+                    "value": "ROLE"
+                },
+                {
+                    "ordinal": 3,
+                    "value": "GROUP"
+                }
+            ]
+        }
+    ],
+    "structDefs": [
+        {
+            "name": "hive_order",
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "order",
+                    "typeName": "int",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "col",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                }
+            ]
+        },
+        {
+            "name": "hive_serde",
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "name",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "serializationLib",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "parameters",
+                    "typeName": "map<string,string>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                }
+            ]
+        }
+    ],
+    "classificationDefs": [],
+    "entityDefs": [
+        {
+            "name": "hive_process",
+            "superTypes": [
+                "Process"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "startTime",
+                    "typeName": "date",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "endTime",
+                    "typeName": "date",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "userName",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "operationType",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "queryText",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "queryPlan",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "queryId",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "recentQueries",
+                    "typeName": "array<string>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "clusterName",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "queryGraph",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                }
+            ]
+        },
+        {
+            "name": "hive_table",
+            "superTypes": [
+                "DataSet"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "db",
+                    "typeName": "hive_db",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "createTime",
+                    "typeName": "date",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "lastAccessTime",
+                    "typeName": "date",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "comment",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "retention",
+                    "typeName": "int",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "sd",
+                    "typeName": "hive_storagedesc",
+                    "cardinality": "SINGLE",
+                    "constraintDefs": [
+                        {
+                            "type": "mappedFromRef",
+                            "params": {
+                                "refAttribute": "table"
+                            }
+                        }
+                    ],
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "partitionKeys",
+                    "typeName": "array<hive_column>",
+                    "cardinality": "SINGLE",
+                    "constraintDefs": [
+                        {
+                            "type": "foreignKey"
+                        }
+                    ],
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "aliases",
+                    "typeName": "array<string>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "columns",
+                    "typeName": "array<hive_column>",
+                    "cardinality": "SINGLE",
+                    "constraintDefs": [
+                        {
+                            "type": "mappedFromRef",
+                            "params": {
+                                "refAttribute": "table"
+                            }
+                        }
+                    ],
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "parameters",
+                    "typeName": "map<string,string>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "viewOriginalText",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "viewExpandedText",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "tableType",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "temporary",
+                    "typeName": "boolean",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": true,
+                    "isUnique": false
+                }
+            ]
+        },
+        {
+            "name": "hive_storagedesc",
+            "superTypes": [
+                "Referenceable"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "table",
+                    "typeName": "hive_table",
+                    "cardinality": "SINGLE",
+                    "constraintDefs": [
+                        {
+                            "type": "foreignKey",
+                            "params": {
+                                "onDelete": "cascade"
+                            }
+                        }
+                    ],
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "location",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "inputFormat",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "outputFormat",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "compressed",
+                    "typeName": "boolean",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "numBuckets",
+                    "typeName": "int",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "serdeInfo",
+                    "typeName": "hive_serde",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "bucketCols",
+                    "typeName": "array<string>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "sortCols",
+                    "typeName": "array<hive_order>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "parameters",
+                    "typeName": "map<string,string>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "storedAsSubDirectories",
+                    "typeName": "boolean",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                }
+            ]
+        },
+        {
+            "name": "hive_db",
+            "superTypes": [
+                "Referenceable",
+                "Asset"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "clusterName",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "location",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "parameters",
+                    "typeName": "map<string,string>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "ownerType",
+                    "typeName": "hive_principal_type",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                }
+            ]
+        },
+        {
+            "name": "hive_column",
+            "superTypes": [
+                "DataSet"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "type",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "comment",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "table",
+                    "typeName": "hive_table",
+                    "cardinality": "SINGLE",
+                    "constraintDefs": [
+                        {
+                            "type": "foreignKey",
+                            "params": {
+                                "onDelete": "cascade"
+                            }
+                        }
+                    ],
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                }
+            ]
+        }
+    ]
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0040-sqoop_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0040-sqoop_model.json b/addons/models/0040-sqoop_model.json
new file mode 100644
index 0000000..f5c7fd9
--- /dev/null
+++ b/addons/models/0040-sqoop_model.json
@@ -0,0 +1,97 @@
+{
+    "enumDefs": [],
+    "structDefs": [],
+    "classificationDefs": [],
+    "entityDefs": [
+        {
+            "name": "sqoop_process",
+            "superTypes": [
+                "Process"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "operation",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "commandlineOpts",
+                    "typeName": "map<string,string>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "startTime",
+                    "typeName": "date",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "endTime",
+                    "typeName": "date",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "userName",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": true,
+                    "isUnique": false
+                }
+            ]
+        },
+        {
+            "name": "sqoop_dbdatastore",
+            "superTypes": [
+                "DataSet"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "dbStoreType",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "storeUse",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "storeUri",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "source",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": true,
+                    "isUnique": false
+                }
+            ]
+        }
+    ]
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0050-falcon_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0050-falcon_model.json b/addons/models/0050-falcon_model.json
new file mode 100644
index 0000000..b7398ef
--- /dev/null
+++ b/addons/models/0050-falcon_model.json
@@ -0,0 +1,147 @@
+{
+    "enumDefs": [],
+    "structDefs": [],
+    "classificationDefs": [],
+    "entityDefs": [
+        {
+            "name": "falcon_feed_replication",
+            "superTypes": [
+                "Process"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": []
+        },
+        {
+            "name": "falcon_cluster",
+            "superTypes": [
+                "Infrastructure"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "colo",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "tags",
+                    "typeName": "map<string,string>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                }
+            ]
+        },
+        {
+            "name": "falcon_feed",
+            "superTypes": [
+                "DataSet"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "frequency",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "stored-in",
+                    "typeName": "falcon_cluster",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "groups",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "tags",
+                    "typeName": "map<string,string>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                }
+            ]
+        },
+        {
+            "name": "falcon_process",
+            "superTypes": [
+                "Process"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "frequency",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "runs-on",
+                    "typeName": "falcon_cluster",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "tags",
+                    "typeName": "map<string,string>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "pipelines",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "workflow-properties",
+                    "typeName": "map<string,string>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                }
+            ]
+        },
+        {
+            "name": "falcon_feed_creation",
+            "superTypes": [
+                "Process"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "stored-in",
+                    "typeName": "falcon_cluster",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                }
+            ]
+        }
+    ]
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0060-hbase_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0060-hbase_model.json b/addons/models/0060-hbase_model.json
new file mode 100644
index 0000000..d249d87
--- /dev/null
+++ b/addons/models/0060-hbase_model.json
@@ -0,0 +1,24 @@
+{
+    "enumDefs": [],
+    "structDefs": [],
+    "classificationDefs": [],
+    "entityDefs": [
+        {
+            "name": "hbase_table",
+            "superTypes": [
+                "DataSet"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "uri",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                }
+            ]
+        }
+    ]
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0070-kafka_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0070-kafka_model.json b/addons/models/0070-kafka_model.json
new file mode 100644
index 0000000..b7f6e33
--- /dev/null
+++ b/addons/models/0070-kafka_model.json
@@ -0,0 +1,57 @@
+{
+    "enumDefs": [],
+    "structDefs": [],
+    "classificationDefs": [],
+    "entityDefs": [
+        {
+            "name": "kafka_topic",
+            "superTypes": [
+                "DataSet"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "topic",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": true
+                },
+                {
+                    "name": "uri",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                }
+            ]
+        },
+        {
+            "name": "jms_topic",
+            "superTypes": [
+                "DataSet"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "topic",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": true
+                },
+                {
+                    "name": "uri",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                }
+            ]
+        }
+    ]
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0080-storm_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0080-storm_model.json b/addons/models/0080-storm_model.json
new file mode 100644
index 0000000..59f3228
--- /dev/null
+++ b/addons/models/0080-storm_model.json
@@ -0,0 +1,150 @@
+{
+    "enumDefs": [],
+    "structDefs": [],
+    "classificationDefs": [],
+    "entityDefs": [
+        {
+            "name": "storm_topology",
+            "superTypes": [
+                "Process"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "id",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": true
+                },
+                {
+                    "name": "startTime",
+                    "typeName": "date",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "endTime",
+                    "typeName": "date",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "conf",
+                    "typeName": "map<string,string>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "clusterName",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "nodes",
+                    "typeName": "array<storm_node>",
+                    "cardinality": "LIST",
+                    "constraintDefs": [
+                        {
+                            "type": "foreignKey"
+                        }
+                    ],
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                }
+            ]
+        },
+        {
+            "name": "storm_node",
+            "superTypes": [],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "name",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "description",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": true,
+                    "isUnique": false
+                },
+                {
+                    "name": "driverClass",
+                    "typeName": "string",
+                    "cardinality": "SINGLE",
+                    "isIndexable": true,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "conf",
+                    "typeName": "map<string,string>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                }
+            ]
+        },
+        {
+            "name": "storm_spout",
+            "superTypes": [
+                "storm_node"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "outputs",
+                    "typeName": "array<string>",
+                    "cardinality": "LIST",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                }
+            ]
+        },
+        {
+            "name": "storm_bolt",
+            "superTypes": [
+                "storm_node"
+            ],
+            "typeVersion": "1.0",
+            "attributeDefs": [
+                {
+                    "name": "inputs",
+                    "typeName": "array<string>",
+                    "cardinality": "LIST",
+                    "isIndexable": false,
+                    "isOptional": false,
+                    "isUnique": false
+                },
+                {
+                    "name": "outputs",
+                    "typeName": "array<string>",
+                    "cardinality": "SINGLE",
+                    "isIndexable": false,
+                    "isOptional": true,
+                    "isUnique": false
+                }
+            ]
+        }
+    ]
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/patches/001-hive_column_add_position.json
----------------------------------------------------------------------
diff --git a/addons/models/patches/001-hive_column_add_position.json b/addons/models/patches/001-hive_column_add_position.json
new file mode 100644
index 0000000..eeab74a
--- /dev/null
+++ b/addons/models/patches/001-hive_column_add_position.json
@@ -0,0 +1,21 @@
+{
+  "patches": [
+    {
+      "action": "ADD_ATTRIBUTE",
+      "typeName": "hive_column",
+      "applyToVersion": "1.0",
+      "updateToVersion": "1.1",
+      "params": null,
+      "attributeDefs": [
+        {
+          "name": "position",
+          "typeName": "int",
+          "cardinality": "SINGLE",
+          "isIndexable": false,
+          "isOptional": true,
+          "isUnique": false
+        }
+      ]
+    }
+  ]
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/sqoop-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/pom.xml b/addons/sqoop-bridge/pom.xml
index 9ac411e..e71d64a 100644
--- a/addons/sqoop-bridge/pom.xml
+++ b/addons/sqoop-bridge/pom.xml
@@ -411,24 +411,6 @@
                 <version>1.2.1</version>
                 <inherited>false</inherited>
                 <executions>
-                    <execution>
-                        <configuration>
-                            <mainClass>org.apache.atlas.sqoop.model.SqoopDataModelGenerator</mainClass>
-                            <systemProperties>
-                                <systemProperty>
-                                    <key>atlas.conf</key>
-                                    <value>${project.build.directory}/../../../typesystem/target/test-classes</value>
-                                </systemProperty>
-                            </systemProperties>
-                            <arguments>
-                                <argument>${project.build.directory}/models/sqoop_model.json</argument>
-                            </arguments>
-                        </configuration>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>java</goal>
-                        </goals>
-                    </execution>
                 </executions>
             </plugin>
         </plugins>

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
index e43fc86..6fb27e5 100644
--- a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
+++ b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
@@ -23,11 +23,9 @@ import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasConstants;
 import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
 import org.apache.atlas.hive.model.HiveDataTypes;
 import org.apache.atlas.hook.AtlasHook;
 import org.apache.atlas.notification.hook.HookNotification;
-import org.apache.atlas.sqoop.model.SqoopDataModelGenerator;
 import org.apache.atlas.sqoop.model.SqoopDataTypes;
 import org.apache.atlas.typesystem.Referenceable;
 import org.apache.commons.configuration.Configuration;
@@ -55,6 +53,20 @@ public class SqoopHook extends SqoopJobDataPublisher {
     public static final String ATLAS_CLUSTER_NAME = "atlas.cluster.name";
     public static final String DEFAULT_CLUSTER_NAME = "primary";
 
+    public static final String USER = "userName";
+    public static final String DB_STORE_TYPE = "dbStoreType";
+    public static final String DB_STORE_USAGE = "storeUse";
+    public static final String SOURCE = "source";
+    public static final String DESCRIPTION = "description";
+    public static final String STORE_URI = "storeUri";
+    public static final String OPERATION = "operation";
+    public static final String START_TIME = "startTime";
+    public static final String END_TIME = "endTime";
+    public static final String CMD_LINE_OPTS = "commandlineOpts";
+    // multiple inputs and outputs for process
+    public static final String INPUTS = "inputs";
+    public static final String OUTPUTS = "outputs";
+
     static {
         org.apache.hadoop.conf.Configuration.addDefaultResource("sqoop-site.xml");
     }
@@ -75,7 +87,7 @@ public class SqoopHook extends SqoopJobDataPublisher {
         tableRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
                 HiveMetaStoreBridge.getTableQualifiedName(clusterName, dbName, tableName));
         tableRef.set(AtlasClient.NAME, tableName.toLowerCase());
-        tableRef.set(HiveDataModelGenerator.DB, dbRef);
+        tableRef.set(HiveMetaStoreBridge.DB, dbRef);
         return tableRef;
     }
 
@@ -94,11 +106,11 @@ public class SqoopHook extends SqoopJobDataPublisher {
         String name = getSqoopDBStoreName(data);
         storeRef.set(AtlasClient.NAME, name);
         storeRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
-        storeRef.set(SqoopDataModelGenerator.DB_STORE_TYPE, data.getStoreType());
-        storeRef.set(SqoopDataModelGenerator.DB_STORE_USAGE, usage);
-        storeRef.set(SqoopDataModelGenerator.STORE_URI, data.getUrl());
-        storeRef.set(SqoopDataModelGenerator.SOURCE, source);
-        storeRef.set(SqoopDataModelGenerator.DESCRIPTION, "");
+        storeRef.set(SqoopHook.DB_STORE_TYPE, data.getStoreType());
+        storeRef.set(SqoopHook.DB_STORE_USAGE, usage);
+        storeRef.set(SqoopHook.STORE_URI, data.getUrl());
+        storeRef.set(SqoopHook.SOURCE, source);
+        storeRef.set(SqoopHook.DESCRIPTION, "");
         storeRef.set(AtlasClient.OWNER, data.getUser());
         return storeRef;
     }
@@ -109,24 +121,24 @@ public class SqoopHook extends SqoopJobDataPublisher {
         final String sqoopProcessName = getSqoopProcessName(data, clusterName);
         procRef.set(AtlasClient.NAME, sqoopProcessName);
         procRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, sqoopProcessName);
-        procRef.set(SqoopDataModelGenerator.OPERATION, data.getOperation());
+        procRef.set(SqoopHook.OPERATION, data.getOperation());
         if (isImportOperation(data)) {
-            procRef.set(SqoopDataModelGenerator.INPUTS, dbStoreRef);
-            procRef.set(SqoopDataModelGenerator.OUTPUTS, hiveTableRef);
+            procRef.set(SqoopHook.INPUTS, dbStoreRef);
+            procRef.set(SqoopHook.OUTPUTS, hiveTableRef);
         } else {
-            procRef.set(SqoopDataModelGenerator.INPUTS, hiveTableRef);
-            procRef.set(SqoopDataModelGenerator.OUTPUTS, dbStoreRef);
+            procRef.set(SqoopHook.INPUTS, hiveTableRef);
+            procRef.set(SqoopHook.OUTPUTS, dbStoreRef);
         }
-        procRef.set(SqoopDataModelGenerator.USER, data.getUser());
-        procRef.set(SqoopDataModelGenerator.START_TIME, new Date(data.getStartTime()));
-        procRef.set(SqoopDataModelGenerator.END_TIME, new Date(data.getEndTime()));
+        procRef.set(SqoopHook.USER, data.getUser());
+        procRef.set(SqoopHook.START_TIME, new Date(data.getStartTime()));
+        procRef.set(SqoopHook.END_TIME, new Date(data.getEndTime()));
 
         Map<String, String> sqoopOptionsMap = new HashMap<>();
         Properties options = data.getOptions();
         for (Object k : options.keySet()) {
             sqoopOptionsMap.put((String)k, (String) options.get(k));
         }
-        procRef.set(SqoopDataModelGenerator.CMD_LINE_OPTS, sqoopOptionsMap);
+        procRef.set(SqoopHook.CMD_LINE_OPTS, sqoopOptionsMap);
 
         return procRef;
     }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java
deleted file mode 100644
index 3373246..0000000
--- a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.sqoop.model;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-
-import org.apache.atlas.AtlasClient;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.addons.ModelDefinitionDump;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.AttributeDefinition;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.EnumType;
-import org.apache.atlas.typesystem.types.EnumTypeDefinition;
-import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.StructTypeDefinition;
-import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Utility that generates Sqoop data model for both metastore entities and DDL/DML queries.
- */
-public class SqoopDataModelGenerator {
-
-    private static final Logger LOG = LoggerFactory.getLogger(SqoopDataModelGenerator.class);
-
-    private final Map<String, HierarchicalTypeDefinition<ClassType>> classTypeDefinitions;
-    private final Map<String, EnumTypeDefinition> enumTypeDefinitionMap;
-    private final Map<String, StructTypeDefinition> structTypeDefinitionMap;
-    private static final DataTypes.MapType STRING_MAP_TYPE =
-            new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE);
-
-    public static final String USER = "userName";
-    public static final String DB_STORE_TYPE = "dbStoreType";
-    public static final String DB_STORE_USAGE = "storeUse";
-    public static final String SOURCE = "source";
-    public static final String DESCRIPTION = "description";
-    public static final String STORE_URI = "storeUri";
-    public static final String OPERATION = "operation";
-    public static final String START_TIME = "startTime";
-    public static final String END_TIME = "endTime";
-    public static final String CMD_LINE_OPTS = "commandlineOpts";
-    // multiple inputs and outputs for process
-    public static final String INPUTS = "inputs";
-    public static final String OUTPUTS = "outputs";
-
-    public SqoopDataModelGenerator() {
-        classTypeDefinitions = new HashMap<>();
-        enumTypeDefinitionMap = new HashMap<>();
-        structTypeDefinitionMap = new HashMap<>();
-    }
-
-    public void createDataModel() throws AtlasException {
-        LOG.info("Generating the Sqoop Data Model....");
-
-        // enums
-
-        // structs
-
-        // classes
-        createSqoopDbStoreClass();
-
-        // DDL/DML Process
-        createSqoopProcessClass();
-    }
-
-    public TypesDef getTypesDef() {
-        return TypesUtil.getTypesDef(getEnumTypeDefinitions(), getStructTypeDefinitions(), getTraitTypeDefinitions(),
-                getClassTypeDefinitions());
-    }
-
-    public String getDataModelAsJSON() {
-        return TypesSerialization.toJson(getTypesDef());
-    }
-
-    public ImmutableList<EnumTypeDefinition> getEnumTypeDefinitions() {
-        return ImmutableList.copyOf(enumTypeDefinitionMap.values());
-    }
-
-    public ImmutableList<StructTypeDefinition> getStructTypeDefinitions() {
-        return ImmutableList.copyOf(structTypeDefinitionMap.values());
-    }
-
-    public ImmutableList<HierarchicalTypeDefinition<ClassType>> getClassTypeDefinitions() {
-        return ImmutableList.copyOf(classTypeDefinitions.values());
-    }
-
-    public ImmutableList<HierarchicalTypeDefinition<TraitType>> getTraitTypeDefinitions() {
-        return ImmutableList.of();
-    }
-
-    private void createSqoopDbStoreClass() throws AtlasException {
-        AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
-                new AttributeDefinition(DB_STORE_TYPE,
-                        DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
-                new AttributeDefinition(DB_STORE_USAGE,
-                        DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
-                new AttributeDefinition(STORE_URI,
-                        DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
-                new AttributeDefinition(SOURCE,
-                        DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null)
-        };
-
-        HierarchicalTypeDefinition<ClassType> definition =
-                new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_DBDATASTORE.getName(), null,
-                    ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
-        classTypeDefinitions.put(SqoopDataTypes.SQOOP_DBDATASTORE.getName(), definition);
-        LOG.debug("Created definition for " + SqoopDataTypes.SQOOP_DBDATASTORE.getName());
-    }
-
-
-    private void createSqoopProcessClass() throws AtlasException {
-        AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
-                new AttributeDefinition(OPERATION,
-                        DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
-                new AttributeDefinition(CMD_LINE_OPTS, STRING_MAP_TYPE.getName(), Multiplicity.REQUIRED, false, null),
-                new AttributeDefinition(START_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
-                new AttributeDefinition(END_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
-                new AttributeDefinition(USER,
-                        DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null),
-        };
-
-        HierarchicalTypeDefinition<ClassType> definition =
-                new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_PROCESS.getName(), null,
-                    ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
-        classTypeDefinitions.put(SqoopDataTypes.SQOOP_PROCESS.getName(), definition);
-        LOG.debug("Created definition for " + SqoopDataTypes.SQOOP_PROCESS.getName());
-    }
-
-    public String getModelAsJson() throws AtlasException {
-        createDataModel();
-        return getDataModelAsJSON();
-    }
-
-    public static void main(String[] args) throws Exception {
-        SqoopDataModelGenerator dataModelGenerator = new SqoopDataModelGenerator();
-        String modelAsJson = dataModelGenerator.getModelAsJson();
-
-        if (args.length == 1) {
-            ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson);
-            return;
-        }
-
-        System.out.println("sqoopDataModelAsJSON = " + modelAsJson);
-
-        TypesDef typesDef = dataModelGenerator.getTypesDef();
-        for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) {
-            System.out.println(String.format("%s(%s) - values %s", enumType.name, EnumType.class.getSimpleName(),
-                    Arrays.toString(enumType.enumValues)));
-        }
-
-        for (HierarchicalTypeDefinition<ClassType> classType : typesDef.classTypesAsJavaList()) {
-            System.out.println(
-                    String.format("%s(%s) - super types [%s] - attributes %s", classType.typeName,
-                            ClassType.class.getSimpleName(), StringUtils.join(classType.superTypes, ","),
-                            Arrays.toString(classType.attributeDefinitions)));
-        }
-
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
index d4fb3af..b1bd115 100644
--- a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
+++ b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
@@ -18,17 +18,13 @@
 
 package org.apache.atlas.sqoop.hook;
 
-import com.sun.jersey.api.client.ClientResponse;
 import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasClient;
-import org.apache.atlas.AtlasServiceException;
 import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
 import org.apache.atlas.hive.model.HiveDataTypes;
-import org.apache.atlas.sqoop.model.SqoopDataModelGenerator;
 import org.apache.atlas.sqoop.model.SqoopDataTypes;
 import org.apache.atlas.utils.AuthenticationUtil;
 import org.apache.commons.configuration.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.sqoop.SqoopJobDataPublisher;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONObject;
@@ -54,28 +50,6 @@ public class SqoopHookIT {
         } else {
             atlasClient = new AtlasClient(configuration.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT));
         }
-        registerDataModels(atlasClient);
-    }
-
-    private void registerDataModels(AtlasClient client) throws Exception {
-        // Make sure hive model exists
-        HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(ApplicationProperties.get(), new HiveConf(), atlasClient);
-        hiveMetaStoreBridge.registerHiveDataModel();
-        SqoopDataModelGenerator dataModelGenerator = new SqoopDataModelGenerator();
-
-        //Register sqoop data model if its not already registered
-        try {
-            client.getType(SqoopDataTypes.SQOOP_PROCESS.getName());
-            LOG.info("Sqoop data model is already registered!");
-        } catch(AtlasServiceException ase) {
-            if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) {
-                //Expected in case types do not exist
-                LOG.info("Registering Sqoop data model");
-                client.createType(dataModelGenerator.getModelAsJson());
-            } else {
-                throw ase;
-            }
-        }
     }
 
     @Test

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/storm-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/storm-bridge/pom.xml b/addons/storm-bridge/pom.xml
index fcb6e01..6be55b4 100644
--- a/addons/storm-bridge/pom.xml
+++ b/addons/storm-bridge/pom.xml
@@ -529,24 +529,6 @@
                 <version>1.2.1</version>
                 <inherited>false</inherited>
                 <executions>
-                    <execution>
-                        <configuration>
-                            <mainClass>org.apache.atlas.storm.model.StormDataModelGenerator</mainClass>
-                            <systemProperties>
-                                <systemProperty>
-                                    <key>atlas.conf</key>
-                                    <value>${project.build.directory}/../../../typesystem/target/test-classes</value>
-                                </systemProperty>
-                            </systemProperties>
-                            <arguments>
-                                <argument>${project.build.directory}/models/storm_model.json</argument>
-                            </arguments>
-                        </configuration>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>java</goal>
-                        </goals>
-                    </execution>
                 </executions>
             </plugin>
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
----------------------------------------------------------------------
diff --git a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
index bb6a476..5193832 100644
--- a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
+++ b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
@@ -26,9 +26,7 @@ import org.apache.storm.generated.TopologyInfo;
 import org.apache.storm.utils.Utils;
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasConstants;
-import org.apache.atlas.fs.model.FSDataTypes;
 import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
 import org.apache.atlas.hook.AtlasHook;
 import org.apache.atlas.storm.model.StormDataTypes;
 import org.apache.atlas.typesystem.Referenceable;
@@ -213,7 +211,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook {
                 break;
 
             case "HdfsBolt":
-                dataSetReferenceable = new Referenceable(FSDataTypes.HDFS_PATH().toString());
+                dataSetReferenceable = new Referenceable(HiveMetaStoreBridge.HDFS_PATH);
                 String hdfsUri = config.get("HdfsBolt.rotationActions") == null
                         ? config.get("HdfsBolt.fileNameFormat.path")
                         : config.get("HdfsBolt.rotationActions");
@@ -241,7 +239,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook {
                 final String tableQualifiedName = HiveMetaStoreBridge.getTableQualifiedName(clusterName,
                         databaseName, hiveTableName);
                 dataSetReferenceable.set(AtlasClient.NAME, hiveTableName);
-                dataSetReferenceable.set(HiveDataModelGenerator.DB, dbReferenceable);
+                dataSetReferenceable.set(HiveMetaStoreBridge.DB, dbReferenceable);
                 dataSetReferenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableQualifiedName);
                 break;
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java
deleted file mode 100644
index 95eead7..0000000
--- a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.storm.model;
-
-import org.apache.atlas.addons.ModelDefinitionDump;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-
-import java.io.IOException;
-
-public class StormDataModelGenerator {
-
-    public static void main(String[] args) throws IOException {
-        StormDataModel.main(new String[]{});
-        TypesDef typesDef = StormDataModel.typesDef();
-        String stormTypesAsJSON = TypesSerialization.toJson(typesDef);
-
-        if (args.length == 1) {
-            ModelDefinitionDump.dumpModelToFile(args[0], stormTypesAsJSON);
-            return;
-        }
-
-        System.out.println("stormTypesAsJSON = " + stormTypesAsJSON);
-    }
-}


Mime
View raw message