atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jma...@apache.org
Subject [10/51] [abbrv] incubator-atlas git commit: Refactor packages and scripts to Atlas (cherry picked from commit 414beba)
Date Sun, 14 Jun 2015 17:44:50 GMT
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/atlas/typesystem/json/TypesSerializationTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/atlas/typesystem/json/TypesSerializationTest.scala b/typesystem/src/test/scala/org/apache/atlas/typesystem/json/TypesSerializationTest.scala
new file mode 100755
index 0000000..ba4782c
--- /dev/null
+++ b/typesystem/src/test/scala/org/apache/atlas/typesystem/json/TypesSerializationTest.scala
@@ -0,0 +1,300 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.json
+
+import com.google.common.collect.ImmutableList
+import org.apache.atlas.typesystem.types._
+import org.junit.{Assert, Test}
+
+class TypesSerializationTest extends BaseTest with TypeHelpers {
+
+    @Test def test1: Unit = {
+
+        val ts = getTypeSystem
+
+        val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
+            optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+            optionalAttr("c", DataTypes.BYTE_TYPE),
+            optionalAttr("d", DataTypes.SHORT_TYPE),
+            optionalAttr("e", DataTypes.INT_TYPE),
+            optionalAttr("f", DataTypes.INT_TYPE),
+            optionalAttr("g", DataTypes.LONG_TYPE),
+            optionalAttr("h", DataTypes.FLOAT_TYPE),
+            optionalAttr("i", DataTypes.DOUBLE_TYPE),
+            optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
+            optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
+            optionalAttr("l", DataTypes.DATE_TYPE),
+            optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
+            optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
+            optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
+
+
+        ts.defineTypes(ImmutableList.of[StructTypeDefinition](sDef),
+            ImmutableList.of[HierarchicalTypeDefinition[TraitType]],
+            ImmutableList.of[HierarchicalTypeDefinition[ClassType]]
+        )
+
+        val A: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("A", List(),
+            requiredAttr("a", DataTypes.INT_TYPE),
+            optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+            optionalAttr("c", DataTypes.BYTE_TYPE),
+            optionalAttr("d", DataTypes.SHORT_TYPE))
+        val B: HierarchicalTypeDefinition[TraitType] =
+            createTraitTypeDef("B", Seq("A"), optionalAttr("b", DataTypes.BOOLEAN_TYPE))
+        val C: HierarchicalTypeDefinition[TraitType] =
+            createTraitTypeDef("C", Seq("A"), optionalAttr("c", DataTypes.BYTE_TYPE))
+        val D: HierarchicalTypeDefinition[TraitType] =
+            createTraitTypeDef("D", Seq("B", "C"), optionalAttr("d", DataTypes.SHORT_TYPE))
+
+        defineTraits(ts, A, B, C, D)
+
+        ts.defineEnumType("HiveObjectType",
+            new EnumValue("GLOBAL", 1),
+            new EnumValue("DATABASE", 2),
+            new EnumValue("TABLE", 3),
+            new EnumValue("PARTITION", 4),
+            new EnumValue("COLUMN", 5))
+
+        ts.defineEnumType("PrincipalType",
+            new EnumValue("USER", 1),
+            new EnumValue("ROLE", 2),
+            new EnumValue("GROUP", 3))
+
+        ts.defineEnumType("TxnState",
+            new EnumValue("COMMITTED", 1),
+            new EnumValue("ABORTED", 2),
+            new EnumValue("OPEN", 3))
+
+        ts.defineEnumType("LockLevel",
+            new EnumValue("DB", 1),
+            new EnumValue("TABLE", 2),
+            new EnumValue("PARTITION", 3))
+
+        defineClassType(ts, createClassTypeDef("t4", List(),
+            requiredAttr("a", DataTypes.INT_TYPE),
+            optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+            optionalAttr("c", DataTypes.BYTE_TYPE),
+            optionalAttr("d", DataTypes.SHORT_TYPE),
+            optionalAttr("enum1", ts.getDataType(classOf[EnumType], "HiveObjectType")),
+            optionalAttr("e", DataTypes.INT_TYPE),
+            optionalAttr("f", DataTypes.INT_TYPE),
+            optionalAttr("g", DataTypes.LONG_TYPE),
+            optionalAttr("enum2", ts.getDataType(classOf[EnumType], "PrincipalType")),
+            optionalAttr("h", DataTypes.FLOAT_TYPE),
+            optionalAttr("i", DataTypes.DOUBLE_TYPE),
+            optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
+            optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
+            optionalAttr("enum3", ts.getDataType(classOf[EnumType], "TxnState")),
+            optionalAttr("l", DataTypes.DATE_TYPE),
+            optionalAttr("m", ts.defineArrayType(DataTypes.INT_TYPE)),
+            optionalAttr("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
+            optionalAttr("o", ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
+            optionalAttr("enum4", ts.getDataType(classOf[EnumType], "LockLevel"))))
+
+        val deptTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Department", List(),
+            requiredAttr("name", DataTypes.STRING_TYPE),
+            new AttributeDefinition("employees", String.format("array<%s>", "Person"),
+                Multiplicity.COLLECTION, true, "department"))
+        val personTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Person", List(),
+            requiredAttr("name", DataTypes.STRING_TYPE),
+            new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
+            new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates")
+        )
+        val managerTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Manager", List("Person"),
+            new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
+                Multiplicity.COLLECTION, false, "manager")
+        )
+        val securityClearanceTypeDef: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("SecurityClearance", List(),
+            requiredAttr("level", DataTypes.INT_TYPE)
+        )
+        ts.defineTypes(ImmutableList.of[StructTypeDefinition],
+            ImmutableList.of[HierarchicalTypeDefinition[TraitType]](securityClearanceTypeDef),
+            ImmutableList.of[HierarchicalTypeDefinition[ClassType]](deptTypeDef, personTypeDef, managerTypeDef))
+
+        val ser = TypesSerialization.toJson(ts, _ => true)
+
+        val typesDef1 = TypesSerialization.fromJson(ser)
+
+        val ts1 = TypeSystem.getInstance()
+        ts1.reset()
+
+        typesDef1.enumTypes.foreach(ts1.defineEnumType(_))
+
+        ts1.defineTypes(ImmutableList.copyOf(typesDef1.structTypes.toArray),
+            ImmutableList.copyOf(typesDef1.traitTypes.toArray),
+            ImmutableList.copyOf(typesDef1.classTypes.toArray)
+        )
+        val ser2 = TypesSerialization.toJson(ts1, _ => true)
+        val typesDef2 = TypesSerialization.fromJson(ser2)
+
+        Assert.assertEquals(typesDef1, typesDef2)
+    }
+
+  @Test def test2: Unit = {
+
+    val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
+      optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+      optionalAttr("c", DataTypes.BYTE_TYPE),
+      optionalAttr("d", DataTypes.SHORT_TYPE),
+      optionalAttr("e", DataTypes.INT_TYPE),
+      optionalAttr("f", DataTypes.INT_TYPE),
+      optionalAttr("g", DataTypes.LONG_TYPE),
+      optionalAttr("h", DataTypes.FLOAT_TYPE),
+      optionalAttr("i", DataTypes.DOUBLE_TYPE),
+      optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
+      optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
+      optionalAttr("l", DataTypes.DATE_TYPE),
+      optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
+      optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
+      optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
+
+
+
+    val ser2 = TypesSerialization.toJson(sDef)
+    val typesDef2 = TypesSerialization.fromJson(ser2)
+
+    Assert.assertEquals(sDef, typesDef2.structTypes(0))
+  }
+
+  @Test def test3: Unit = {
+
+    val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
+      optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+      optionalAttr("c", DataTypes.BYTE_TYPE),
+      optionalAttr("d", DataTypes.SHORT_TYPE),
+      optionalAttr("e", DataTypes.INT_TYPE),
+      optionalAttr("f", DataTypes.INT_TYPE),
+      optionalAttr("g", DataTypes.LONG_TYPE),
+      optionalAttr("h", DataTypes.FLOAT_TYPE),
+      optionalAttr("i", DataTypes.DOUBLE_TYPE),
+      optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
+      optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
+      optionalAttr("l", DataTypes.DATE_TYPE),
+      optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
+      optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
+      optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
+
+
+
+    val ser2 = TypesSerialization.toJson(sDef)
+    val typesDef2 = TypesSerialization.fromJson(ser2)
+
+    Assert.assertEquals(sDef, typesDef2.structTypes(0))
+  }
+
+  @Test def test4 : Unit = {
+
+    val A: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("A", List(),
+      requiredAttr("a", DataTypes.INT_TYPE),
+      optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+      optionalAttr("c", DataTypes.BYTE_TYPE),
+      optionalAttr("d", DataTypes.SHORT_TYPE))
+    val B: HierarchicalTypeDefinition[TraitType] =
+      createTraitTypeDef("B", Seq("A"), optionalAttr("b", DataTypes.BOOLEAN_TYPE))
+    val C: HierarchicalTypeDefinition[TraitType] =
+      createTraitTypeDef("C", Seq("A"), optionalAttr("c", DataTypes.BYTE_TYPE))
+    val D: HierarchicalTypeDefinition[TraitType] =
+      createTraitTypeDef("D", Seq("B", "C"), optionalAttr("d", DataTypes.SHORT_TYPE))
+
+    val typDefs = Seq(A,B,C,D)
+    typDefs.foreach { tDef =>
+      val ser2 = TypesSerialization.toJson(tDef, true)
+      val typesDef2 = TypesSerialization.fromJson(ser2)
+      Assert.assertEquals(tDef, typesDef2.traitTypes(0))
+
+    }
+  }
+
+  @Test def test5 : Unit = {
+    val e1 = new EnumTypeDefinition("HiveObjectType",
+      new EnumValue("GLOBAL", 1),
+      new EnumValue("DATABASE", 2),
+      new EnumValue("TABLE", 3),
+      new EnumValue("PARTITION", 4),
+      new EnumValue("COLUMN", 5))
+
+    val e2 = new EnumTypeDefinition("PrincipalType",
+      new EnumValue("USER", 1),
+      new EnumValue("ROLE", 2),
+      new EnumValue("GROUP", 3))
+
+    val e3 = new EnumTypeDefinition("TxnState",
+      new EnumValue("COMMITTED", 1),
+      new EnumValue("ABORTED", 2),
+      new EnumValue("OPEN", 3))
+
+    val e4 = new EnumTypeDefinition("LockLevel",
+      new EnumValue("DB", 1),
+      new EnumValue("TABLE", 2),
+      new EnumValue("PARTITION", 3))
+
+    val typDefs = Seq(e1,e2,e3,e4)
+    typDefs.foreach { tDef =>
+      val ser2 = TypesSerialization.toJson(tDef)
+      val typesDef2 = TypesSerialization.fromJson(ser2)
+      Assert.assertEquals(tDef, typesDef2.enumTypes(0))
+
+    }
+  }
+
+  @Test def test6 : Unit = {
+    val typDef = createClassTypeDef("t4", List(),
+      requiredAttr("a", DataTypes.INT_TYPE),
+      optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+      optionalAttr("c", DataTypes.BYTE_TYPE),
+      optionalAttr("d", DataTypes.SHORT_TYPE),
+      optionalAttr("enum1", "HiveObjectType"),
+      optionalAttr("e", DataTypes.INT_TYPE),
+      optionalAttr("f", DataTypes.INT_TYPE),
+      optionalAttr("g", DataTypes.LONG_TYPE),
+      optionalAttr("enum2", "PrincipalType"),
+      optionalAttr("h", DataTypes.FLOAT_TYPE),
+      optionalAttr("i", DataTypes.DOUBLE_TYPE),
+      optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
+      optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
+      optionalAttr("enum3", "TxnState"),
+      optionalAttr("l", DataTypes.DATE_TYPE),
+      optionalAttr("m", DataTypes.INT_TYPE),
+      optionalAttr("n", DataTypes.BIGDECIMAL_TYPE),
+      optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
+      optionalAttr("enum4", "LockLevel"))
+
+    val deptTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Department", List(),
+      requiredAttr("name", DataTypes.STRING_TYPE),
+      new AttributeDefinition("employees", String.format("array<%s>", "Person"),
+        Multiplicity.COLLECTION, true, "department"))
+    val personTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Person", List(),
+      requiredAttr("name", DataTypes.STRING_TYPE),
+      new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
+      new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates")
+    )
+    val managerTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Manager", List("Person"),
+      new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
+        Multiplicity.COLLECTION, false, "manager")
+    )
+
+    val typDefs = Seq(typDef, deptTypeDef, personTypeDef, managerTypeDef)
+    typDefs.foreach { tDef =>
+      val ser2 = TypesSerialization.toJson(tDef, false)
+      val typesDef2 = TypesSerialization.fromJson(ser2)
+      Assert.assertEquals(tDef, typesDef2.classTypes(0))
+
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/BuilderTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/BuilderTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/BuilderTest.scala
deleted file mode 100644
index 496ebf9..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/BuilderTest.scala
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.builders
-
-import org.apache.hadoop.metadata.typesystem.TypesDef
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem
-import org.scalatest.{BeforeAndAfter, FunSuite}
-
-abstract class BuilderTest extends FunSuite with BeforeAndAfter {
-
-  var tDef : TypesDef = null
-
-  before {
-    TypeSystem.getInstance().reset()
-
-    val b = new TypesBuilder
-    import b._
-
-    tDef = types {
-
-      _trait("Dimension") {}
-      _trait("PII") {}
-      _trait("Metric") {}
-      _trait("ETL") {}
-      _trait("JdbcAccess") {}
-
-      _class("DB") {
-        "name" ~ (string, required, indexed, unique)
-        "owner" ~ (string)
-        "createTime" ~ (int)
-      }
-
-      _class("StorageDesc") {
-        "inputFormat" ~ (string, required)
-        "outputFormat" ~ (string, required)
-      }
-
-      _class("Column") {
-        "name" ~ (string, required)
-        "dataType" ~ (string, required)
-        "sd" ~ ("StorageDesc", required)
-      }
-
-      _class("Table", List()) {
-        "name" ~ (string,  required,  indexed)
-        "db" ~ ("DB", required)
-        "sd" ~ ("StorageDesc", required)
-      }
-
-      _class("LoadProcess") {
-        "name" ~ (string, required)
-        "inputTables" ~ (array("Table"), collection)
-        "outputTable" ~ ("Table", required)
-
-      }
-
-      _class("View") {
-        "name" ~ (string, required)
-        "inputTables" ~ (array("Table"), collection)
-      }
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/InstanceBuilderTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/InstanceBuilderTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/InstanceBuilderTest.scala
deleted file mode 100644
index 227b524..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/InstanceBuilderTest.scala
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.builders
-
-import org.apache.hadoop.metadata.typesystem.types.{Multiplicity, ClassType, TypeSystem}
-
-
-class InstanceBuilderTest extends BuilderTest {
-
-  test("test1") {
-    TypeSystem.getInstance().defineTypes(tDef)
-
-    val b = new InstanceBuilder
-    import b._
-
-    val instances = b create {
-
-      val salesDB = instance("DB") {  // use instance to create Referenceables. use closure to
-                                         // set attributes of instance
-        'name ~ "Sales"                  // use '~' to set attributes. Use a Symbol (names starting with ') for
-                                         // attribute names.
-        'owner ~ "John ETL"
-        'createTime ~ 1000
-      }
-
-      val salesFact = instance("Table") {
-        'name ~ "sales_fact"
-        'db ~ salesDB
-        val sd = instance("StorageDesc") {    // any valid scala allowed in closure.
-          'inputFormat ~ "TextIputFormat"
-          'outputFormat ~ "TextOutputFormat"
-        }
-        'sd ~ sd                              // use ~ to set references, collections and maps.
-        val columns = Seq(
-          instance("Column") {
-            'name ~ "time_id"
-            'dataType ~ "int"
-            'sd ~ sd
-          },
-          instance("Column") {
-            'name ~ "product_id"
-            'dataType ~ "int"
-            'sd ~ sd
-          },
-          instance("Column") {
-            'name ~ "customer_id"
-            'dataType ~ "int"
-            'sd ~ sd
-          },
-          instance("Column", "Metric") {
-            'name ~ "sales"
-            'dataType ~ "int"
-            'sd ~ sd
-            'Metric("x") ~ 1                // use 'TraitName("attrName") to set values on traits.
-          }
-        )
-
-        'columns ~ columns
-
-      }
-
-      salesFact.sd.inputFormat ~ "TextInputFormat"   // use dot navigation to alter attributes in the object graph.
-                                                     // here I am fixing the typo in "TextInputFormat"
-      // dot navigation also works for arrays.
-      // here I am fixing column(3). Metric trait has no attributes.
-      val c = salesFact.columns
-      c(3) = instance("Column", "Metric") {
-        'name ~ "sales"
-        'dataType ~ "int"
-        'sd ~ salesFact.sd
-      }
-
-    }
-
-    val ts = TypeSystem.getInstance()
-
-    import scala.collection.JavaConversions._
-    val typedInstances = instances.map { i =>
-      val iTyp = ts.getDataType(classOf[ClassType], i.getTypeName)
-      iTyp.convert(i, Multiplicity.REQUIRED)
-    }
-
-    typedInstances.foreach { i =>
-      println(i)
-    }
-
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/MultiplicityTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/MultiplicityTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/MultiplicityTest.scala
deleted file mode 100644
index 73d912b..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/MultiplicityTest.scala
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.builders
-
-import org.apache.hadoop.metadata.MetadataException
-import org.apache.hadoop.metadata.typesystem.types.{Multiplicity, ClassType, TypeSystem}
-import org.scalatest.{BeforeAndAfterAll, FunSuite}
-
-
-class MultiplicityTest extends FunSuite with BeforeAndAfterAll {
-
-  override  def beforeAll() = {
-    TypeSystem.getInstance().reset()
-
-    val b = new TypesBuilder
-    import b._
-
-    val tDef = types {
-
-      _trait("Dimension") {}
-      _trait("PII") {}
-      _trait("Metric") {}
-      _trait("ETL") {}
-      _trait("JdbcAccess") {}
-
-      _class("DB") {
-        "name" ~ (string, required, indexed, unique)
-        "owner" ~ (string)
-        "createTime" ~ (int)
-      }
-
-      _class("StorageDesc") {
-        "inputFormat" ~ (string, required)
-        "outputFormat" ~ (string, required)
-      }
-
-      _class("Column") {
-        "name" ~ (string, required)
-        "dataType" ~ (string, required)
-        "sd" ~ ("StorageDesc", required)
-      }
-
-      _class("Table", List()) {
-        "name" ~ (string,  required,  indexed)
-        "db" ~ ("DB", required)
-        "sd" ~ ("StorageDesc", required)
-      }
-
-      _class("LoadProcess") {
-        "name" ~ (string, required)
-        "inputTables" ~ (array("Table"), collection)
-        "outputTable" ~ ("Table", required)
-
-      }
-
-      _class("View") {
-        "name" ~ (string, required)
-        "inputTables" ~ (array("Table"), collection)
-      }
-
-      _class("AT") {
-        "name" ~ (string, required)
-        "stringSet" ~ (array("string"), multiplicty(0, Int.MaxValue, true))
-      }
-    }
-
-    TypeSystem.getInstance().defineTypes(tDef)
-  }
-
-  test("test1") {
-
-    val b = new InstanceBuilder
-    import b._
-
-    val instances = b create {
-      val a = instance("AT") {  // use instance to create Referenceables. use closure to
-        // set attributes of instance
-        'name ~ "A1"                  // use '~' to set attributes. Use a Symbol (names starting with ') for
-        'stringSet ~ Seq("a", "a")
-      }
-    }
-
-    val ts = TypeSystem.getInstance()
-    import scala.collection.JavaConversions._
-    val typedInstances = instances.map { i =>
-      val iTyp = ts.getDataType(classOf[ClassType], i.getTypeName)
-      iTyp.convert(i, Multiplicity.REQUIRED)
-    }
-
-    typedInstances.foreach { i =>
-      println(i)
-    }
-  }
-
-  test("WrongMultiplicity") {
-    val b = new TypesBuilder
-    import b._
-    val tDef = types {
-      _class("Wrong") {
-        "name" ~ (string, required)
-        "stringSet" ~ (string, multiplicty(0, Int.MaxValue, true))
-      }
-    }
-    val me = intercept[MetadataException] {
-      TypeSystem.getInstance().defineTypes(tDef)
-    }
-    assert("A multiplicty of more than one requires a collection type for attribute 'stringSet'" == me.getMessage)
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/TypesBuilderTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/TypesBuilderTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/TypesBuilderTest.scala
deleted file mode 100644
index f1f9024..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/TypesBuilderTest.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.builders
-
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization
-import org.apache.hadoop.metadata.typesystem.types.{TypeSystem, BaseTest}
-import org.junit.runner.RunWith
-import org.scalatest.{BeforeAndAfter, FunSuite}
-import org.scalatest.junit.JUnitRunner
-
-@RunWith(classOf[JUnitRunner])
-class TypesBuilderTest extends BuilderTest {
-
-
-  test("test1") {
-    TypeSystem.getInstance().defineTypes(tDef)
-
-    println(TypesSerialization.toJson(TypeSystem.getInstance(), x => true))
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/SerializationTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/SerializationTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/SerializationTest.scala
deleted file mode 100755
index 86cdb72..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/SerializationTest.scala
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.json
-
-import com.google.common.collect.ImmutableList
-import org.apache.hadoop.metadata.typesystem.persistence.{ReferenceableInstance, StructInstance, Id}
-import org.apache.hadoop.metadata.typesystem.types._
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil
-import org.apache.hadoop.metadata.typesystem.{ITypedReferenceableInstance, ITypedStruct, Referenceable, Struct}
-import org.json4s.native.JsonMethods._
-import org.json4s.native.Serialization.{write => swrite, _}
-import org.json4s.{NoTypeHints, _}
-import org.junit.{Assert, Before, Test}
-
-class SerializationTest extends BaseTest {
-
-    private[metadata] var structType: StructType = null
-    private[metadata] var recursiveStructType: StructType = null
-
-    @Before
-    override def setup {
-        super.setup
-        structType = getTypeSystem.getDataType(classOf[StructType], BaseTest.STRUCT_TYPE_1).asInstanceOf[StructType]
-        recursiveStructType = getTypeSystem.getDataType(classOf[StructType], BaseTest.STRUCT_TYPE_2).asInstanceOf[StructType]
-    }
-
-    @Test def test1 {
-        val s: Struct = BaseTest.createStruct()
-        val ts: ITypedStruct = structType.convert(s, Multiplicity.REQUIRED)
-
-        Assert.assertEquals(ts.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.1, 1.1]\n\to : \t{b=2.0, a=1.0}\n}")
-
-        implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
-            new BigDecimalSerializer + new BigIntegerSerializer
-
-        //Json representation
-        val ser = swrite(ts)
-        Assert.assertEquals(ser, "{\"$typeName$\":\"t1\",\"e\":1,\"n\":[1.1,1.1],\"h\":1.0,\"b\":true,\"k\":1,\"j\":1,\"d\":2,\"m\":[1,1],\"g\":1,\"a\":1,\"i\":1.0,\"c\":1,\"l\":\"" + BaseTest.TEST_DATE + "\",\"f\":1,\"o\":{\"b\":2.0,\"a\":1.0}}")
-
-        // Typed Struct read back
-        val ts1 = read[StructInstance](ser)
-        Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}")
-    }
-
-    @Test def test2 {
-        val s: Struct = BaseTest.createStruct()
-        val ts: ITypedStruct = structType.convert(s, Multiplicity.REQUIRED)
-
-        implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
-            new BigDecimalSerializer + new BigIntegerSerializer
-
-        val ts1 = read[StructInstance](
-            """
-        {"$typeName$":"t1","e":1,"n":[1.1,1.1],"h":1.0,"b":true,"k":1,"j":1,"d":2,"m":[1,1],"g":1,"a":1,"i":1.0,
-        "c":1,"l":"2014-12-03T19:38:55.053Z","f":1,"o":{"b":2.0,"a":1.0}}""")
-        // Typed Struct read from string
-        Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-03T19:38:55.053Z\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}")
-    }
-
-    @Test def testTrait {
-        val A: HierarchicalTypeDefinition[TraitType] = TypesUtil.createTraitTypeDef("A", null,
-            TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
-            TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
-            TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
-            TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE))
-        val B: HierarchicalTypeDefinition[TraitType] = TypesUtil.createTraitTypeDef(
-            "B", ImmutableList.of[String]("A"),
-            TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE))
-        val C: HierarchicalTypeDefinition[TraitType] = TypesUtil.createTraitTypeDef(
-            "C", ImmutableList.of[String]("A"),
-            TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE))
-        val D: HierarchicalTypeDefinition[TraitType] = TypesUtil.createTraitTypeDef(
-            "D", ImmutableList.of[String]("B", "C"),
-            TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE))
-
-        defineTraits(A, B, C, D)
-
-        val DType: TraitType = getTypeSystem.getDataType(classOf[TraitType], "D").asInstanceOf[TraitType]
-        val s1: Struct = new Struct("D")
-        s1.set("d", 1)
-        s1.set("c", 1)
-        s1.set("b", true)
-        s1.set("a", 1)
-        s1.set("A.B.D.b", true)
-        s1.set("A.B.D.c", 2)
-        s1.set("A.B.D.d", 2)
-        s1.set("A.C.D.a", 3)
-        s1.set("A.C.D.b", false)
-        s1.set("A.C.D.c", 3)
-        s1.set("A.C.D.d", 3)
-
-        val s: Struct = BaseTest.createStruct()
-        val ts: ITypedStruct = DType.convert(s1, Multiplicity.REQUIRED)
-
-        implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
-            new BigDecimalSerializer + new BigIntegerSerializer
-
-        // Typed Struct :
-        Assert.assertEquals(ts.toString, "{\n\td : \t1\n\tb : \ttrue\n\tc : \t1\n\ta : \t1\n\tA.B.D.b : \ttrue\n\tA.B.D.c : \t2\n\tA.B.D.d : \t2\n\tA.C.D.a : \t3\n\tA.C.D.b : \tfalse\n\tA.C.D.c : \t3\n\tA.C.D.d : \t3\n}")
-
-        // Json representation :
-        val ser = swrite(ts)
-        Assert.assertEquals(ser, "{\"$typeName$\":\"D\",\"A.C.D.d\":3,\"A.B.D.c\":2,\"b\":true,\"A.C.D.c\":3,\"d\":1,\"A.B.D.b\":true,\"a\":1,\"A.C.D.b\":false,\"A.B.D.d\":2,\"c\":1,\"A.C.D.a\":3}")
-
-        val ts1 = read[StructInstance](
-            """
-        {"$typeName$":"D","A.C.D.d":3,"A.B.D.c":2,"b":true,"A.C.D.c":3,"d":1,
-        "A.B.D.b":true,"a":1,"A.C.D.b":false,"A.B.D.d":2,"c":1,"A.C.D.a":3}""")
-        // Typed Struct read from string:
-        Assert.assertEquals(ts1.toString, "{\n\td : \t1\n\tb : \ttrue\n\tc : \t1\n\ta : \t1\n\tA.B.D.b : \ttrue\n\tA.B.D.c : \t2\n\tA.B.D.d : \t2\n\tA.C.D.a : \t3\n\tA.C.D.b : \tfalse\n\tA.C.D.c : \t3\n\tA.C.D.d : \t3\n}")
-    }
-
-  def defineHRTypes(ts: TypeSystem) : Unit = {
-    val deptTypeDef: HierarchicalTypeDefinition[ClassType] = TypesUtil.createClassTypeDef(
-      "Department",
-      ImmutableList.of[String],
-      TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
-      new AttributeDefinition("employees", String.format("array<%s>", "Person"),
-        Multiplicity.COLLECTION, true, "department"))
-    val personTypeDef: HierarchicalTypeDefinition[ClassType] = TypesUtil.createClassTypeDef(
-      "Person", ImmutableList.of[String],
-      TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
-      new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
-      new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates"))
-    val managerTypeDef: HierarchicalTypeDefinition[ClassType] = TypesUtil.createClassTypeDef(
-      "Manager", ImmutableList.of[String]("Person"),
-      new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
-        Multiplicity.COLLECTION, false, "manager"))
-    val securityClearanceTypeDef: HierarchicalTypeDefinition[TraitType] =
-      TypesUtil.createTraitTypeDef("SecurityClearance", ImmutableList.of[String],
-        TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE))
-
-    ts.defineTypes(ImmutableList.of[StructTypeDefinition],
-      ImmutableList.of[HierarchicalTypeDefinition[TraitType]](securityClearanceTypeDef),
-      ImmutableList.of[HierarchicalTypeDefinition[ClassType]](deptTypeDef, personTypeDef, managerTypeDef)
-    )
-
-  }
-
-  def defineHRDept() : Referenceable = {
-    val hrDept: Referenceable = new Referenceable("Department")
-    val john: Referenceable = new Referenceable("Person")
-    val jane: Referenceable = new Referenceable("Manager", "SecurityClearance")
-    hrDept.set("name", "hr")
-    john.set("name", "John")
-    john.set("department", hrDept.getId)
-    jane.set("name", "Jane")
-    jane.set("department", hrDept.getId)
-    john.set("manager", jane.getId)
-    hrDept.set("employees", ImmutableList.of[Referenceable](john, jane))
-    jane.set("subordinates", ImmutableList.of[Id](john.getId))
-    jane.getTrait("SecurityClearance").set("level", 1)
-    hrDept
-  }
-
-  @Test def testClass {
-
-    val ts: TypeSystem = getTypeSystem
-    defineHRTypes(ts)
-    val hrDept: Referenceable = defineHRDept()
-
-    val deptType: ClassType = ts.getDataType(classOf[ClassType], "Department")
-    val hrDept2: ITypedReferenceableInstance = deptType.convert(hrDept, Multiplicity.REQUIRED)
-
-    println(s"HR Dept Object Graph:\n${hrDept2}\n")
-
-    implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
-      new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
-
-    val ser = swrite(hrDept2)
-    println(s"HR Dept JSON:\n${pretty(render(parse(ser)))}\n")
-
-    println(s"HR Dept Object Graph read from JSON:${read[ReferenceableInstance](ser)}\n")
-  }
-
-  @Test def testReference {
-
-    val ts: TypeSystem = getTypeSystem
-    defineHRTypes(ts)
-    val hrDept: Referenceable = defineHRDept()
-
-
-    val jsonStr = InstanceSerialization.toJson(hrDept)
-    val hrDept2 = InstanceSerialization.fromJsonReferenceable(jsonStr)
-
-    val deptType: ClassType = ts.getDataType(classOf[ClassType], "Department")
-    val hrDept3: ITypedReferenceableInstance = deptType.convert(hrDept2, Multiplicity.REQUIRED)
-
-    println(s"HR Dept Object Graph:\n${hrDept3}\n")
-
-    implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
-      new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
-
-    val ser = swrite(hrDept3)
-    println(s"HR Dept JSON:\n${pretty(render(parse(ser)))}\n")
-
-    println(s"HR Dept Object Graph read from JSON:${read[ReferenceableInstance](ser)}\n")
-  }
-
-  @Test def testReference2 {
-
-    val ts: TypeSystem = getTypeSystem
-    defineHRTypes(ts)
-    val hrDept: Referenceable = defineHRDept()
-
-    val deptType: ClassType = ts.getDataType(classOf[ClassType], "Department")
-    val hrDept2: ITypedReferenceableInstance = deptType.convert(hrDept, Multiplicity.REQUIRED)
-
-    val jsonStr = InstanceSerialization.toJson(hrDept2)
-    val hrDept3 = InstanceSerialization.fromJsonReferenceable(jsonStr)
-
-    val hrDept4: ITypedReferenceableInstance = deptType.convert(hrDept2, Multiplicity.REQUIRED)
-
-    println(s"HR Dept Object Graph:\n${hrDept4}\n")
-
-    implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
-      new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
-
-    val ser = swrite(hrDept4)
-    println(s"HR Dept JSON:\n${pretty(render(parse(ser)))}\n")
-
-    println(s"HR Dept Object Graph read from JSON:${read[ReferenceableInstance](ser)}\n")
-
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/TypesSerializationTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/TypesSerializationTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/TypesSerializationTest.scala
deleted file mode 100755
index 7b4e44d..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/TypesSerializationTest.scala
+++ /dev/null
@@ -1,300 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.json
-
-import com.google.common.collect.ImmutableList
-import org.apache.hadoop.metadata.typesystem.types._
-import org.junit.{Assert, Test}
-
-class TypesSerializationTest extends BaseTest with TypeHelpers {
-
-    @Test def test1: Unit = {
-
-        val ts = getTypeSystem
-
-        val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
-            optionalAttr("b", DataTypes.BOOLEAN_TYPE),
-            optionalAttr("c", DataTypes.BYTE_TYPE),
-            optionalAttr("d", DataTypes.SHORT_TYPE),
-            optionalAttr("e", DataTypes.INT_TYPE),
-            optionalAttr("f", DataTypes.INT_TYPE),
-            optionalAttr("g", DataTypes.LONG_TYPE),
-            optionalAttr("h", DataTypes.FLOAT_TYPE),
-            optionalAttr("i", DataTypes.DOUBLE_TYPE),
-            optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
-            optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
-            optionalAttr("l", DataTypes.DATE_TYPE),
-            optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
-            optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
-            optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
-
-
-        ts.defineTypes(ImmutableList.of[StructTypeDefinition](sDef),
-            ImmutableList.of[HierarchicalTypeDefinition[TraitType]],
-            ImmutableList.of[HierarchicalTypeDefinition[ClassType]]
-        )
-
-        val A: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("A", List(),
-            requiredAttr("a", DataTypes.INT_TYPE),
-            optionalAttr("b", DataTypes.BOOLEAN_TYPE),
-            optionalAttr("c", DataTypes.BYTE_TYPE),
-            optionalAttr("d", DataTypes.SHORT_TYPE))
-        val B: HierarchicalTypeDefinition[TraitType] =
-            createTraitTypeDef("B", Seq("A"), optionalAttr("b", DataTypes.BOOLEAN_TYPE))
-        val C: HierarchicalTypeDefinition[TraitType] =
-            createTraitTypeDef("C", Seq("A"), optionalAttr("c", DataTypes.BYTE_TYPE))
-        val D: HierarchicalTypeDefinition[TraitType] =
-            createTraitTypeDef("D", Seq("B", "C"), optionalAttr("d", DataTypes.SHORT_TYPE))
-
-        defineTraits(ts, A, B, C, D)
-
-        ts.defineEnumType("HiveObjectType",
-            new EnumValue("GLOBAL", 1),
-            new EnumValue("DATABASE", 2),
-            new EnumValue("TABLE", 3),
-            new EnumValue("PARTITION", 4),
-            new EnumValue("COLUMN", 5))
-
-        ts.defineEnumType("PrincipalType",
-            new EnumValue("USER", 1),
-            new EnumValue("ROLE", 2),
-            new EnumValue("GROUP", 3))
-
-        ts.defineEnumType("TxnState",
-            new EnumValue("COMMITTED", 1),
-            new EnumValue("ABORTED", 2),
-            new EnumValue("OPEN", 3))
-
-        ts.defineEnumType("LockLevel",
-            new EnumValue("DB", 1),
-            new EnumValue("TABLE", 2),
-            new EnumValue("PARTITION", 3))
-
-        defineClassType(ts, createClassTypeDef("t4", List(),
-            requiredAttr("a", DataTypes.INT_TYPE),
-            optionalAttr("b", DataTypes.BOOLEAN_TYPE),
-            optionalAttr("c", DataTypes.BYTE_TYPE),
-            optionalAttr("d", DataTypes.SHORT_TYPE),
-            optionalAttr("enum1", ts.getDataType(classOf[EnumType], "HiveObjectType")),
-            optionalAttr("e", DataTypes.INT_TYPE),
-            optionalAttr("f", DataTypes.INT_TYPE),
-            optionalAttr("g", DataTypes.LONG_TYPE),
-            optionalAttr("enum2", ts.getDataType(classOf[EnumType], "PrincipalType")),
-            optionalAttr("h", DataTypes.FLOAT_TYPE),
-            optionalAttr("i", DataTypes.DOUBLE_TYPE),
-            optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
-            optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
-            optionalAttr("enum3", ts.getDataType(classOf[EnumType], "TxnState")),
-            optionalAttr("l", DataTypes.DATE_TYPE),
-            optionalAttr("m", ts.defineArrayType(DataTypes.INT_TYPE)),
-            optionalAttr("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
-            optionalAttr("o", ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
-            optionalAttr("enum4", ts.getDataType(classOf[EnumType], "LockLevel"))))
-
-        val deptTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Department", List(),
-            requiredAttr("name", DataTypes.STRING_TYPE),
-            new AttributeDefinition("employees", String.format("array<%s>", "Person"),
-                Multiplicity.COLLECTION, true, "department"))
-        val personTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Person", List(),
-            requiredAttr("name", DataTypes.STRING_TYPE),
-            new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
-            new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates")
-        )
-        val managerTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Manager", List("Person"),
-            new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
-                Multiplicity.COLLECTION, false, "manager")
-        )
-        val securityClearanceTypeDef: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("SecurityClearance", List(),
-            requiredAttr("level", DataTypes.INT_TYPE)
-        )
-        ts.defineTypes(ImmutableList.of[StructTypeDefinition],
-            ImmutableList.of[HierarchicalTypeDefinition[TraitType]](securityClearanceTypeDef),
-            ImmutableList.of[HierarchicalTypeDefinition[ClassType]](deptTypeDef, personTypeDef, managerTypeDef))
-
-        val ser = TypesSerialization.toJson(ts, _ => true)
-
-        val typesDef1 = TypesSerialization.fromJson(ser)
-
-        val ts1 = TypeSystem.getInstance()
-        ts1.reset()
-
-        typesDef1.enumTypes.foreach(ts1.defineEnumType(_))
-
-        ts1.defineTypes(ImmutableList.copyOf(typesDef1.structTypes.toArray),
-            ImmutableList.copyOf(typesDef1.traitTypes.toArray),
-            ImmutableList.copyOf(typesDef1.classTypes.toArray)
-        )
-        val ser2 = TypesSerialization.toJson(ts1, _ => true)
-        val typesDef2 = TypesSerialization.fromJson(ser2)
-
-        Assert.assertEquals(typesDef1, typesDef2)
-    }
-
-  @Test def test2: Unit = {
-
-    val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
-      optionalAttr("b", DataTypes.BOOLEAN_TYPE),
-      optionalAttr("c", DataTypes.BYTE_TYPE),
-      optionalAttr("d", DataTypes.SHORT_TYPE),
-      optionalAttr("e", DataTypes.INT_TYPE),
-      optionalAttr("f", DataTypes.INT_TYPE),
-      optionalAttr("g", DataTypes.LONG_TYPE),
-      optionalAttr("h", DataTypes.FLOAT_TYPE),
-      optionalAttr("i", DataTypes.DOUBLE_TYPE),
-      optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
-      optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
-      optionalAttr("l", DataTypes.DATE_TYPE),
-      optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
-      optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
-      optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
-
-
-
-    val ser2 = TypesSerialization.toJson(sDef)
-    val typesDef2 = TypesSerialization.fromJson(ser2)
-
-    Assert.assertEquals(sDef, typesDef2.structTypes(0))
-  }
-
-  @Test def test3: Unit = {
-
-    val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
-      optionalAttr("b", DataTypes.BOOLEAN_TYPE),
-      optionalAttr("c", DataTypes.BYTE_TYPE),
-      optionalAttr("d", DataTypes.SHORT_TYPE),
-      optionalAttr("e", DataTypes.INT_TYPE),
-      optionalAttr("f", DataTypes.INT_TYPE),
-      optionalAttr("g", DataTypes.LONG_TYPE),
-      optionalAttr("h", DataTypes.FLOAT_TYPE),
-      optionalAttr("i", DataTypes.DOUBLE_TYPE),
-      optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
-      optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
-      optionalAttr("l", DataTypes.DATE_TYPE),
-      optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
-      optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
-      optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
-
-
-
-    val ser2 = TypesSerialization.toJson(sDef)
-    val typesDef2 = TypesSerialization.fromJson(ser2)
-
-    Assert.assertEquals(sDef, typesDef2.structTypes(0))
-  }
-
-  @Test def test4 : Unit = {
-
-    val A: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("A", List(),
-      requiredAttr("a", DataTypes.INT_TYPE),
-      optionalAttr("b", DataTypes.BOOLEAN_TYPE),
-      optionalAttr("c", DataTypes.BYTE_TYPE),
-      optionalAttr("d", DataTypes.SHORT_TYPE))
-    val B: HierarchicalTypeDefinition[TraitType] =
-      createTraitTypeDef("B", Seq("A"), optionalAttr("b", DataTypes.BOOLEAN_TYPE))
-    val C: HierarchicalTypeDefinition[TraitType] =
-      createTraitTypeDef("C", Seq("A"), optionalAttr("c", DataTypes.BYTE_TYPE))
-    val D: HierarchicalTypeDefinition[TraitType] =
-      createTraitTypeDef("D", Seq("B", "C"), optionalAttr("d", DataTypes.SHORT_TYPE))
-
-    val typDefs = Seq(A,B,C,D)
-    typDefs.foreach { tDef =>
-      val ser2 = TypesSerialization.toJson(tDef, true)
-      val typesDef2 = TypesSerialization.fromJson(ser2)
-      Assert.assertEquals(tDef, typesDef2.traitTypes(0))
-
-    }
-  }
-
-  @Test def test5 : Unit = {
-    val e1 = new EnumTypeDefinition("HiveObjectType",
-      new EnumValue("GLOBAL", 1),
-      new EnumValue("DATABASE", 2),
-      new EnumValue("TABLE", 3),
-      new EnumValue("PARTITION", 4),
-      new EnumValue("COLUMN", 5))
-
-    val e2 = new EnumTypeDefinition("PrincipalType",
-      new EnumValue("USER", 1),
-      new EnumValue("ROLE", 2),
-      new EnumValue("GROUP", 3))
-
-    val e3 = new EnumTypeDefinition("TxnState",
-      new EnumValue("COMMITTED", 1),
-      new EnumValue("ABORTED", 2),
-      new EnumValue("OPEN", 3))
-
-    val e4 = new EnumTypeDefinition("LockLevel",
-      new EnumValue("DB", 1),
-      new EnumValue("TABLE", 2),
-      new EnumValue("PARTITION", 3))
-
-    val typDefs = Seq(e1,e2,e3,e4)
-    typDefs.foreach { tDef =>
-      val ser2 = TypesSerialization.toJson(tDef)
-      val typesDef2 = TypesSerialization.fromJson(ser2)
-      Assert.assertEquals(tDef, typesDef2.enumTypes(0))
-
-    }
-  }
-
-  @Test def test6 : Unit = {
-    val typDef = createClassTypeDef("t4", List(),
-      requiredAttr("a", DataTypes.INT_TYPE),
-      optionalAttr("b", DataTypes.BOOLEAN_TYPE),
-      optionalAttr("c", DataTypes.BYTE_TYPE),
-      optionalAttr("d", DataTypes.SHORT_TYPE),
-      optionalAttr("enum1", "HiveObjectType"),
-      optionalAttr("e", DataTypes.INT_TYPE),
-      optionalAttr("f", DataTypes.INT_TYPE),
-      optionalAttr("g", DataTypes.LONG_TYPE),
-      optionalAttr("enum2", "PrincipalType"),
-      optionalAttr("h", DataTypes.FLOAT_TYPE),
-      optionalAttr("i", DataTypes.DOUBLE_TYPE),
-      optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
-      optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
-      optionalAttr("enum3", "TxnState"),
-      optionalAttr("l", DataTypes.DATE_TYPE),
-      optionalAttr("m", DataTypes.INT_TYPE),
-      optionalAttr("n", DataTypes.BIGDECIMAL_TYPE),
-      optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
-      optionalAttr("enum4", "LockLevel"))
-
-    val deptTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Department", List(),
-      requiredAttr("name", DataTypes.STRING_TYPE),
-      new AttributeDefinition("employees", String.format("array<%s>", "Person"),
-        Multiplicity.COLLECTION, true, "department"))
-    val personTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Person", List(),
-      requiredAttr("name", DataTypes.STRING_TYPE),
-      new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
-      new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates")
-    )
-    val managerTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Manager", List("Person"),
-      new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
-        Multiplicity.COLLECTION, false, "manager")
-    )
-
-    val typDefs = Seq(typDef, deptTypeDef, personTypeDef, managerTypeDef)
-    typDefs.foreach { tDef =>
-      val ser2 = TypesSerialization.toJson(tDef, false)
-      val typesDef2 = TypesSerialization.fromJson(ser2)
-      Assert.assertEquals(tDef, typesDef2.classTypes(0))
-
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/Main.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/Main.java b/webapp/src/main/java/org/apache/atlas/Main.java
new file mode 100755
index 0000000..76ca965
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/Main.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+import org.apache.atlas.web.service.EmbeddedServer;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+
+/**
+ * Driver for running Metadata as a standalone server with embedded jetty server.
+ */
+public final class Main {
+    private static final Logger LOG = LoggerFactory.getLogger(Main.class);
+    private static final String APP_PATH = "app";
+    private static final String APP_PORT = "port";
+    private static final String METADATA_HOME = "atlas.home";
+    private static final String METADATA_LOG_DIR = "atlas.log.dir";
+
+    /**
+     * Prevent users from constructing this.
+     */
+    private Main() {
+    }
+
+    private static CommandLine parseArgs(String[] args) throws ParseException {
+        Options options = new Options();
+        Option opt;
+
+        opt = new Option(APP_PATH, true, "Application Path");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option(APP_PORT, true, "Application Port");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        return new GnuParser().parse(options, args);
+    }
+
+    public static void main(String[] args) throws Exception {
+        CommandLine cmd = parseArgs(args);
+        PropertiesConfiguration buildConfiguration =
+                new PropertiesConfiguration("atlas-buildinfo.properties");
+        String appPath = "webapp/target/atlas-webapp-" + getProjectVersion(buildConfiguration);
+
+        if (cmd.hasOption(APP_PATH)) {
+            appPath = cmd.getOptionValue(APP_PATH);
+        }
+
+        setApplicationHome();
+        PropertiesConfiguration configuration = PropertiesUtil.getApplicationProperties();
+        final String enableTLSFlag = configuration.getString("atlas.enableTLS");
+        final int appPort = getApplicationPort(cmd, enableTLSFlag);
+        final boolean enableTLS = isTLSEnabled(enableTLSFlag, appPort);
+        configuration.setProperty("atlas.enableTLS", String.valueOf(enableTLS));
+
+        showStartupInfo(buildConfiguration, enableTLS, appPort);
+        EmbeddedServer server = EmbeddedServer.newServer(appPort, appPath, enableTLS);
+        server.start();
+    }
+
+    private static void setApplicationHome() {
+        if (System.getProperty(METADATA_HOME) == null) {
+            System.setProperty(METADATA_HOME, "target");
+        }
+        if (System.getProperty(METADATA_LOG_DIR) == null) {
+            System.setProperty(METADATA_LOG_DIR, "target/logs");
+        }
+    }
+
+    public static String getProjectVersion(PropertiesConfiguration buildConfiguration) {
+        return buildConfiguration.getString("project.version");
+    }
+
+    private static int getApplicationPort(CommandLine cmd, String enableTLSFlag) {
+        final int appPort;
+        if (cmd.hasOption(APP_PORT)) {
+            appPort = Integer.valueOf(cmd.getOptionValue(APP_PORT));
+        } else {
+            // default : metadata.enableTLS is true
+            appPort = StringUtils.isEmpty(enableTLSFlag)
+                    || enableTLSFlag.equals("true") ? 21443 : 21000;
+        }
+
+        return appPort;
+    }
+
+    private static boolean isTLSEnabled(String enableTLSFlag, int appPort) {
+        return Boolean.valueOf(StringUtils.isEmpty(enableTLSFlag)
+                ? System
+                .getProperty("atlas.enableTLS", (appPort % 1000) == 443 ? "true" : "false")
+                : enableTLSFlag);
+    }
+
+    private static void showStartupInfo(PropertiesConfiguration buildConfiguration,
+                                        boolean enableTLS, int appPort) {
+        StringBuilder buffer = new StringBuilder();
+        buffer.append("\n############################################");
+        buffer.append("############################################");
+        buffer.append("\n                               DGI Server (STARTUP)");
+        buffer.append("\n");
+        try {
+            final Iterator<String> keys = buildConfiguration.getKeys();
+            while (keys.hasNext()) {
+                String key = keys.next();
+                buffer.append('\n').append('\t').append(key).
+                        append(":\t").append(buildConfiguration.getProperty(key));
+            }
+        } catch (Throwable e) {
+            buffer.append("*** Unable to get build info ***");
+        }
+        buffer.append("\n############################################");
+        buffer.append("############################################");
+        LOG.info(buffer.toString());
+        LOG.info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+        LOG.info("Server starting with TLS ? {} on port {}", enableTLS, appPort);
+        LOG.info("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<");
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java b/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
new file mode 100755
index 0000000..4c0ffe8
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
@@ -0,0 +1,478 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.examples;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.json.InstanceSerialization;
+import org.apache.atlas.typesystem.json.TypesSerialization;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeUtils;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.util.List;
+
+/**
+ * A driver that sets up sample types and data for testing purposes.
+ * Please take a look at QueryDSL in docs for the Meta Model.
+ * todo - move this to examples module.
+ */
+public class QuickStart {
+
+    public static void main(String[] args) throws Exception {
+        String baseUrl = getServerUrl(args);
+        QuickStart quickStart = new QuickStart(baseUrl);
+
+        // Shows how to create types in DGI for your meta model
+        quickStart.createTypes();
+
+        // Shows how to create entities (instances) for the added types in DGI
+        quickStart.createEntities();
+
+        // Shows some search queries using DSL based on types
+        quickStart.search();
+    }
+
+    static String getServerUrl(String[] args) {
+        String baseUrl = "http://localhost:21000";
+        if (args.length > 0) {
+            baseUrl = args[0];
+        }
+
+        return baseUrl;
+    }
+
+    private static final String DATABASE_TYPE = "DB";
+    private static final String COLUMN_TYPE = "Column";
+    private static final String TABLE_TYPE = "Table";
+    private static final String VIEW_TYPE = "View";
+    private static final String LOAD_PROCESS_TYPE = "LoadProcess";
+    private static final String STORAGE_DESC_TYPE = "StorageDesc";
+
+    private static final String[] TYPES = {
+            DATABASE_TYPE, TABLE_TYPE, STORAGE_DESC_TYPE, COLUMN_TYPE, LOAD_PROCESS_TYPE, VIEW_TYPE,
+            "JdbcAccess", "ETL", "Metric", "PII", "Fact", "Dimension"
+    };
+
+    private final MetadataServiceClient metadataServiceClient;
+
+    QuickStart(String baseUrl) {
+        metadataServiceClient = new MetadataServiceClient(baseUrl);
+    }
+
+    void createTypes() throws Exception {
+        TypesDef typesDef = createTypeDefinitions();
+
+        String typesAsJSON = TypesSerialization.toJson(typesDef);
+        System.out.println("typesAsJSON = " + typesAsJSON);
+        metadataServiceClient.createType(typesAsJSON);
+
+        // verify types created
+        verifyTypesCreated();
+    }
+
+    TypesDef createTypeDefinitions() throws Exception {
+        HierarchicalTypeDefinition<ClassType> dbClsDef
+                = TypesUtil.createClassTypeDef(DATABASE_TYPE, null,
+                attrDef("name", DataTypes.STRING_TYPE),
+                attrDef("description", DataTypes.STRING_TYPE),
+                attrDef("locationUri", DataTypes.STRING_TYPE),
+                attrDef("owner", DataTypes.STRING_TYPE),
+                attrDef("createTime", DataTypes.INT_TYPE)
+        );
+
+        HierarchicalTypeDefinition<ClassType> storageDescClsDef =
+                TypesUtil.createClassTypeDef(STORAGE_DESC_TYPE, null,
+                        attrDef("location", DataTypes.STRING_TYPE),
+                        attrDef("inputFormat", DataTypes.STRING_TYPE),
+                        attrDef("outputFormat", DataTypes.STRING_TYPE),
+                        attrDef("compressed", DataTypes.STRING_TYPE,
+                                Multiplicity.REQUIRED, false, null)
+                );
+
+        HierarchicalTypeDefinition<ClassType> columnClsDef =
+                TypesUtil.createClassTypeDef(COLUMN_TYPE, null,
+                        attrDef("name", DataTypes.STRING_TYPE),
+                        attrDef("dataType", DataTypes.STRING_TYPE),
+                        attrDef("comment", DataTypes.STRING_TYPE)
+                );
+
+        HierarchicalTypeDefinition<ClassType> tblClsDef =
+                 TypesUtil.createClassTypeDef(TABLE_TYPE, ImmutableList.of("DataSet"),
+                        new AttributeDefinition("db", DATABASE_TYPE,
+                                Multiplicity.REQUIRED, false, null),
+                        new AttributeDefinition("sd", STORAGE_DESC_TYPE,
+                                Multiplicity.REQUIRED, true, null),
+                        attrDef("owner", DataTypes.STRING_TYPE),
+                        attrDef("createTime", DataTypes.INT_TYPE),
+                        attrDef("lastAccessTime", DataTypes.INT_TYPE),
+                        attrDef("retention", DataTypes.INT_TYPE),
+                        attrDef("viewOriginalText", DataTypes.STRING_TYPE),
+                        attrDef("viewExpandedText", DataTypes.STRING_TYPE),
+                        attrDef("tableType", DataTypes.STRING_TYPE),
+                        attrDef("temporary", DataTypes.BOOLEAN_TYPE),
+                        new AttributeDefinition("columns",
+                                DataTypes.arrayTypeName(COLUMN_TYPE),
+                                Multiplicity.COLLECTION, true, null)
+                );
+
+        HierarchicalTypeDefinition<ClassType> loadProcessClsDef =
+                 TypesUtil.createClassTypeDef(LOAD_PROCESS_TYPE, ImmutableList.of("Process"),
+                        attrDef("userName", DataTypes.STRING_TYPE),
+                        attrDef("startTime", DataTypes.INT_TYPE),
+                        attrDef("endTime", DataTypes.INT_TYPE),
+                        attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
+                        attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
+                        attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
+                        attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED)
+                );
+
+        HierarchicalTypeDefinition<ClassType> viewClsDef =
+                TypesUtil.createClassTypeDef(VIEW_TYPE, null,
+                        attrDef("name", DataTypes.STRING_TYPE),
+                        new AttributeDefinition("db", DATABASE_TYPE,
+                                Multiplicity.REQUIRED, false, null),
+                        new AttributeDefinition("inputTables",
+                                DataTypes.arrayTypeName(TABLE_TYPE),
+                                Multiplicity.COLLECTION, false, null)
+                );
+
+        HierarchicalTypeDefinition<TraitType> dimTraitDef =
+                TypesUtil.createTraitTypeDef("Dimension", null);
+
+        HierarchicalTypeDefinition<TraitType> factTraitDef =
+                TypesUtil.createTraitTypeDef("Fact", null);
+
+        HierarchicalTypeDefinition<TraitType> piiTraitDef =
+                TypesUtil.createTraitTypeDef("PII", null);
+
+        HierarchicalTypeDefinition<TraitType> metricTraitDef =
+                TypesUtil.createTraitTypeDef("Metric", null);
+
+        HierarchicalTypeDefinition<TraitType> etlTraitDef =
+                TypesUtil.createTraitTypeDef("ETL", null);
+
+        HierarchicalTypeDefinition<TraitType> jdbcTraitDef =
+                TypesUtil.createTraitTypeDef("JdbcAccess", null);
+
+        return TypeUtils.getTypesDef(
+                ImmutableList.<EnumTypeDefinition>of(),
+                ImmutableList.<StructTypeDefinition>of(),
+                ImmutableList.of(dimTraitDef, factTraitDef,
+                        piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef),
+                ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef,
+                        tblClsDef, loadProcessClsDef, viewClsDef)
+        );
+    }
+
+    AttributeDefinition attrDef(String name, IDataType dT) {
+        return attrDef(name, dT, Multiplicity.OPTIONAL, false, null);
+    }
+
+    AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m) {
+        return attrDef(name, dT, m, false, null);
+    }
+
+    AttributeDefinition attrDef(String name, IDataType dT,
+                                Multiplicity m, boolean isComposite, String reverseAttributeName) {
+        Preconditions.checkNotNull(name);
+        Preconditions.checkNotNull(dT);
+        return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName);
+    }
+
+    void createEntities() throws Exception {
+        Id salesDB = database(
+                "Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
+
+
+        Referenceable sd = rawStorageDescriptor("hdfs://host:8000/apps/warehouse/sales",
+                "TextInputFormat", "TextOutputFormat", true);
+
+        List<Referenceable> salesFactColumns = ImmutableList.of(
+            rawColumn("time_id", "int", "time id"),
+            rawColumn("product_id", "int", "product id"),
+            rawColumn("customer_id", "int", "customer id", "PII"),
+            rawColumn("sales", "double", "product id", "Metric")
+        );
+
+        Id salesFact = table("sales_fact", "sales fact table",
+                salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact");
+
+        List<Referenceable> productDimColumns = ImmutableList.of(
+            rawColumn("product_id", "int", "product id"),
+            rawColumn("product_name", "string", "product name"),
+            rawColumn("brand_name", "int", "brand name")
+        );
+
+        Id productDim = table("product_dim", "product dimension table",
+                salesDB, sd, "John Doe", "Managed", productDimColumns, "Dimension");
+
+        List<Referenceable> timeDimColumns = ImmutableList.of(
+            rawColumn("time_id", "int", "time id"),
+            rawColumn("dayOfYear", "int", "day Of Year"),
+            rawColumn("weekDay", "int", "week Day")
+        );
+
+        Id timeDim = table("time_dim", "time dimension table",
+                salesDB, sd, "John Doe", "External", timeDimColumns, "Dimension");
+
+
+        List<Referenceable> customerDimColumns = ImmutableList.of(
+            rawColumn("customer_id", "int", "customer id", "PII"),
+            rawColumn("name", "string", "customer name", "PII"),
+            rawColumn("address", "string", "customer address", "PII")
+        );
+
+        Id customerDim = table("customer_dim", "customer dimension table",
+                salesDB, sd, "fetl", "External", customerDimColumns, "Dimension");
+
+
+        Id reportingDB = database("Reporting", "reporting database", "Jane BI",
+                "hdfs://host:8000/apps/warehouse/reporting");
+
+        Id salesFactDaily = table("sales_fact_daily_mv",
+                "sales fact daily materialized view", reportingDB, sd,
+                "Joe BI", "Managed", salesFactColumns, "Metric");
+
+        loadProcess("loadSalesDaily", "hive query for daily summary", "John ETL",
+                ImmutableList.of(salesFact, timeDim), ImmutableList.of(salesFactDaily),
+                "create table as select ", "plan", "id", "graph",
+                "ETL");
+
+        view("product_dim_view", reportingDB,
+                ImmutableList.of(productDim), "Dimension", "JdbcAccess");
+
+        view("customer_dim_view", reportingDB,
+                ImmutableList.of(customerDim), "Dimension", "JdbcAccess");
+
+        Id salesFactMonthly = table("sales_fact_monthly_mv",
+                "sales fact monthly materialized view",
+                reportingDB, sd, "Jane BI", "Managed", salesFactColumns, "Metric");
+
+        loadProcess("loadSalesMonthly", "hive query for monthly summary", "John ETL",
+                ImmutableList.of(salesFactDaily), ImmutableList.of(salesFactMonthly),
+                "create table as select ", "plan", "id", "graph",
+                "ETL");
+    }
+
+    private Id createInstance(Referenceable referenceable) throws Exception {
+        String typeName = referenceable.getTypeName();
+
+        String entityJSON = InstanceSerialization.toJson(referenceable, true);
+        System.out.println("Submitting new entity= " + entityJSON);
+        JSONObject jsonObject = metadataServiceClient.createEntity(entityJSON);
+        String guid = jsonObject.getString(MetadataServiceClient.GUID);
+        System.out.println("created instance for type " + typeName + ", guid: " + guid);
+
+        // return the Id for created instance with guid
+        return new Id(guid, referenceable.getId().getVersion(), referenceable.getTypeName());
+    }
+
+    Id database(String name, String description,
+                           String owner, String locationUri,
+                           String... traitNames) throws Exception {
+        Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames);
+        referenceable.set("name", name);
+        referenceable.set("description", description);
+        referenceable.set("owner", owner);
+        referenceable.set("locationUri", locationUri);
+        referenceable.set("createTime", System.currentTimeMillis());
+
+        return createInstance(referenceable);
+    }
+
+    Referenceable rawStorageDescriptor(String location, String inputFormat,
+                                       String outputFormat,
+                                       boolean compressed) throws Exception {
+        Referenceable referenceable = new Referenceable(STORAGE_DESC_TYPE);
+        referenceable.set("location", location);
+        referenceable.set("inputFormat", inputFormat);
+        referenceable.set("outputFormat", outputFormat);
+        referenceable.set("compressed", compressed);
+
+        return referenceable;
+    }
+
+    Referenceable rawColumn(String name, String dataType, String comment,
+                            String... traitNames) throws Exception {
+        Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames);
+        referenceable.set("name", name);
+        referenceable.set("dataType", dataType);
+        referenceable.set("comment", comment);
+
+        return referenceable;
+    }
+
+    Id table(String name, String description,
+             Id dbId, Referenceable sd,
+             String owner, String tableType,
+             List<Referenceable> columns,
+             String... traitNames) throws Exception {
+        Referenceable referenceable = new Referenceable(TABLE_TYPE, traitNames);
+        referenceable.set("name", name);
+        referenceable.set("description", description);
+        referenceable.set("owner", owner);
+        referenceable.set("tableType", tableType);
+        referenceable.set("createTime", System.currentTimeMillis());
+        referenceable.set("lastAccessTime", System.currentTimeMillis());
+        referenceable.set("retention", System.currentTimeMillis());
+        referenceable.set("db", dbId);
+        referenceable.set("sd", sd);
+        referenceable.set("columns", columns);
+
+        return createInstance(referenceable);
+    }
+
+    Id loadProcess(String name, String description, String user,
+                   List<Id> inputTables,
+                   List<Id> outputTables,
+                   String queryText, String queryPlan,
+                   String queryId, String queryGraph,
+                   String... traitNames) throws Exception {
+        Referenceable referenceable = new Referenceable(LOAD_PROCESS_TYPE, traitNames);
+        // super type attributes
+        referenceable.set("name", name);
+        referenceable.set("description", description);
+        referenceable.set("inputs", inputTables);
+        referenceable.set("outputs", outputTables);
+
+        referenceable.set("user", user);
+        referenceable.set("startTime", System.currentTimeMillis());
+        referenceable.set("endTime", System.currentTimeMillis() + 10000);
+
+        referenceable.set("queryText", queryText);
+        referenceable.set("queryPlan", queryPlan);
+        referenceable.set("queryId", queryId);
+        referenceable.set("queryGraph", queryGraph);
+
+        return createInstance(referenceable);
+    }
+
+    Id view(String name, Id dbId,
+            List<Id> inputTables,
+            String... traitNames) throws Exception {
+        Referenceable referenceable = new Referenceable(VIEW_TYPE, traitNames);
+        referenceable.set("name", name);
+        referenceable.set("db", dbId);
+
+        referenceable.set("inputTables", inputTables);
+
+        return createInstance(referenceable);
+    }
+
+    private void verifyTypesCreated() throws Exception {
+        List<String> types = metadataServiceClient.listTypes();
+        for (String type : TYPES) {
+            assert types.contains(type);
+        }
+    }
+
+    private String[] getDSLQueries() {
+        return new String[]{
+            "from DB",
+            "DB",
+            "DB where name=\"Reporting\"",
+            "DB where DB.name=\"Reporting\"",
+            "DB name = \"Reporting\"",
+            "DB DB.name = \"Reporting\"",
+            "DB where name=\"Reporting\" select name, owner",
+            "DB where DB.name=\"Reporting\" select name, owner",
+            "DB has name",
+            "DB where DB has name",
+            "DB, Table",
+            "DB is JdbcAccess",
+            /*
+            "DB, hive_process has name",
+            "DB as db1, Table where db1.name = \"Reporting\"",
+            "DB where DB.name=\"Reporting\" and DB.createTime < " + System.currentTimeMillis()},
+            */
+            "from Table",
+            "Table",
+            "Table is Dimension",
+            "Column where Column isa PII",
+            "View is Dimension",
+            /*"Column where Column isa PII select Column.name",*/
+            "Column select Column.name",
+            "Column select name",
+            "Column where Column.name=\"customer_id\"",
+            "from Table select Table.name",
+            "DB where (name = \"Reporting\")",
+            "DB where (name = \"Reporting\") select name as _col_0, owner as _col_1",
+            "DB where DB is JdbcAccess",
+            "DB where DB has name",
+            "DB Table",
+            "DB where DB has name",
+            "DB as db1 Table where (db1.name = \"Reporting\")",
+            "DB where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 ",
+            /*
+            todo: does not work
+            "DB where (name = \"Reporting\") and ((createTime + 1) > 0)",
+            "DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName",
+            "DB as db1 Table as tab where ((db1.createTime + 1) > 0) or (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName",
+            "DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName",
+            "DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName",
+            */
+            // trait searches
+            "Dimension",
+            /*"Fact", - todo: does not work*/
+            "JdbcAccess",
+            "ETL",
+            "Metric",
+            "PII",
+            /*
+            // Lineage - todo - fix this, its not working
+            "Table hive_process outputTables",
+            "Table loop (hive_process outputTables)",
+            "Table as _loop0 loop (hive_process outputTables) withPath",
+            "Table as src loop (hive_process outputTables) as dest select src.name as srcTable, dest.name as destTable withPath",
+            */
+            "Table where name=\"sales_fact\", columns",
+            "Table where name=\"sales_fact\", columns as column select column.name, column.dataType, column.comment",
+            "from DataSet",
+            "from Process",
+        };
+    }
+
+    private void search() throws Exception {
+        for (String dslQuery : getDSLQueries()) {
+            JSONObject response = metadataServiceClient.searchEntity(dslQuery);
+            JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS);
+            if (!results.isNull("rows")) {
+                JSONArray rows = results.getJSONArray("rows");
+                System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
+            } else {
+                System.out.println("query [" + dslQuery + "] failed, results:" + results.toString());
+            }
+        }
+    }
+}


Mime
View raw message