atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shweth...@apache.org
Subject [3/4] incubator-atlas git commit: ATLAS-128 DSL - Add support for comparisions on list type (suma.shivaprasad via shwethags)
Date Mon, 05 Oct 2015 15:48:34 GMT
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7660c9b2/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
b/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
index db51ae5..0e6913d 100644
--- a/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
@@ -18,32 +18,10 @@
 
 package org.apache.atlas.discovery;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.thinkaurelius.titan.core.util.TitanCleanup;
+import org.apache.atlas.BaseHiveRepositoryTest;
 import org.apache.atlas.RepositoryMetadataModule;
 import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
 import org.apache.atlas.repository.EntityNotFoundException;
-import org.apache.atlas.repository.graph.GraphProvider;
-import org.apache.atlas.services.DefaultMetadataService;
-import org.apache.atlas.typesystem.Referenceable;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.InstanceSerialization;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.persistence.Id;
-import org.apache.atlas.typesystem.types.AttributeDefinition;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.EnumTypeDefinition;
-import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.atlas.typesystem.types.IDataType;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.StructTypeDefinition;
-import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.TypeSystem;
-import org.apache.atlas.typesystem.types.TypeUtils;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONObject;
 import org.testng.Assert;
@@ -60,37 +38,22 @@ import java.util.List;
  * Unit tests for Hive LineageService.
  */
 @Guice(modules = RepositoryMetadataModule.class)
-public class HiveLineageServiceTest {
+public class HiveLineageServiceTest extends BaseHiveRepositoryTest {
 
     @Inject
     private GraphBackedDiscoveryService discoveryService;
 
     @Inject
-    private DefaultMetadataService metadataService;
-
-    @Inject
-    private GraphProvider<TitanGraph> graphProvider;
-
-    @Inject
     private HiveLineageService hiveLineageService;
 
-
     @BeforeClass
     public void setUp() throws Exception {
-        setUpTypes();
-        setupInstances();
-        // TestUtils.dumpGraph(graphProvider.get());
+        super.setUp();
     }
 
     @AfterClass
     public void tearDown() throws Exception {
-        TypeSystem.getInstance().reset();
-        graphProvider.get().shutdown();
-        try {
-            TitanCleanup.clear(graphProvider.get());
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
+        super.tearDown();
     }
 
     @DataProvider(name = "dslQueriesProvider")
@@ -296,248 +259,4 @@ public class HiveLineageServiceTest {
         hiveLineageService.getSchema("blah");
         Assert.fail();
     }
-
-    private void setUpTypes() throws Exception {
-        TypesDef typesDef = createTypeDefinitions();
-        String typesAsJSON = TypesSerialization.toJson(typesDef);
-        metadataService.createType(typesAsJSON);
-    }
-
-    private static final String DATABASE_TYPE = "hive_db";
-    private static final String HIVE_TABLE_TYPE = "hive_table";
-    private static final String COLUMN_TYPE = "hive_column";
-    private static final String HIVE_PROCESS_TYPE = "hive_process";
-    private static final String STORAGE_DESC_TYPE = "StorageDesc";
-    private static final String VIEW_TYPE = "View";
-
-    private TypesDef createTypeDefinitions() {
-        HierarchicalTypeDefinition<ClassType> dbClsDef = TypesUtil
-            .createClassTypeDef(DATABASE_TYPE, null, attrDef("name", DataTypes.STRING_TYPE),
-                attrDef("description", DataTypes.STRING_TYPE), attrDef("locationUri", DataTypes.STRING_TYPE),
-                attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE));
-
-        HierarchicalTypeDefinition<ClassType> storageDescClsDef = TypesUtil
-            .createClassTypeDef(STORAGE_DESC_TYPE, null, attrDef("location", DataTypes.STRING_TYPE),
-                attrDef("inputFormat", DataTypes.STRING_TYPE), attrDef("outputFormat", DataTypes.STRING_TYPE),
-                attrDef("compressed", DataTypes.STRING_TYPE, Multiplicity.REQUIRED, false,
null));
-
-        HierarchicalTypeDefinition<ClassType> columnClsDef = TypesUtil
-            .createClassTypeDef(COLUMN_TYPE, null, attrDef("name", DataTypes.STRING_TYPE),
-                attrDef("dataType", DataTypes.STRING_TYPE), attrDef("comment", DataTypes.STRING_TYPE));
-
-        HierarchicalTypeDefinition<ClassType> tblClsDef = TypesUtil
-            .createClassTypeDef(HIVE_TABLE_TYPE, ImmutableList.of("DataSet"),
-                attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE),
-                attrDef("lastAccessTime", DataTypes.LONG_TYPE), attrDef("tableType", DataTypes.STRING_TYPE),
-                attrDef("temporary", DataTypes.BOOLEAN_TYPE),
-                new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false,
null),
-                // todo - uncomment this, something is broken
-                //                        new AttributeDefinition("sd", STORAGE_DESC_TYPE,
-                //                                Multiplicity.REQUIRED, true, null),
-                new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE),
-                    Multiplicity.COLLECTION, true, null));
-
-        HierarchicalTypeDefinition<ClassType> loadProcessClsDef = TypesUtil
-            .createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableList.of("Process"),
-                attrDef("userName", DataTypes.STRING_TYPE), attrDef("startTime", DataTypes.LONG_TYPE),
-                attrDef("endTime", DataTypes.LONG_TYPE),
-                attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
-                attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
-                attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
-                attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED));
-
-        HierarchicalTypeDefinition<ClassType> viewClsDef = TypesUtil
-            .createClassTypeDef(VIEW_TYPE, null, attrDef("name", DataTypes.STRING_TYPE),
-                new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false,
null),
-                new AttributeDefinition("inputTables", DataTypes.arrayTypeName(HIVE_TABLE_TYPE),
-                    Multiplicity.COLLECTION, false, null));
-
-        HierarchicalTypeDefinition<TraitType> dimTraitDef = TypesUtil.createTraitTypeDef("Dimension",
null);
-
-        HierarchicalTypeDefinition<TraitType> factTraitDef = TypesUtil.createTraitTypeDef("Fact",
null);
-
-        HierarchicalTypeDefinition<TraitType> metricTraitDef = TypesUtil.createTraitTypeDef("Metric",
null);
-
-        HierarchicalTypeDefinition<TraitType> etlTraitDef = TypesUtil.createTraitTypeDef("ETL",
null);
-
-        HierarchicalTypeDefinition<TraitType> piiTraitDef = TypesUtil.createTraitTypeDef("PII",
null);
-
-        HierarchicalTypeDefinition<TraitType> jdbcTraitDef = TypesUtil.createTraitTypeDef("JdbcAccess",
null);
-
-        return TypeUtils.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
-            ImmutableList.of(dimTraitDef, factTraitDef, piiTraitDef, metricTraitDef, etlTraitDef,
jdbcTraitDef),
-            ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef, tblClsDef, loadProcessClsDef,
viewClsDef));
-    }
-
-    AttributeDefinition attrDef(String name, IDataType dT) {
-        return attrDef(name, dT, Multiplicity.OPTIONAL, false, null);
-    }
-
-    AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m) {
-        return attrDef(name, dT, m, false, null);
-    }
-
-    AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m, boolean isComposite,
-        String reverseAttributeName) {
-        Preconditions.checkNotNull(name);
-        Preconditions.checkNotNull(dT);
-        return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName);
-    }
-
-    private void setupInstances() throws Exception {
-        Id salesDB = database("Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
-
-        Referenceable sd =
-            storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat",
"TextOutputFormat", true);
-
-        List<Referenceable> salesFactColumns = ImmutableList
-            .of(column("time_id", "int", "time id"), column("product_id", "int", "product
id"),
-                column("customer_id", "int", "customer id", "PII"),
-                column("sales", "double", "product id", "Metric"));
-
-        Id salesFact = table("sales_fact", "sales fact table", salesDB, sd, "Joe", "Managed",
salesFactColumns, "Fact");
-
-        List<Referenceable> timeDimColumns = ImmutableList
-            .of(column("time_id", "int", "time id"), column("dayOfYear", "int", "day Of Year"),
-                column("weekDay", "int", "week Day"));
-
-        Id timeDim = table("time_dim", "time dimension table", salesDB, sd, "John Doe", "External",
timeDimColumns,
-            "Dimension");
-
-        Id reportingDB =
-            database("Reporting", "reporting database", "Jane BI", "hdfs://host:8000/apps/warehouse/reporting");
-
-        Id salesFactDaily =
-            table("sales_fact_daily_mv", "sales fact daily materialized view", reportingDB,
sd, "Joe BI", "Managed",
-                salesFactColumns, "Metric");
-
-        loadProcess("loadSalesDaily", "hive query for daily summary", "John ETL", ImmutableList.of(salesFact,
timeDim),
-            ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph",
"ETL");
-
-        List<Referenceable> productDimColumns = ImmutableList
-            .of(column("product_id", "int", "product id"), column("product_name", "string",
"product name"),
-                column("brand_name", "int", "brand name"));
-
-        Id productDim =
-            table("product_dim", "product dimension table", salesDB, sd, "John Doe", "Managed",
productDimColumns,
-                "Dimension");
-
-        view("product_dim_view", reportingDB, ImmutableList.of(productDim), "Dimension",
"JdbcAccess");
-
-        List<Referenceable> customerDimColumns = ImmutableList.of(column("customer_id",
"int", "customer id", "PII"),
-            column("name", "string", "customer name", "PII"),
-            column("address", "string", "customer address", "PII"));
-
-        Id customerDim =
-            table("customer_dim", "customer dimension table", salesDB, sd, "fetl", "External",
customerDimColumns,
-                "Dimension");
-
-        view("customer_dim_view", reportingDB, ImmutableList.of(customerDim), "Dimension",
"JdbcAccess");
-
-        Id salesFactMonthly =
-            table("sales_fact_monthly_mv", "sales fact monthly materialized view", reportingDB,
sd, "Jane BI",
-                "Managed", salesFactColumns, "Metric");
-
-        loadProcess("loadSalesMonthly", "hive query for monthly summary", "John ETL", ImmutableList.of(salesFactDaily),
-            ImmutableList.of(salesFactMonthly), "create table as select ", "plan", "id",
"graph", "ETL");
-    }
-
-    Id database(String name, String description, String owner, String locationUri, String...
traitNames)
-        throws Exception {
-        Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames);
-        referenceable.set("name", name);
-        referenceable.set("description", description);
-        referenceable.set("owner", owner);
-        referenceable.set("locationUri", locationUri);
-        referenceable.set("createTime", System.currentTimeMillis());
-
-        return createInstance(referenceable);
-    }
-
-    Referenceable storageDescriptor(String location, String inputFormat, String outputFormat,
boolean compressed)
-        throws Exception {
-        Referenceable referenceable = new Referenceable(STORAGE_DESC_TYPE);
-        referenceable.set("location", location);
-        referenceable.set("inputFormat", inputFormat);
-        referenceable.set("outputFormat", outputFormat);
-        referenceable.set("compressed", compressed);
-
-        return referenceable;
-    }
-
-    Referenceable column(String name, String dataType, String comment, String... traitNames)
throws Exception {
-        Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames);
-        referenceable.set("name", name);
-        referenceable.set("dataType", dataType);
-        referenceable.set("comment", comment);
-
-        return referenceable;
-    }
-
-    Id table(String name, String description, Id dbId, Referenceable sd, String owner, String
tableType,
-        List<Referenceable> columns, String... traitNames) throws Exception {
-        Referenceable referenceable = new Referenceable(HIVE_TABLE_TYPE, traitNames);
-        referenceable.set("name", name);
-        referenceable.set("description", description);
-        referenceable.set("owner", owner);
-        referenceable.set("tableType", tableType);
-        referenceable.set("temporary", false);
-        referenceable.set("createTime", System.currentTimeMillis());
-        referenceable.set("lastAccessTime", System.currentTimeMillis());
-        referenceable.set("retention", System.currentTimeMillis());
-
-        referenceable.set("db", dbId);
-        // todo - uncomment this, something is broken
-        // referenceable.set("sd", sd);
-        referenceable.set("columns", columns);
-
-        return createInstance(referenceable);
-    }
-
-    Id loadProcess(String name, String description, String user, List<Id> inputTables,
List<Id> outputTables,
-        String queryText, String queryPlan, String queryId, String queryGraph, String...
traitNames)
-        throws Exception {
-        Referenceable referenceable = new Referenceable(HIVE_PROCESS_TYPE, traitNames);
-        referenceable.set("name", name);
-        referenceable.set("description", description);
-        referenceable.set("user", user);
-        referenceable.set("startTime", System.currentTimeMillis());
-        referenceable.set("endTime", System.currentTimeMillis() + 10000);
-
-        referenceable.set("inputs", inputTables);
-        referenceable.set("outputs", outputTables);
-
-        referenceable.set("queryText", queryText);
-        referenceable.set("queryPlan", queryPlan);
-        referenceable.set("queryId", queryId);
-        referenceable.set("queryGraph", queryGraph);
-
-        return createInstance(referenceable);
-    }
-
-    Id view(String name, Id dbId, List<Id> inputTables, String... traitNames) throws
Exception {
-        Referenceable referenceable = new Referenceable(VIEW_TYPE, traitNames);
-        referenceable.set("name", name);
-        referenceable.set("db", dbId);
-
-        referenceable.set("inputTables", inputTables);
-
-        return createInstance(referenceable);
-    }
-
-    private Id createInstance(Referenceable referenceable) throws Exception {
-        String typeName = referenceable.getTypeName();
-        System.out.println("creating instance of type " + typeName);
-
-        String entityJSON = InstanceSerialization.toJson(referenceable, true);
-        System.out.println("Submitting new entity= " + entityJSON);
-        JSONArray jsonArray = new JSONArray();
-        jsonArray.put(entityJSON);
-        String response = metadataService.createEntities(jsonArray.toString());
-        String guid = new JSONArray(response).getString(0);
-        System.out.println("created instance for type " + typeName + ", guid: " + guid);
-
-        // return the reference to created instance with guid
-        return new Id(guid, 0, referenceable.getTypeName());
-    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7660c9b2/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
index bec3067..39be47d 100755
--- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
@@ -7,7 +7,7 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -101,12 +101,12 @@ public class GraphBackedMetadataRepositoryTest {
         try {
             //TODO - Fix failure during shutdown while using BDB
             graphProvider.get().shutdown();
-        } catch(Exception e) {
+        } catch (Exception e) {
             e.printStackTrace();
         }
         try {
             TitanCleanup.clear(graphProvider.get());
-        } catch(Exception e) {
+        } catch (Exception e) {
             e.printStackTrace();
         }
     }
@@ -153,8 +153,8 @@ public class GraphBackedMetadataRepositoryTest {
     @Test(dependsOnMethods = "testSubmitEntity")
     public void testGetTraitLabel() throws Exception {
         Assert.assertEquals(
-                repositoryService.getTraitLabel(typeSystem.getDataType(ClassType.class, TestUtils.TABLE_TYPE),
-                        TestUtils.CLASSIFICATION), TestUtils.TABLE_TYPE + "." + TestUtils.CLASSIFICATION);
+            repositoryService.getTraitLabel(typeSystem.getDataType(ClassType.class, TestUtils.TABLE_TYPE),
+                TestUtils.CLASSIFICATION), TestUtils.TABLE_TYPE + "." + TestUtils.CLASSIFICATION);
     }
 
     @Test
@@ -264,8 +264,8 @@ public class GraphBackedMetadataRepositoryTest {
         final String traitName = "P_I_I";
 
         HierarchicalTypeDefinition<TraitType> piiTrait = TypesUtil
-                .createTraitTypeDef(traitName, ImmutableList.<String>of(),
-                        TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
+            .createTraitTypeDef(traitName, ImmutableList.<String>of(),
+                TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
         TraitType traitType = typeSystem.defineTraitType(piiTrait);
         ITypedStruct traitInstance = traitType.createInstance();
         traitInstance.set("type", "SSN");
@@ -414,7 +414,7 @@ public class GraphBackedMetadataRepositoryTest {
     @Test(dependsOnMethods = "testCreateEntity")
     public void testBug37860() throws Exception {
         String dslQuery = "hive_table as t where name = 'bar' "
-                + "database where name = 'foo' and description = 'foo database' select t";
+            + "database where name = 'foo' and description = 'foo database' select t";
         System.out.println("Executing dslQuery = " + dslQuery);
         String jsonResults = discoveryService.searchByDSL(dslQuery);
         Assert.assertNotNull(jsonResults);

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7660c9b2/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
b/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
index 20781db..85ab738 100755
--- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
@@ -40,6 +40,7 @@ import org.apache.atlas.typesystem.types.IDataType;
 import org.apache.atlas.typesystem.types.Multiplicity;
 import org.apache.atlas.typesystem.types.TypeSystem;
 import org.apache.commons.io.FileUtils;
+import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
@@ -147,7 +148,7 @@ public class GraphRepoMapperScaleTest {
 
         searchWithOutIndex("hive_table.name", "bar-999");
         searchWithIndex("hive_table.name", "bar-999");
-        searchWithIndex("hive_table.created", Compare.GREATER_THAN_EQUAL, TestUtils.TEST_DATE_IN_LONG);
+        searchWithIndex("hive_table.created", Compare.GREATER_THAN_EQUAL, TestUtils.TEST_DATE_IN_LONG,
1000);
 
         for (int index = 500; index < 600; index++) {
             searchWithIndex("hive_table.name", "bar-" + index);
@@ -185,7 +186,7 @@ public class GraphRepoMapperScaleTest {
         }
     }
 
-    private void searchWithIndex(String key, Predicate searchPredicate, Object value) {
+    private void  searchWithIndex(String key, Predicate searchPredicate, Object value, int
expectedResults) {
         TitanGraph graph = graphProvider.get();
         long start = System.currentTimeMillis();
         int count = 0;
@@ -197,6 +198,7 @@ public class GraphRepoMapperScaleTest {
         } finally {
             System.out.println("Search on [" + key + "=" + value + "] returned results: "
+ count + ", took " + (
                     System.currentTimeMillis() - start) + " ms");
+            Assert.assertEquals(count, expectedResults);
         }
     }
 


Mime
View raw message