atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shweth...@apache.org
Subject incubator-atlas git commit: ATLAS-17 Parameterize schema API query per typeName (shwethags)
Date Wed, 17 Jun 2015 08:40:26 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/branch-0.5-incubating bb887fdbf -> e46c19937


ATLAS-17 Parameterize schema API query per typeName (shwethags)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/e46c1993
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/e46c1993
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/e46c1993

Branch: refs/heads/branch-0.5-incubating
Commit: e46c19937ffd373eef9b9b1cd96635718235ab3b
Parents: bb887fd
Author: Shwetha GS <sshivalingamurthy@hortonworks.com>
Authored: Wed Jun 17 14:10:21 2015 +0530
Committer: Shwetha GS <sshivalingamurthy@hortonworks.com>
Committed: Wed Jun 17 14:10:21 2015 +0530

----------------------------------------------------------------------
 release-log.txt                                 | 10 +-----
 .../atlas/discovery/HiveLineageService.java     | 32 ++++++++++++--------
 .../src/test/resources/application.properties   |  2 +-
 src/conf/application.properties                 |  3 +-
 .../src/main/resources/application.properties   |  2 +-
 5 files changed, 24 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e46c1993/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 56be318..0bfac32 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,18 +1,10 @@
 Apache Atlas Release Notes
 ==========================
 
---trunk - unreleased
-
-INCOMPATIBLE CHANGES:
-
-ALL CHANGES:
-
-ATLAS-10 Update trunk version to 0.6-incubating-SNAPSHOT (shwethags)
-
 --Release 0.5-incubating
 
 ALL CHANGES:
-
+ATLAS-17 Parameterize schema API query per typeName (shwethags)
 ATLAS-18 Build fails because of a missing artifact (Ajay Yadava via Venkatesh Seetharam)
 ATLAS-13 Add project website (Venkatesh Seetharam)
 ATLAS-12 Update the copyright in Notice and License files (Venkatesh Seetharam)

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e46c1993/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java
index 9764f8f..76d94a3 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java
@@ -32,6 +32,7 @@ import org.apache.atlas.query.HiveWhereUsedQuery;
 import org.apache.atlas.repository.EntityNotFoundException;
 import org.apache.atlas.repository.MetadataRepository;
 import org.apache.atlas.repository.graph.GraphProvider;
+import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
 import org.apache.commons.configuration.PropertiesConfiguration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -53,26 +54,27 @@ public class HiveLineageService implements LineageService {
     private static final Option<List<String>> SELECT_ATTRIBUTES =
             Some.<List<String>>apply(List.<String>fromArray(new String[]{"name"}));
 
+    public static final String HIVE_TABLE_SCHEMA_QUERY_PREFIX = "atlas.lineage.hive.table.schema.query.";
+
     private static final String HIVE_TABLE_TYPE_NAME;
     private static final String HIVE_PROCESS_TYPE_NAME;
     private static final String HIVE_PROCESS_INPUT_ATTRIBUTE_NAME;
     private static final String HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME;
 
-    private static final String HIVE_TABLE_SCHEMA_QUERY;
     private static final String HIVE_TABLE_EXISTS_QUERY;
 
+    private static final PropertiesConfiguration propertiesConf;
+
     static {
         // todo - externalize this using type system - dog food
         try {
-            PropertiesConfiguration conf = PropertiesUtil.getApplicationProperties();
-            HIVE_TABLE_TYPE_NAME = conf.getString("atlas.lineage.hive.table.type.name", "DataSet");
-            HIVE_PROCESS_TYPE_NAME = conf.getString("atlas.lineage.hive.process.type.name",
"Process");
-            HIVE_PROCESS_INPUT_ATTRIBUTE_NAME = conf.getString("atlas.lineage.hive.process.inputs.name",
"inputs");
-            HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME = conf.getString("atlas.lineage.hive.process.outputs.name",
"outputs");
-
-            HIVE_TABLE_SCHEMA_QUERY =
-                    conf.getString("atlas.lineage.hive.table.schema.query", "hive_table where
name=\"%s\", columns");
-            HIVE_TABLE_EXISTS_QUERY = conf.getString("atlas.lineage.hive.table.exists.query",
+            propertiesConf = PropertiesUtil.getApplicationProperties();
+            HIVE_TABLE_TYPE_NAME = propertiesConf.getString("atlas.lineage.hive.table.type.name",
"DataSet");
+            HIVE_PROCESS_TYPE_NAME = propertiesConf.getString("atlas.lineage.hive.process.type.name",
"Process");
+            HIVE_PROCESS_INPUT_ATTRIBUTE_NAME = propertiesConf.getString("atlas.lineage.hive.process.inputs.name",
"inputs");
+            HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME = propertiesConf.getString("atlas.lineage.hive.process.outputs.name",
"outputs");
+
+            HIVE_TABLE_EXISTS_QUERY = propertiesConf.getString("atlas.lineage.hive.table.exists.query",
                     "from " + HIVE_TABLE_TYPE_NAME + " where name=\"%s\"");
         } catch (AtlasException e) {
             throw new RuntimeException(e);
@@ -195,9 +197,10 @@ public class HiveLineageService implements LineageService {
     public String getSchema(String tableName) throws AtlasException {
         LOG.info("Fetching schema for tableName={}", tableName);
         ParamChecker.notEmpty(tableName, "table name cannot be null");
-        validateTableExists(tableName);
+        String typeName = validateTableExists(tableName);
 
-        final String schemaQuery = String.format(HIVE_TABLE_SCHEMA_QUERY, tableName);
+        final String schemaQuery =
+                String.format(propertiesConf.getString(HIVE_TABLE_SCHEMA_QUERY_PREFIX + typeName),
tableName);
         return discoveryService.searchByDSL(schemaQuery);
     }
 
@@ -206,11 +209,14 @@ public class HiveLineageService implements LineageService {
      *
      * @param tableName table name
      */
-    private void validateTableExists(String tableName) throws AtlasException {
+    private String validateTableExists(String tableName) throws AtlasException {
         final String tableExistsQuery = String.format(HIVE_TABLE_EXISTS_QUERY, tableName);
         GremlinQueryResult queryResult = discoveryService.evaluate(tableExistsQuery);
         if (!(queryResult.rows().length() > 0)) {
             throw new EntityNotFoundException(tableName + " does not exist");
         }
+
+        ReferenceableInstance referenceable = (ReferenceableInstance)queryResult.rows().apply(0);
+        return referenceable.getTypeName();
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e46c1993/repository/src/test/resources/application.properties
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/application.properties b/repository/src/test/resources/application.properties
index 32d40b0..20c0c51 100755
--- a/repository/src/test/resources/application.properties
+++ b/repository/src/test/resources/application.properties
@@ -36,7 +36,7 @@ atlas.graph.index.search.elasticsearch.local-mode=true
 #atlas.lineage.hive.process.outputs.name=outputs
 
 ## Schema
-#atlas.lineage.hive.table.schema.query=hive_table where name=?, columns
+atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns
 
 
 #########  Security Properties  #########

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e46c1993/src/conf/application.properties
----------------------------------------------------------------------
diff --git a/src/conf/application.properties b/src/conf/application.properties
index 5818701..6c4c7d2 100755
--- a/src/conf/application.properties
+++ b/src/conf/application.properties
@@ -36,7 +36,8 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000
 #atlas.lineage.hive.process.outputs.name=outputs
 
 ## Schema
-#atlas.lineage.hive.table.schema.query=hive_table where name=?, columns
+atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns
+atlas.lineage.hive.table.schema.query.Table=Table where name='%s'\, columns
 
 
 #########  Security Properties  #########

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e46c1993/webapp/src/main/resources/application.properties
----------------------------------------------------------------------
diff --git a/webapp/src/main/resources/application.properties b/webapp/src/main/resources/application.properties
index 9ade2a7..c9b8408 100755
--- a/webapp/src/main/resources/application.properties
+++ b/webapp/src/main/resources/application.properties
@@ -36,7 +36,7 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000
 #atlas.lineage.hive.process.outputs.name=outputs
 
 ## Schema
-#atlas.lineage.hive.table.schema.query=hive_table where name=?, columns
+atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns
 
 
 #########  Security Properties  #########


Mime
View raw message