hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From khorg...@apache.org
Subject [1/3] hive git commit: HIVE-10227 : Concrete implementation of Export/Import based ReplicationTaskFactory (Sushanth Sowmyan, reviewed by Alan Gates, Mithun Radhakrishnan)
Date Thu, 23 Apr 2015 00:35:13 GMT
Repository: hive
Updated Branches:
  refs/heads/master f4d6fdca1 -> 7eaccc3d3


http://git-wip-us.apache.org/repos/asf/hive/blob/7eaccc3d/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
----------------------------------------------------------------------
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
index 8ef5394..91cc03e 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
@@ -263,9 +263,10 @@ public class TestDbNotificationListener {
     assertEquals(HCatConstants.HCAT_ALTER_PARTITION_EVENT, event.getEventType());
     assertEquals("default", event.getDbName());
     assertEquals("alterparttable", event.getTableName());
-    assertTrue(event.getMessage().matches( "\\{\"eventType\":\"ALTER_PARTITION\",\"server\":\"\","
+
+    assertTrue(event.getMessage(),
+        event.getMessage().matches("\\{\"eventType\":\"ALTER_PARTITION\",\"server\":\"\","
+
         "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":\"alterparttable\"," +
-        "\"timestamp\":[0-9]+,\"values\":\\[\"today\"]}"));
+        "\"timestamp\":[0-9]+,\"keyValues\":\\{\"ds\":\"today\"}}"));
   }
 
   @Test
@@ -303,55 +304,81 @@ public class TestDbNotificationListener {
 
   @Test
   public void insertTable() throws Exception {
+    List<FieldSchema> cols = new ArrayList<FieldSchema>();
+    cols.add(new FieldSchema("col1", "int", "nocomment"));
+    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false,
0,
+        serde, null, null, emptyParameters);
+    Table table = new Table("insertTable", "default", "me", startTime, startTime, 0, sd,
null,
+        emptyParameters, null, null, null);
+    msClient.createTable(table);
+
     FireEventRequestData data = new FireEventRequestData();
     InsertEventRequestData insertData = new InsertEventRequestData();
     data.setInsertData(insertData);
     insertData.addToFilesAdded("/warehouse/mytable/b1");
     FireEventRequest rqst = new FireEventRequest(true, data);
-    rqst.setDbName("mydb");
-    rqst.setTableName("mytable");
+    rqst.setDbName("default");
+    rqst.setTableName("insertTable");
     msClient.fireListenerEvent(rqst);
 
     NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
-    assertEquals(1, rsp.getEventsSize());
+    assertEquals(2, rsp.getEventsSize());
 
-    NotificationEvent event = rsp.getEvents().get(0);
-    assertEquals(firstEventId + 1, event.getEventId());
+    NotificationEvent event = rsp.getEvents().get(1);
+    assertEquals(firstEventId + 2, event.getEventId());
     assertTrue(event.getEventTime() >= startTime);
     assertEquals(HCatConstants.HCAT_INSERT_EVENT, event.getEventType());
-    assertEquals("mydb", event.getDbName());
-    assertEquals("mytable", event.getTableName());
-    assertTrue(event.getMessage().matches("\\{\"eventType\":\"INSERT\",\"server\":\"\","
+
-        "\"servicePrincipal\":\"\",\"db\":\"mydb\",\"table\":" +
-        "\"mytable\",\"timestamp\":[0-9]+,\"partitionValues\":null," +
-        "\"files\":\\[\"/warehouse/mytable/b1\"]}"));
+    assertEquals("default", event.getDbName());
+    assertEquals("insertTable", event.getTableName());
+    assertTrue(event.getMessage(),
+        event.getMessage().matches("\\{\"eventType\":\"INSERT\",\"server\":\"\"," +
+        "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":" +
+        "\"insertTable\",\"timestamp\":[0-9]+,\"files\":\\[\"/warehouse/mytable/b1\"]," +
+        "\"partKeyVals\":\\{},\"partitionKeyValues\":\\{}}"));
   }
 
   @Test
   public void insertPartition() throws Exception {
+    List<FieldSchema> cols = new ArrayList<FieldSchema>();
+    cols.add(new FieldSchema("col1", "int", "nocomment"));
+    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
+    partCols.add(new FieldSchema("ds", "string", ""));
+    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false,
0,
+        serde, null, null, emptyParameters);
+    Table table = new Table("insertPartition", "default", "me", startTime, startTime, 0,
sd,
+        partCols, emptyParameters, null, null, null);
+    msClient.createTable(table);
+    Partition partition = new Partition(Arrays.asList("today"), "default", "insertPartition",
+        startTime, startTime, sd, emptyParameters);
+    msClient.add_partition(partition);
+
     FireEventRequestData data = new FireEventRequestData();
     InsertEventRequestData insertData = new InsertEventRequestData();
     data.setInsertData(insertData);
     insertData.addToFilesAdded("/warehouse/mytable/today/b1");
     FireEventRequest rqst = new FireEventRequest(true, data);
-    rqst.setDbName("mydb");
-    rqst.setTableName("mytable");
+    rqst.setDbName("default");
+    rqst.setTableName("insertPartition");
     rqst.setPartitionVals(Arrays.asList("today"));
     msClient.fireListenerEvent(rqst);
 
     NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
-    assertEquals(1, rsp.getEventsSize());
+    assertEquals(3, rsp.getEventsSize());
 
-    NotificationEvent event = rsp.getEvents().get(0);
-    assertEquals(firstEventId + 1, event.getEventId());
+    NotificationEvent event = rsp.getEvents().get(2);
+    assertEquals(firstEventId + 3, event.getEventId());
     assertTrue(event.getEventTime() >= startTime);
     assertEquals(HCatConstants.HCAT_INSERT_EVENT, event.getEventType());
-    assertEquals("mydb", event.getDbName());
-    assertEquals("mytable", event.getTableName());
-    assertTrue(event.getMessage().matches("\\{\"eventType\":\"INSERT\",\"server\":\"\","
+
-        "\"servicePrincipal\":\"\",\"db\":\"mydb\",\"table\":" +
-        "\"mytable\",\"timestamp\":[0-9]+,\"partitionValues\":\\[\"today\"]," +
-        "\"files\":\\[\"/warehouse/mytable/today/b1\"]}"));
+    assertEquals("default", event.getDbName());
+    assertEquals("insertPartition", event.getTableName());
+    assertTrue(event.getMessage(),
+        event.getMessage().matches("\\{\"eventType\":\"INSERT\",\"server\":\"\"," +
+        "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":" +
+        "\"insertPartition\",\"timestamp\":[0-9]+," +
+        "\"files\":\\[\"/warehouse/mytable/today/b1\"],\"partKeyVals\":\\{\"ds\":\"today\"},"
+
+        "\"partitionKeyValues\":\\{\"ds\":\"today\"}}"));
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/hive/blob/7eaccc3d/metastore/src/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java
index be66cbe..2a313b0 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java
@@ -19,13 +19,14 @@
 package org.apache.hadoop.hive.metastore.events;
 
 import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
-import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
 
-import java.util.Arrays;
-import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 
 public class InsertEvent extends ListenerEvent {
 
@@ -33,24 +34,30 @@ public class InsertEvent extends ListenerEvent {
   // we have just the string names, but that's fine for what we need.
   private final String db;
   private final String table;
-  private final List<String> partVals;
+  private final Map<String,String> keyValues;
   private final List<String> files;
 
   /**
    *
    * @param db name of the database the table is in
    * @param table name of the table being inserted into
-   * @param partitions list of partition values, can be null
+   * @param partVals list of partition values, can be null
    * @param status status of insert, true = success, false = failure
    * @param handler handler that is firing the event
    */
-  public InsertEvent(String db, String table, List<String> partitions, List<String>
files,
-                     boolean status, HMSHandler handler) {
+  public InsertEvent(String db, String table, List<String> partVals, List<String>
files,
+                     boolean status, HMSHandler handler) throws MetaException, NoSuchObjectException
{
     super(status, handler);
     this.db = db;
     this.table = table;
-    this.partVals = partitions;
     this.files = files;
+    Table t = handler.get_table(db,table);
+    keyValues = new LinkedHashMap<String, String>();
+    if (partVals != null) {
+      for (int i = 0; i < partVals.size(); i++) {
+        keyValues.put(t.getPartitionKeys().get(i).getName(), partVals.get(i));
+      }
+    }
   }
 
   public String getDb() {
@@ -64,10 +71,10 @@ public class InsertEvent extends ListenerEvent {
   }
 
   /**
-   * @return List of partitions.
+   * @return List of values for the partition keys.
    */
-  public List<String> getPartitions() {
-    return partVals;
+  public Map<String,String> getPartitionKeyValues() {
+    return keyValues;
   }
 
   /**


Mime
View raw message