hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From khorg...@apache.org
Subject svn commit: r1655185 [1/6] - in /hive/trunk: hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/ hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/ hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/mess...
Date Tue, 27 Jan 2015 22:54:04 GMT
Author: khorgath
Date: Tue Jan 27 22:54:03 2015
New Revision: 1655185

URL: http://svn.apache.org/r1655185
Log:
HIVE-9271 : Add ability for client to request metastore to fire an event (Alan Gates via Sushanth
Sowmyan)

Added:
    hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/InsertMessage.java
    hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONInsertMessage.java
    hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/EventRequestType.java
    hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FireEventRequest.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java
Modified:
    hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatConstants.java
    hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
    hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java
    hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/HCatEventMessage.java
    hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java
    hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java
    hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
    hive/trunk/metastore/if/hive_metastore.thrift
    hive/trunk/metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
    hive/trunk/metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
    hive/trunk/metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp
    hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
    hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
    hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
    hive/trunk/metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
    hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php
    hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote
    hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
    hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
    hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
    hive/trunk/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java

Modified: hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatConstants.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatConstants.java?rev=1655185&r1=1655184&r2=1655185&view=diff
==============================================================================
--- hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatConstants.java
(original)
+++ hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatConstants.java
Tue Jan 27 22:54:03 2015
@@ -152,6 +152,7 @@ public final class HCatConstants {
   public static final String HCAT_DROP_TABLE_EVENT = "DROP_TABLE";
   public static final String HCAT_CREATE_DATABASE_EVENT = "CREATE_DATABASE";
   public static final String HCAT_DROP_DATABASE_EVENT = "DROP_DATABASE";
+  public static final String HCAT_INSERT_EVENT = "INSERT";
   public static final String HCAT_MESSAGE_VERSION = "HCAT_MESSAGE_VERSION";
   public static final String HCAT_MESSAGE_FORMAT = "HCAT_MESSAGE_FORMAT";
   public static final String CONF_LABEL_HCAT_MESSAGE_FACTORY_IMPL_PREFIX = "hcatalog.message.factory.impl.";

Modified: hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java?rev=1655185&r1=1655184&r2=1655185&view=diff
==============================================================================
--- hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
(original)
+++ hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
Tue Jan 27 22:54:03 2015
@@ -38,6 +38,7 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
 import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
 import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hadoop.hive.metastore.events.InsertEvent;
 import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent;
 import org.apache.hive.hcatalog.common.HCatConstants;
 import org.apache.hive.hcatalog.messaging.MessageFactory;
@@ -227,6 +228,16 @@ public class DbNotificationListener exte
     enqueue(event);
   }
 
+  @Override
+  public void onInsert(InsertEvent insertEvent) throws MetaException {
+    NotificationEvent event = new NotificationEvent(0, now(), HCatConstants.HCAT_INSERT_EVENT,
+        msgFactory.buildInsertMessage(insertEvent.getDb(), insertEvent.getTable(), insertEvent
+            .getPartitions()).toString());
+    event.setDbName(insertEvent.getDb());
+    event.setTableName(insertEvent.getTable());
+    enqueue(event);
+  }
+
   /**
    * @param partSetDoneEvent
    * @throws MetaException

Modified: hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java?rev=1655185&r1=1655184&r2=1655185&view=diff
==============================================================================
--- hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java
(original)
+++ hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java
Tue Jan 27 22:54:03 2015
@@ -57,6 +57,7 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
 import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
 import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hadoop.hive.metastore.events.InsertEvent;
 import org.apache.hadoop.hive.metastore.events.ListenerEvent;
 import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent;
 import org.apache.hive.hcatalog.common.HCatConstants;

Modified: hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/HCatEventMessage.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/HCatEventMessage.java?rev=1655185&r1=1655184&r2=1655185&view=diff
==============================================================================
--- hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/HCatEventMessage.java
(original)
+++ hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/HCatEventMessage.java
Tue Jan 27 22:54:03 2015
@@ -39,7 +39,8 @@ public abstract class HCatEventMessage {
     ADD_PARTITION(HCatConstants.HCAT_ADD_PARTITION_EVENT),
     DROP_PARTITION(HCatConstants.HCAT_DROP_PARTITION_EVENT),
     ALTER_TABLE(HCatConstants.HCAT_ALTER_TABLE_EVENT),
-    ALTER_PARTITION(HCatConstants.HCAT_ALTER_PARTITION_EVENT);
+    ALTER_PARTITION(HCatConstants.HCAT_ALTER_PARTITION_EVENT),
+    INSERT(HCatConstants.HCAT_INSERT_EVENT);
 
     private String typeString;
 

Added: hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/InsertMessage.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/InsertMessage.java?rev=1655185&view=auto
==============================================================================
--- hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/InsertMessage.java
(added)
+++ hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/InsertMessage.java
Tue Jan 27 22:54:03 2015
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hive.hcatalog.messaging;
+
+import java.util.List;
+
+/**
+ * HCat message sent when an insert is done to a table or partition.
+ */
+public abstract class InsertMessage extends HCatEventMessage {
+
+  protected InsertMessage() {
+    super(EventType.INSERT);
+  }
+
+  /**
+   * Getter for the name of the table being insert into.
+   * @return Table-name (String).
+   */
+  public abstract String getTable();
+
+  /**
+   * Get the list of partition values.  Will be null if this insert is to a table and not
a
+   * partition.
+   * @return List of partition values, or null.
+   */
+  public abstract List<String> getPartitionValues();
+
+  @Override
+  public HCatEventMessage checkValid() {
+    if (getTable() == null)
+      throw new IllegalStateException("Table name unset.");
+    return super.checkValid();
+  }
+}

Modified: hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java?rev=1655185&r1=1655184&r2=1655185&view=diff
==============================================================================
--- hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java
(original)
+++ hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java
Tue Jan 27 22:54:03 2015
@@ -170,4 +170,6 @@ public abstract class MessageFactory {
    * @return DropPartitionMessage instance.
    */
   public abstract DropPartitionMessage buildDropPartitionMessage(Table table, Partition partition);
+
+  public abstract InsertMessage buildInsertMessage(String db, String table, List<String>
partVals);
 }

Added: hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONInsertMessage.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONInsertMessage.java?rev=1655185&view=auto
==============================================================================
--- hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONInsertMessage.java
(added)
+++ hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONInsertMessage.java
Tue Jan 27 22:54:03 2015
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hive.hcatalog.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.DropTableMessage;
+import org.apache.hive.hcatalog.messaging.InsertMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+
+/**
+ * JSON implementation of DropTableMessage.
+ */
+public class JSONInsertMessage extends InsertMessage {
+
+  @JsonProperty
+  String server, servicePrincipal, db, table;
+
+  @JsonProperty
+  Long timestamp;
+
+  @JsonProperty
+  List<String> partitionValues;
+
+  /**
+   * Default constructor, needed for Jackson.
+   */
+  public JSONInsertMessage() {}
+
+  public JSONInsertMessage(String server, String servicePrincipal, String db, String table,
+                           List<String> partVals, Long timestamp) {
+    this.server = server;
+    this.servicePrincipal = servicePrincipal;
+    this.db = db;
+    this.table = table;
+    this.timestamp = timestamp;
+    partitionValues = partVals;
+    checkValid();
+  }
+
+
+  @Override
+  public String getTable() { return table; }
+
+  @Override
+  public String getServer() { return server; }
+
+  @Override
+  public List<String> getPartitionValues() {
+    return partitionValues;
+  }
+
+  @Override
+  public String getServicePrincipal() { return servicePrincipal; }
+
+  @Override
+  public String getDB() { return db; }
+
+  @Override
+  public Long getTimestamp() { return timestamp; }
+
+  @Override
+  public String toString() {
+    try {
+      return JSONMessageDeserializer.mapper.writeValueAsString(this);
+    }
+    catch (Exception exception) {
+      throw new IllegalArgumentException("Could not serialize: ", exception);
+    }
+  }
+
+}
\ No newline at end of file

Modified: hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java?rev=1655185&r1=1655184&r2=1655185&view=diff
==============================================================================
--- hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java
(original)
+++ hive/trunk/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java
Tue Jan 27 22:54:03 2015
@@ -35,6 +35,7 @@ import org.apache.hive.hcatalog.messagin
 import org.apache.hive.hcatalog.messaging.DropDatabaseMessage;
 import org.apache.hive.hcatalog.messaging.DropPartitionMessage;
 import org.apache.hive.hcatalog.messaging.DropTableMessage;
+import org.apache.hive.hcatalog.messaging.InsertMessage;
 import org.apache.hive.hcatalog.messaging.MessageDeserializer;
 import org.apache.hive.hcatalog.messaging.MessageFactory;
 
@@ -122,6 +123,12 @@ public class JSONMessageFactory extends
         partition.getTableName(), Arrays.asList(getPartitionKeyValues(table, partition)),
now());
   }
 
+  @Override
+  public InsertMessage buildInsertMessage(String db, String table, List<String> partVals)
{
+    return new JSONInsertMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db, table, partVals,
+        now());
+  }
+
   private long now() {
     return System.currentTimeMillis() / 1000;
   }

Modified: hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java?rev=1655185&r1=1655184&r2=1655185&view=diff
==============================================================================
--- hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
(original)
+++ hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
Tue Jan 27 22:54:03 2015
@@ -30,7 +30,9 @@ import org.apache.hadoop.hive.conf.HiveC
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.IMetaStoreClient;
 import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.EventRequestType;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.FireEventRequest;
 import org.apache.hadoop.hive.metastore.api.NotificationEvent;
 import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
 import org.apache.hadoop.hive.metastore.api.Partition;
@@ -293,6 +295,49 @@ public class TestDbNotificationListener
   }
 
   @Test
+  public void insertTable() throws Exception {
+    FireEventRequest rqst = new FireEventRequest(EventRequestType.INSERT, "mydb", true);
+    rqst.setTableName("mytable");
+    msClient.fireNotificationEvent(rqst);
+
+    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
+    assertEquals(1, rsp.getEventsSize());
+
+    NotificationEvent event = rsp.getEvents().get(0);
+    assertEquals(firstEventId + 1, event.getEventId());
+    assertTrue(event.getEventTime() >= startTime);
+    assertEquals(HCatConstants.HCAT_INSERT_EVENT, event.getEventType());
+    assertEquals("mydb", event.getDbName());
+    assertEquals("mytable", event.getTableName());
+    System.out.println(event.getMessage());
+    assertTrue(event.getMessage().matches("\\{\"eventType\":\"INSERT\",\"server\":\"\","
+
+        "\"servicePrincipal\":\"\",\"db\":\"mydb\",\"table\":" +
+        "\"mytable\",\"timestamp\":[0-9]+,\"partitionValues\":null}"));
+  }
+
+  @Test
+  public void insertPartition() throws Exception {
+    FireEventRequest rqst = new FireEventRequest(EventRequestType.INSERT, "mydb", true);
+    rqst.setTableName("mytable");
+    rqst.setPartitionVals(Arrays.asList("today"));
+    msClient.fireNotificationEvent(rqst);
+
+    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
+    assertEquals(1, rsp.getEventsSize());
+
+    NotificationEvent event = rsp.getEvents().get(0);
+    assertEquals(firstEventId + 1, event.getEventId());
+    assertTrue(event.getEventTime() >= startTime);
+    assertEquals(HCatConstants.HCAT_INSERT_EVENT, event.getEventType());
+    assertEquals("mydb", event.getDbName());
+    assertEquals("mytable", event.getTableName());
+    System.out.println(event.getMessage());
+    assertTrue(event.getMessage().matches("\\{\"eventType\":\"INSERT\",\"server\":\"\","
+
+        "\"servicePrincipal\":\"\",\"db\":\"mydb\",\"table\":" +
+        "\"mytable\",\"timestamp\":[0-9]+,\"partitionValues\":\\[\"today\"]}"));
+  }
+
+  @Test
   public void getOnlyMaxEvents() throws Exception {
     Database db = new Database("db1", "no description", "file:/tmp", emptyParameters);
     msClient.createDatabase(db);

Modified: hive/trunk/metastore/if/hive_metastore.thrift
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/if/hive_metastore.thrift?rev=1655185&r1=1655184&r2=1655185&view=diff
==============================================================================
--- hive/trunk/metastore/if/hive_metastore.thrift (original)
+++ hive/trunk/metastore/if/hive_metastore.thrift Tue Jan 27 22:54:03 2015
@@ -106,6 +106,15 @@ enum GrantRevokeType {
     REVOKE = 2,
 }
 
+// Types of events the client can request that the metastore fire.  For now just support
DML operations, as the metastore knows
+// about DDL operations and there's no reason for the client to request such an event.
+enum EventRequestType {
+    INSERT = 1,
+    UPDATE = 2,
+    DELETE = 3,
+}
+
+
 struct HiveObjectRef{
   1: HiveObjectType objectType,
   2: string dbName,
@@ -664,6 +673,14 @@ struct CurrentNotificationEventId {
     1: required i64 eventId,
 }
 
+struct FireEventRequest {
+    1: required EventRequestType eventType,
+    2: required string dbName,
+    3: required bool successful,
+    4: optional string tableName,
+    5: optional list<string> partitionVals
+}
+
 
 exception MetaException {
   1: string message
@@ -1134,6 +1151,7 @@ service ThriftHiveMetastore extends fb30
   // Notification logging calls
   NotificationEventResponse get_next_notification(1:NotificationEventRequest rqst) 
   CurrentNotificationEventId get_current_notificationEventId()
+  void fire_notification_event(1:FireEventRequest rqst)
 }
 
 // * Note about the DDL_TIME: When creating or altering a table or a partition,



Mime
View raw message