hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1639928 - in /hive/trunk: metastore/if/ metastore/src/gen/thrift/gen-cpp/ metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ metastore/src/gen/thrift/gen-php/metastore/ metastore/src/gen/thrift/gen-py/hive_metastor...
Date Sat, 15 Nov 2014 21:42:25 GMT
Author: hashutosh
Date: Sat Nov 15 21:42:25 2014
New Revision: 1639928

URL: http://svn.apache.org/r1639928
Log:
HIVE-8308 : Acid related table properties should be defined in one place and should be case
insensitive (Alan Gates via Ashutosh Chauhan)

Modified:
    hive/trunk/metastore/if/hive_metastore.thrift
    hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
    hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h
    hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
    hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php
    hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py
    hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java

Modified: hive/trunk/metastore/if/hive_metastore.thrift
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/if/hive_metastore.thrift?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/if/hive_metastore.thrift (original)
+++ hive/trunk/metastore/if/hive_metastore.thrift Sat Nov 15 21:42:25 2014
@@ -1139,6 +1139,7 @@ const string META_TABLE_PARTITION_COLUMN
 const string FILE_INPUT_FORMAT    = "file.inputformat",
 const string FILE_OUTPUT_FORMAT   = "file.outputformat",
 const string META_TABLE_STORAGE   = "storage_handler",
-
+const string TABLE_IS_TRANSACTIONAL = "transactional",
+const string TABLE_NO_AUTO_COMPACT = "no_auto_compaction",
 
 

Modified: hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp Sat Nov 15 21:42:25
2014
@@ -53,6 +53,10 @@ hive_metastoreConstants::hive_metastoreC
 
   META_TABLE_STORAGE = "storage_handler";
 
+  TABLE_IS_TRANSACTIONAL = "transactional";
+
+  TABLE_NO_AUTO_COMPACT = "no_auto_compaction";
+
 }
 
 }}} // namespace

Modified: hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h Sat Nov 15 21:42:25
2014
@@ -36,6 +36,8 @@ class hive_metastoreConstants {
   std::string FILE_INPUT_FORMAT;
   std::string FILE_OUTPUT_FORMAT;
   std::string META_TABLE_STORAGE;
+  std::string TABLE_IS_TRANSACTIONAL;
+  std::string TABLE_NO_AUTO_COMPACT;
 };
 
 extern const hive_metastoreConstants g_hive_metastore_constants;

Modified: hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
(original)
+++ hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
Sat Nov 15 21:42:25 2014
@@ -75,4 +75,8 @@ public class hive_metastoreConstants {
 
   public static final String META_TABLE_STORAGE = "storage_handler";
 
+  public static final String TABLE_IS_TRANSACTIONAL = "transactional";
+
+  public static final String TABLE_NO_AUTO_COMPACT = "no_auto_compaction";
+
 }

Modified: hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php Sat Nov 15 21:42:25 2014
@@ -12890,4 +12890,8 @@ $GLOBALS['hive_metastore_CONSTANTS']['FI
 
 $GLOBALS['hive_metastore_CONSTANTS']['META_TABLE_STORAGE'] = "storage_handler";
 
+$GLOBALS['hive_metastore_CONSTANTS']['TABLE_IS_TRANSACTIONAL'] = "transactional";
+
+$GLOBALS['hive_metastore_CONSTANTS']['TABLE_NO_AUTO_COMPACT'] = "no_auto_compaction";
+
 

Modified: hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py Sat Nov 15 21:42:25
2014
@@ -30,3 +30,5 @@ META_TABLE_PARTITION_COLUMN_TYPES = "par
 FILE_INPUT_FORMAT = "file.inputformat"
 FILE_OUTPUT_FORMAT = "file.outputformat"
 META_TABLE_STORAGE = "storage_handler"
+TABLE_IS_TRANSACTIONAL = "transactional"
+TABLE_NO_AUTO_COMPACT = "no_auto_compaction"

Modified: hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb Sat Nov 15 21:42:25
2014
@@ -49,3 +49,7 @@ FILE_OUTPUT_FORMAT = %q"file.outputforma
 
 META_TABLE_STORAGE = %q"storage_handler"
 
+TABLE_IS_TRANSACTIONAL = %q"transactional"
+
+TABLE_NO_AUTO_COMPACT = %q"no_auto_compaction"
+

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Sat Nov 15
21:42:25 2014
@@ -51,6 +51,7 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.QueryProperties;
 import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator;
@@ -318,9 +319,6 @@ public class SemanticAnalyzer extends Ba
 
   private static final String VALUES_TMP_TABLE_NAME_PREFIX = "Values__Tmp__Table__";
 
-  @VisibleForTesting
-  static final String ACID_TABLE_PROPERTY = "transactional";
-
   private HashMap<TableScanOperator, ExprNodeDesc> opToPartPruner;
   private HashMap<TableScanOperator, PrunedPartitionList> opToPartList;
   private HashMap<String, Operator<? extends OperatorDesc>> topOps;
@@ -12415,7 +12413,9 @@ public class SemanticAnalyzer extends Ba
   private boolean isAcidTable(Table tab) {
     if (tab == null) return false;
     if (!SessionState.get().getTxnMgr().supportsAcid()) return false;
-    return tab.getProperty(ACID_TABLE_PROPERTY) != null;
+    String tableIsTransactional =
+        tab.getProperty(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
+    return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true");
   }
 
   private boolean isAcidOutputFormat(Class<? extends HiveOutputFormat> of) {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java Sat Nov
15 21:42:25 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
 import org.apache.hadoop.hive.metastore.txn.TxnHandler;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
@@ -53,8 +54,6 @@ public class Initiator extends Compactor
   static final private String CLASS_NAME = Initiator.class.getName();
   static final private Log LOG = LogFactory.getLog(CLASS_NAME);
 
-  static final private String NO_COMPACTION = "NO_AUTO_COMPACTION";
-
   private long checkInterval;
 
   @Override
@@ -86,9 +85,8 @@ public class Initiator extends Compactor
             try {
               Table t = resolveTable(ci);
               // check if no compaction set for this table
-              if (t.getParameters().get(NO_COMPACTION) != null) {
-                LOG.info("Table " + tableName(t) + " marked " +  NO_COMPACTION +
-                    " so we will not compact it.");
+              if (noAutoCompactSet(t)) {
+                LOG.info("Table " + tableName(t) + " marked true so we will not compact it.");
                 continue;
               }
 
@@ -279,4 +277,16 @@ public class Initiator extends Compactor
     rqst.setRunas(runAs);
     txnHandler.compact(rqst);
   }
+
+  // Because TABLE_NO_AUTO_COMPACT was originally assumed to be NO_AUTO_COMPACT and then
was moved
+  // to no_auto_compact, we need to check it in both cases.
+  private boolean noAutoCompactSet(Table t) {
+    String noAutoCompact =
+        t.getParameters().get(hive_metastoreConstants.TABLE_NO_AUTO_COMPACT);
+    if (noAutoCompact == null) {
+      noAutoCompact =
+          t.getParameters().get(hive_metastoreConstants.TABLE_NO_AUTO_COMPACT.toUpperCase());
+    }
+    return noAutoCompact != null && noAutoCompact.equalsIgnoreCase("true");
+  }
 }

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
(original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
Sat Nov 15 21:42:25 2014
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FSDataInputS
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.hive.ql.Context;
 import org.apache.hadoop.hive.ql.QueryPlan;
 import org.apache.hadoop.hive.ql.exec.ExplainTask;
@@ -267,7 +268,7 @@ public class TestUpdateDeleteSemanticAna
     // I have to create the tables here (rather than in setup()) because I need the Hive
     // connection, which is conviently created by the semantic analyzer.
     Map<String, String> params = new HashMap<String, String>(1);
-    params.put(SemanticAnalyzer.ACID_TABLE_PROPERTY, "true");
+    params.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true");
     db.createTable("T", Arrays.asList("a", "b"), null, OrcInputFormat.class,
         OrcOutputFormat.class, 2, Arrays.asList("a"), params);
     db.createTable("U", Arrays.asList("a", "b"), Arrays.asList("ds"), OrcInputFormat.class,

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java Sat
Nov 15 21:42:25 2014
@@ -243,6 +243,32 @@ public class TestInitiator extends Compa
   }
 
   @Test
+  public void noCompactWhenNoCompactSetLowerCase() throws Exception {
+    Map<String, String> parameters = new HashMap<String, String>(1);
+    parameters.put("no_auto_compaction", "true");
+    Table t = newTable("default", "ncwncs", false, parameters);
+
+    HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10);
+
+    for (int i = 0; i < 11; i++) {
+      long txnid = openTxn();
+      LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default");
+      comp.setTablename("ncwncs");
+      List<LockComponent> components = new ArrayList<LockComponent>(1);
+      components.add(comp);
+      LockRequest req = new LockRequest(components, "me", "localhost");
+      req.setTxnid(txnid);
+      LockResponse res = txnHandler.lock(req);
+      txnHandler.abortTxn(new AbortTxnRequest(txnid));
+    }
+
+    startInitiator();
+
+    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
+    Assert.assertEquals(0, rsp.getCompactsSize());
+  }
+
+  @Test
   public void noCompactWhenCompactAlreadyScheduled() throws Exception {
     Table t = newTable("default", "ncwcas", false);
 



Mime
View raw message