hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ekoif...@apache.org
Subject hive git commit: HIVE-14004 Minor compaction produces ArrayIndexOutOfBoundsException: 7 in SchemaEvolution.getFileType (Owen O'Malley, reviewed by Prasanth Jayachandran)
Date Fri, 15 Jul 2016 16:44:48 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-1 826909d82 -> 5a71a203f


HIVE-14004 Minor compaction produces ArrayIndexOutOfBoundsException: 7 in SchemaEvolution.getFileType
(Owen O'Malley, reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5a71a203
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5a71a203
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5a71a203

Branch: refs/heads/branch-1
Commit: 5a71a203f2a5e75f87fdc3124541ebe67adf9415
Parents: 826909d
Author: Eugene Koifman <ekoifman@hortonworks.com>
Authored: Fri Jul 15 09:44:41 2016 -0700
Committer: Eugene Koifman <ekoifman@hortonworks.com>
Committed: Fri Jul 15 09:44:41 2016 -0700

----------------------------------------------------------------------
 .../hadoop/hive/ql/io/orc/ReaderImpl.java       |  1 +
 .../apache/hadoop/hive/ql/TestTxnCommands2.java | 28 ++++++++++++++++++--
 2 files changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/5a71a203/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
index 30a3f6a..296544c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
@@ -480,6 +480,7 @@ public class ReaderImpl implements Reader {
     boolean[] include = options.getInclude();
     // if included columns is null, then include all columns
     if (include == null) {
+      options = options.clone();
       include = new boolean[footer.getTypesCount()];
       Arrays.fill(include, true);
       options.include(include);

http://git-wip-us.apache.org/repos/asf/hive/blob/5a71a203/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index b3b5bfd..68f621f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -846,7 +846,7 @@ public class TestTxnCommands2 {
     init.run();
     int numAttemptedCompactions = 1;
     checkCompactionState(new CompactionsByState(numAttemptedCompactions,numFailedCompactions,0,0,0,0,numFailedCompactions
+ numAttemptedCompactions), countCompacts(txnHandler));
-    
+
     hiveConf.setTimeVar(HiveConf.ConfVars.COMPACTOR_HISTORY_REAPER_INTERVAL, 10, TimeUnit.MILLISECONDS);
     AcidCompactionHistoryService compactionHistoryService = new AcidCompactionHistoryService();
     runHouseKeeperService(compactionHistoryService, hiveConf);//should not remove anything
from history
@@ -868,7 +868,7 @@ public class TestTxnCommands2 {
       hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED),
       hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED),0,0,0,0,
       hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED) + hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED)),
countCompacts(txnHandler));
-    
+
     hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILCOMPACTION, false);
     txnHandler.compact(new CompactionRequest("default", tblName, CompactionType.MINOR));
     //at this point "show compactions" should have (COMPACTOR_HISTORY_RETENTION_FAILED) failed
+ 1 initiated (explicitly by user)
@@ -1138,6 +1138,30 @@ public class TestTxnCommands2 {
     Assert.assertNull(exception);
   }
 
+  @Test
+  public void testCompactWithDelete() throws Exception {
+    int[][] tableData = {{1,2},{3,4}};
+    runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData));
+    runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'");
+    Worker t = new Worker();
+    t.setThreadId((int) t.getId());
+    t.setHiveConf(hiveConf);
+    AtomicBoolean stop = new AtomicBoolean();
+    AtomicBoolean looped = new AtomicBoolean();
+    stop.set(true);
+    t.init(stop, looped);
+    t.run();
+    runStatementOnDriver("delete from " + Table.ACIDTBL + " where b = 4");
+    runStatementOnDriver("update " + Table.ACIDTBL + " set b = -2 where b = 2");
+    runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MINOR'");
+    t.run();
+    TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
+    ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest());
+    Assert.assertEquals("Unexpected number of compactions in history", 2, resp.getCompactsSize());
+    Assert.assertEquals("Unexpected 0 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(0).getState());
+    Assert.assertEquals("Unexpected 1 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(1).getState());
+  }
+
   /**
    * takes raw data and turns it into a string as if from Driver.getResults()
    * sorts rows in dictionary order


Mime
View raw message