hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject [01/16] hive git commit: HIVE-11890. Create ORC submodue. (omalley reviewed by prasanthj)
Date Fri, 11 Dec 2015 23:27:58 GMT
Repository: hive
Updated Branches:
  refs/heads/master 3e3d966f3 -> 9c7a78ee3


http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestMemoryManager.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestMemoryManager.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestMemoryManager.java
deleted file mode 100644
index 19aaff3..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestMemoryManager.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.hamcrest.BaseMatcher;
-import org.hamcrest.Description;
-import org.junit.Test;
-
-import java.lang.management.ManagementFactory;
-
-import static junit.framework.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.doubleThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-/**
- * Test the ORC memory manager.
- */
-public class TestMemoryManager {
-  private static final double ERROR = 0.000001;
-
-  private static class NullCallback implements MemoryManager.Callback {
-    public boolean checkMemory(double newScale) {
-      return false;
-    }
-  }
-
-  @Test
-  public void testBasics() throws Exception {
-    Configuration conf = new Configuration();
-    MemoryManager mgr = new MemoryManager(conf);
-    NullCallback callback = new NullCallback();
-    long poolSize = mgr.getTotalMemoryPool();
-    assertEquals(Math.round(ManagementFactory.getMemoryMXBean().
-        getHeapMemoryUsage().getMax() * 0.5d), poolSize);
-    assertEquals(1.0, mgr.getAllocationScale(), 0.00001);
-    mgr.addWriter(new Path("p1"), 1000, callback);
-    assertEquals(1.0, mgr.getAllocationScale(), 0.00001);
-    mgr.addWriter(new Path("p1"), poolSize / 2, callback);
-    assertEquals(1.0, mgr.getAllocationScale(), 0.00001);
-    mgr.addWriter(new Path("p2"), poolSize / 2, callback);
-    assertEquals(1.0, mgr.getAllocationScale(), 0.00001);
-    mgr.addWriter(new Path("p3"), poolSize / 2, callback);
-    assertEquals(0.6666667, mgr.getAllocationScale(), 0.00001);
-    mgr.addWriter(new Path("p4"), poolSize / 2, callback);
-    assertEquals(0.5, mgr.getAllocationScale(), 0.000001);
-    mgr.addWriter(new Path("p4"), 3 * poolSize / 2, callback);
-    assertEquals(0.3333333, mgr.getAllocationScale(), 0.000001);
-    mgr.removeWriter(new Path("p1"));
-    mgr.removeWriter(new Path("p2"));
-    assertEquals(0.5, mgr.getAllocationScale(), 0.00001);
-    mgr.removeWriter(new Path("p4"));
-    assertEquals(1.0, mgr.getAllocationScale(), 0.00001);
-  }
-
-  @Test
-  public void testConfig() throws Exception {
-    Configuration conf = new Configuration();
-    conf.set("hive.exec.orc.memory.pool", "0.9");
-    MemoryManager mgr = new MemoryManager(conf);
-    long mem =
-        ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
-    System.err.print("Memory = " + mem);
-    long pool = mgr.getTotalMemoryPool();
-    assertTrue("Pool too small: " + pool, mem * 0.899 < pool);
-    assertTrue("Pool too big: " + pool, pool < mem * 0.901);
-  }
-
-  private static class DoubleMatcher extends BaseMatcher<Double> {
-    final double expected;
-    final double error;
-    DoubleMatcher(double expected, double error) {
-      this.expected = expected;
-      this.error = error;
-    }
-
-    @Override
-    public boolean matches(Object val) {
-      double dbl = (Double) val;
-      return Math.abs(dbl - expected) <= error;
-    }
-
-    @Override
-    public void describeTo(Description description) {
-      description.appendText("not sufficiently close to ");
-      description.appendText(Double.toString(expected));
-    }
-  }
-
-  private static DoubleMatcher closeTo(double value, double error) {
-    return new DoubleMatcher(value, error);
-  }
-
-  @Test
-  public void testCallback() throws Exception {
-    Configuration conf = new Configuration();
-    MemoryManager mgr = new MemoryManager(conf);
-    long pool = mgr.getTotalMemoryPool();
-    MemoryManager.Callback[] calls = new MemoryManager.Callback[20];
-    for(int i=0; i < calls.length; ++i) {
-      calls[i] = mock(MemoryManager.Callback.class);
-      mgr.addWriter(new Path(Integer.toString(i)), pool/4, calls[i]);
-    }
-    // add enough rows to get the memory manager to check the limits
-    for(int i=0; i < 10000; ++i) {
-      mgr.addedRow(1);
-    }
-    for(int call=0; call < calls.length; ++call) {
-      verify(calls[call], times(2))
-          .checkMemory(doubleThat(closeTo(0.2, ERROR)));
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
index 0f606a4..2fd13c7 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
@@ -29,12 +29,12 @@ import java.util.Random;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.io.orc.OrcFile.EncodingStrategy;
 import org.apache.hadoop.hive.serde2.io.TimestampWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
+import org.apache.orc.CompressionKind;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -49,16 +49,16 @@ import com.google.common.primitives.Longs;
 @RunWith(value = Parameterized.class)
 public class TestNewIntegerEncoding {
 
-  private EncodingStrategy encodingStrategy;
+  private OrcFile.EncodingStrategy encodingStrategy;
 
-  public TestNewIntegerEncoding(EncodingStrategy es) {
+  public TestNewIntegerEncoding( OrcFile.EncodingStrategy es) {
     this.encodingStrategy = es;
   }
 
   @Parameters
   public static Collection<Object[]> data() {
-    Object[][] data = new Object[][] { { EncodingStrategy.COMPRESSION },
-        { EncodingStrategy.SPEED } };
+    Object[][] data = new Object[][] { {  OrcFile.EncodingStrategy.COMPRESSION },
+        {  OrcFile.EncodingStrategy.SPEED } };
     return Arrays.asList(data);
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
index 2992f3c..ebe3096 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
@@ -38,12 +38,12 @@ import java.util.List;
 import java.util.Map;
 import java.util.Random;
 
+import com.google.common.primitives.Longs;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.type.HiveDecimal;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.io.orc.OrcFile.Version;
 import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
@@ -77,6 +77,21 @@ import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hive.common.util.HiveTestUtils;
+import org.apache.orc.BinaryColumnStatistics;
+import org.apache.orc.BooleanColumnStatistics;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.DecimalColumnStatistics;
+import org.apache.orc.DoubleColumnStatistics;
+import org.apache.orc.IntegerColumnStatistics;
+import org.apache.orc.impl.MemoryManager;
+import org.apache.orc.OrcProto;
+
+import org.apache.orc.OrcUtils;
+import org.apache.orc.StringColumnStatistics;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.StripeStatistics;
+import org.apache.orc.TypeDescription;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -1794,7 +1809,7 @@ public class TestOrcFile {
     }
 
     @Override
-    void addWriter(Path path, long requestedAllocation,
+    public void addWriter(Path path, long requestedAllocation,
                    MemoryManager.Callback callback) {
       this.path = path;
       this.lastAllocation = requestedAllocation;
@@ -1802,23 +1817,23 @@ public class TestOrcFile {
     }
 
     @Override
-    synchronized void removeWriter(Path path) {
+    public synchronized void removeWriter(Path path) {
       this.path = null;
       this.lastAllocation = 0;
     }
 
     @Override
-    long getTotalMemoryPool() {
+    public long getTotalMemoryPool() {
       return totalSpace;
     }
 
     @Override
-    double getAllocationScale() {
+    public double getAllocationScale() {
       return rate;
     }
 
     @Override
-    void addedRow(int count) throws IOException {
+    public void addedRow(int count) throws IOException {
       rows += count;
       if (rows % 100 == 0) {
         callback.checkMemory(rate);
@@ -1843,7 +1858,7 @@ public class TestOrcFile {
                                          .bufferSize(100)
                                          .rowIndexStride(0)
                                          .memory(memory)
-                                         .version(Version.V_0_11));
+                                         .version(OrcFile.Version.V_0_11));
     assertEquals(testFilePath, memory.path);
     for(int i=0; i < 2500; ++i) {
       writer.addRow(new InnerStruct(i*300, Integer.toHexString(10*i)));
@@ -1879,7 +1894,7 @@ public class TestOrcFile {
                                          .bufferSize(100)
                                          .rowIndexStride(0)
                                          .memory(memory)
-                                         .version(Version.V_0_12));
+                                         .version(OrcFile.Version.V_0_12));
     assertEquals(testFilePath, memory.path);
     for(int i=0; i < 2500; ++i) {
       writer.addRow(new InnerStruct(i*300, Integer.toHexString(10*i)));
@@ -1986,4 +2001,36 @@ public class TestOrcFile {
     assertTrue(!rows.hasNext());
     assertEquals(3500, rows.getRowNumber());
   }
+
+  @Test
+  public void testBitPack64Large() throws Exception {
+    ObjectInspector inspector;
+    synchronized (TestOrcFile.class) {
+      inspector = ObjectInspectorFactory.getReflectionObjectInspector(Long.class,
+          ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+    }
+
+    int size = 1080832;
+    long[] inp = new long[size];
+    Random rand = new Random(1234);
+    for (int i = 0; i < size; i++) {
+      inp[i] = rand.nextLong();
+    }
+    List<Long> input = Lists.newArrayList(Longs.asList(inp));
+
+    Writer writer = OrcFile.createWriter(testFilePath,
+        OrcFile.writerOptions(conf).inspector(inspector).compress(CompressionKind.ZLIB));
+    for (Long l : input) {
+      writer.addRow(l);
+    }
+    writer.close();
+
+    Reader reader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs));
+    RecordReader rows = reader.rows();
+    int idx = 0;
+    while (rows.hasNext()) {
+      Object row = rows.next(null);
+      Assert.assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java
index 340951b..e96c809 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java
@@ -34,6 +34,13 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.io.BooleanWritable;
 import org.apache.hadoop.io.IntWritable;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.IntegerColumnStatistics;
+import org.apache.orc.OrcProto;
+
+import org.apache.orc.StringColumnStatistics;
+import org.apache.orc.StripeInformation;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
index 032ac4b..966621c 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
@@ -18,6 +18,9 @@
 
 package org.apache.hadoop.hive.ql.io.orc;
 
+import org.apache.orc.CompressionKind;
+import org.apache.orc.impl.MemoryManager;
+import org.apache.orc.StripeInformation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -47,6 +50,8 @@ import org.apache.hadoop.mapred.InputFormat;
 import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Reporter;
+import org.apache.orc.OrcProto;
+
 import org.junit.Test;
 import org.mockito.MockSettings;
 import org.mockito.Mockito;
@@ -850,7 +855,7 @@ public class TestOrcRawRecordMerger {
       int rowsAddedSinceCheck = 0;
 
       @Override
-      synchronized void addedRow(int rows) throws IOException {
+      public synchronized void addedRow(int rows) throws IOException {
         rowsAddedSinceCheck += rows;
         if (rowsAddedSinceCheck >= 2) {
           notifyWriters();
@@ -951,7 +956,7 @@ public class TestOrcRawRecordMerger {
       int rowsAddedSinceCheck = 0;
 
       @Override
-      synchronized void addedRow(int rows) throws IOException {
+      public synchronized void addedRow(int rows) throws IOException {
         rowsAddedSinceCheck += rows;
         if (rowsAddedSinceCheck >= 2) {
           notifyWriters();

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSerDeStats.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSerDeStats.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSerDeStats.java
index dde653f..575a948 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSerDeStats.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSerDeStats.java
@@ -43,6 +43,13 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspe
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hive.common.util.HiveTestUtils;
+import org.apache.orc.BinaryColumnStatistics;
+import org.apache.orc.BooleanColumnStatistics;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.DoubleColumnStatistics;
+import org.apache.orc.IntegerColumnStatistics;
+import org.apache.orc.StringColumnStatistics;
+import org.apache.orc.StripeInformation;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java
index 4b6af07..a409be8 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java
@@ -42,10 +42,13 @@ import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.Location;
 import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue;
 import org.apache.hadoop.hive.ql.io.sarg.TestSearchArgumentImpl;
-import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.serde2.io.DateWritable;
 import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
 import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.impl.ColumnStatisticsImpl;
+import org.apache.orc.OrcProto;
+
 import org.junit.Test;
 import org.mockito.MockSettings;
 import org.mockito.Mockito;

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthByteReader.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthByteReader.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthByteReader.java
deleted file mode 100644
index 90541f0..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthByteReader.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static junit.framework.Assert.assertEquals;
-
-import java.nio.ByteBuffer;
-
-import org.junit.Test;
-
-public class TestRunLengthByteReader {
-
-  @Test
-  public void testUncompressedSeek() throws Exception {
-    TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
-    RunLengthByteWriter out = new RunLengthByteWriter(new OutStream("test", 100,
-        null, collect));
-    TestInStream.PositionCollector[] positions =
-        new TestInStream.PositionCollector[2048];
-    for(int i=0; i < 2048; ++i) {
-      positions[i] = new TestInStream.PositionCollector();
-      out.getPosition(positions[i]);
-      if (i < 1024) {
-        out.write((byte) (i/4));
-      } else {
-        out.write((byte) i);
-      }
-    }
-    out.flush();
-    ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
-    collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
-    inBuf.flip();
-    RunLengthByteReader in = new RunLengthByteReader(InStream.create("test",
-        new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(), null, 100));
-    for(int i=0; i < 2048; ++i) {
-      int x = in.next() & 0xff;
-      if (i < 1024) {
-        assertEquals((i/4) & 0xff, x);
-      } else {
-        assertEquals(i & 0xff, x);
-      }
-    }
-    for(int i=2047; i >= 0; --i) {
-      in.seek(positions[i]);
-      int x = in.next() & 0xff;
-      if (i < 1024) {
-        assertEquals((i/4) & 0xff, x);
-      } else {
-        assertEquals(i & 0xff, x);
-      }
-    }
-  }
-
-  @Test
-  public void testCompressedSeek() throws Exception {
-    CompressionCodec codec = new SnappyCodec();
-    TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
-    RunLengthByteWriter out = new RunLengthByteWriter(new OutStream("test", 500,
-        codec, collect));
-    TestInStream.PositionCollector[] positions =
-        new TestInStream.PositionCollector[2048];
-    for(int i=0; i < 2048; ++i) {
-      positions[i] = new TestInStream.PositionCollector();
-      out.getPosition(positions[i]);
-      if (i < 1024) {
-        out.write((byte) (i/4));
-      } else {
-        out.write((byte) i);
-      }
-    }
-    out.flush();
-    ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
-    collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
-    inBuf.flip();
-    RunLengthByteReader in = new RunLengthByteReader(InStream.create("test",
-        new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(), codec, 500));
-    for(int i=0; i < 2048; ++i) {
-      int x = in.next() & 0xff;
-      if (i < 1024) {
-        assertEquals((i/4) & 0xff, x);
-      } else {
-        assertEquals(i & 0xff, x);
-      }
-    }
-    for(int i=2047; i >= 0; --i) {
-      in.seek(positions[i]);
-      int x = in.next() & 0xff;
-      if (i < 1024) {
-        assertEquals((i/4) & 0xff, x);
-      } else {
-        assertEquals(i & 0xff, x);
-      }
-    }
-  }
-
-  @Test
-  public void testSkips() throws Exception {
-    TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
-    RunLengthByteWriter out = new RunLengthByteWriter(new OutStream("test", 100,
-        null, collect));
-    for(int i=0; i < 2048; ++i) {
-      if (i < 1024) {
-        out.write((byte) (i/16));
-      } else {
-        out.write((byte) i);
-      }
-    }
-    out.flush();
-    ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
-    collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
-    inBuf.flip();
-    RunLengthByteReader in = new RunLengthByteReader(InStream.create("test",
-        new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(), null, 100));
-    for(int i=0; i < 2048; i += 10) {
-      int x = in.next() & 0xff;
-      if (i < 1024) {
-        assertEquals((i/16) & 0xff, x);
-      } else {
-        assertEquals(i & 0xff, x);
-      }
-      if (i < 2038) {
-        in.skip(9);
-      }
-      in.skip(0);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthIntegerReader.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthIntegerReader.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthIntegerReader.java
deleted file mode 100644
index 562b489..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthIntegerReader.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static junit.framework.Assert.assertEquals;
-
-import java.nio.ByteBuffer;
-import java.util.Random;
-
-import org.junit.Test;
-
-public class TestRunLengthIntegerReader {
-
-  public void runSeekTest(CompressionCodec codec) throws Exception {
-    TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
-    RunLengthIntegerWriter out = new RunLengthIntegerWriter(
-        new OutStream("test", 1000, codec, collect), true);
-    TestInStream.PositionCollector[] positions =
-        new TestInStream.PositionCollector[4096];
-    Random random = new Random(99);
-    int[] junk = new int[2048];
-    for(int i=0; i < junk.length; ++i) {
-      junk[i] = random.nextInt();
-    }
-    for(int i=0; i < 4096; ++i) {
-      positions[i] = new TestInStream.PositionCollector();
-      out.getPosition(positions[i]);
-      // test runs, incrementing runs, non-runs
-      if (i < 1024) {
-        out.write(i/4);
-      } else if (i < 2048) {
-        out.write(2*i);
-      } else {
-        out.write(junk[i-2048]);
-      }
-    }
-    out.flush();
-    ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
-    collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
-    inBuf.flip();
-    RunLengthIntegerReader in = new RunLengthIntegerReader(InStream.create
-        ("test", new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(),
-            codec, 1000), true);
-    for(int i=0; i < 2048; ++i) {
-      int x = (int) in.next();
-      if (i < 1024) {
-        assertEquals(i/4, x);
-      } else if (i < 2048) {
-        assertEquals(2*i, x);
-      } else {
-        assertEquals(junk[i-2048], x);
-      }
-    }
-    for(int i=2047; i >= 0; --i) {
-      in.seek(positions[i]);
-      int x = (int) in.next();
-      if (i < 1024) {
-        assertEquals(i/4, x);
-      } else if (i < 2048) {
-        assertEquals(2*i, x);
-      } else {
-        assertEquals(junk[i-2048], x);
-      }
-    }
-  }
-
-  @Test
-  public void testUncompressedSeek() throws Exception {
-    runSeekTest(null);
-  }
-
-  @Test
-  public void testCompressedSeek() throws Exception {
-    runSeekTest(new ZlibCodec());
-  }
-
-  @Test
-  public void testSkips() throws Exception {
-    TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
-    RunLengthIntegerWriter out = new RunLengthIntegerWriter(
-        new OutStream("test", 100, null, collect), true);
-    for(int i=0; i < 2048; ++i) {
-      if (i < 1024) {
-        out.write(i);
-      } else {
-        out.write(256 * i);
-      }
-    }
-    out.flush();
-    ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
-    collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
-    inBuf.flip();
-    RunLengthIntegerReader in = new RunLengthIntegerReader(InStream.create
-        ("test", new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(),
-            null, 100), true);
-    for(int i=0; i < 2048; i += 10) {
-      int x = (int) in.next();
-      if (i < 1024) {
-        assertEquals(i, x);
-      } else {
-        assertEquals(256 * i, x);
-      }
-      if (i < 2038) {
-        in.skip(9);
-      }
-      in.skip(0);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestSerializationUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestSerializationUtils.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestSerializationUtils.java
deleted file mode 100644
index b3f9cf1..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestSerializationUtils.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.InputStream;
-import java.math.BigInteger;
-
-import org.junit.Test;
-
-import com.google.common.math.LongMath;
-
-public class TestSerializationUtils {
-
-  private InputStream fromBuffer(ByteArrayOutputStream buffer) {
-    return new ByteArrayInputStream(buffer.toByteArray());
-  }
-
-  @Test
-  public void testDoubles() throws Exception {
-    double tolerance = 0.0000000000000001;
-    ByteArrayOutputStream buffer = new ByteArrayOutputStream();
-    SerializationUtils utils = new SerializationUtils();
-    utils.writeDouble(buffer, 1343822337.759);
-    assertEquals(1343822337.759, utils.readDouble(fromBuffer(buffer)), tolerance);
-    buffer = new ByteArrayOutputStream();
-    utils.writeDouble(buffer, 0.8);
-    double got = utils.readDouble(fromBuffer(buffer));
-    assertEquals(0.8, got, tolerance);
-  }
-
-  @Test
-  public void testBigIntegers() throws Exception {
-    ByteArrayOutputStream buffer = new ByteArrayOutputStream();
-    SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(0));
-    assertArrayEquals(new byte[]{0}, buffer.toByteArray());
-    assertEquals(0L,
-        SerializationUtils.readBigInteger(fromBuffer(buffer)).longValue());
-    buffer.reset();
-    SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(1));
-    assertArrayEquals(new byte[]{2}, buffer.toByteArray());
-    assertEquals(1L,
-        SerializationUtils.readBigInteger(fromBuffer(buffer)).longValue());
-    buffer.reset();
-    SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(-1));
-    assertArrayEquals(new byte[]{1}, buffer.toByteArray());
-    assertEquals(-1L,
-        SerializationUtils.readBigInteger(fromBuffer(buffer)).longValue());
-    buffer.reset();
-    SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(50));
-    assertArrayEquals(new byte[]{100}, buffer.toByteArray());
-    assertEquals(50L,
-        SerializationUtils.readBigInteger(fromBuffer(buffer)).longValue());
-    buffer.reset();
-    SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(-50));
-    assertArrayEquals(new byte[]{99}, buffer.toByteArray());
-    assertEquals(-50L,
-        SerializationUtils.readBigInteger(fromBuffer(buffer)).longValue());
-    for(int i=-8192; i < 8192; ++i) {
-      buffer.reset();
-        SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(i));
-      assertEquals("compare length for " + i,
-            i >= -64 && i < 64 ? 1 : 2, buffer.size());
-      assertEquals("compare result for " + i,
-          i, SerializationUtils.readBigInteger(fromBuffer(buffer)).intValue());
-    }
-    buffer.reset();
-    SerializationUtils.writeBigInteger(buffer,
-        new BigInteger("123456789abcdef0",16));
-    assertEquals(new BigInteger("123456789abcdef0",16),
-        SerializationUtils.readBigInteger(fromBuffer(buffer)));
-    buffer.reset();
-    SerializationUtils.writeBigInteger(buffer,
-        new BigInteger("-123456789abcdef0",16));
-    assertEquals(new BigInteger("-123456789abcdef0",16),
-        SerializationUtils.readBigInteger(fromBuffer(buffer)));
-    StringBuilder buf = new StringBuilder();
-    for(int i=0; i < 256; ++i) {
-      String num = Integer.toHexString(i);
-      if (num.length() == 1) {
-        buf.append('0');
-      }
-      buf.append(num);
-    }
-    buffer.reset();
-    SerializationUtils.writeBigInteger(buffer,
-        new BigInteger(buf.toString(),16));
-    assertEquals(new BigInteger(buf.toString(),16),
-        SerializationUtils.readBigInteger(fromBuffer(buffer)));
-    buffer.reset();
-    SerializationUtils.writeBigInteger(buffer,
-        new BigInteger("ff000000000000000000000000000000000000000000ff",16));
-    assertEquals(
-        new BigInteger("ff000000000000000000000000000000000000000000ff",16),
-        SerializationUtils.readBigInteger(fromBuffer(buffer)));
-  }
-
-  @Test
-  public void testSubtractionOverflow() {
-    // cross check results with Guava results below
-    SerializationUtils utils = new SerializationUtils();
-    assertEquals(false, utils.isSafeSubtract(22222222222L, Long.MIN_VALUE));
-    assertEquals(false, utils.isSafeSubtract(-22222222222L, Long.MAX_VALUE));
-    assertEquals(false, utils.isSafeSubtract(Long.MIN_VALUE, Long.MAX_VALUE));
-    assertEquals(true, utils.isSafeSubtract(-1553103058346370095L, 6553103058346370095L));
-    assertEquals(true, utils.isSafeSubtract(0, Long.MAX_VALUE));
-    assertEquals(true, utils.isSafeSubtract(Long.MIN_VALUE, 0));
-  }
-
-  @Test
-  public void testSubtractionOverflowGuava() {
-    try {
-      LongMath.checkedSubtract(22222222222L, Long.MIN_VALUE);
-      fail("expected ArithmeticException for overflow");
-    } catch (ArithmeticException ex) {
-      assertEquals(ex.getMessage(), "overflow");
-    }
-
-    try {
-      LongMath.checkedSubtract(-22222222222L, Long.MAX_VALUE);
-      fail("expected ArithmeticException for overflow");
-    } catch (ArithmeticException ex) {
-      assertEquals(ex.getMessage(), "overflow");
-    }
-
-    try {
-      LongMath.checkedSubtract(Long.MIN_VALUE, Long.MAX_VALUE);
-      fail("expected ArithmeticException for overflow");
-    } catch (ArithmeticException ex) {
-      assertEquals(ex.getMessage(), "overflow");
-    }
-
-    assertEquals(-8106206116692740190L,
-        LongMath.checkedSubtract(-1553103058346370095L, 6553103058346370095L));
-    assertEquals(-Long.MAX_VALUE, LongMath.checkedSubtract(0, Long.MAX_VALUE));
-    assertEquals(Long.MIN_VALUE, LongMath.checkedSubtract(Long.MIN_VALUE, 0));
-  }
-
-  public static void main(String[] args) throws Exception {
-    TestSerializationUtils test = new TestSerializationUtils();
-    test.testDoubles();
-    test.testBigIntegers();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStreamName.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStreamName.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStreamName.java
index f4d91ea..dfccd9a 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStreamName.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStreamName.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.hive.ql.io.orc;
 
+import org.apache.orc.OrcProto;
+import org.apache.orc.impl.StreamName;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringDictionary.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringDictionary.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringDictionary.java
index 7dca820..41a211b 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringDictionary.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringDictionary.java
@@ -26,10 +26,13 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.ql.io.orc.OrcFile.Version;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.io.Text;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.OrcProto;
+
+import org.apache.orc.StripeInformation;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -228,7 +231,7 @@ public class TestStringDictionary {
     Writer writer = OrcFile.createWriter(
         testFilePath,
         OrcFile.writerOptions(conf).inspector(inspector).compress(CompressionKind.NONE)
-            .version(Version.V_0_11).bufferSize(10000));
+            .version(OrcFile.Version.V_0_11).bufferSize(10000));
     for (int i = 0; i < 20000; i++) {
       writer.addRow(new Text(String.valueOf(i)));
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java
deleted file mode 100644
index 5581ad3..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.junit.Test;
-
-import java.io.BufferedOutputStream;
-import java.io.IOException;
-
-import static junit.framework.Assert.assertEquals;
-
-/**
- * Test the red-black tree with string keys.
- */
-public class TestStringRedBlackTree {
-
-  /**
-   * Checks the red-black tree rules to make sure that we have correctly built
-   * a valid tree.
-   *
-   * Properties:
-   *   1. Red nodes must have black children
-   *   2. Each node must have the same black height on both sides.
-   *
-   * @param node The id of the root of the subtree to check for the red-black
-   *        tree properties.
-   * @return The black-height of the subtree.
-   */
-  private int checkSubtree(RedBlackTree tree, int node, IntWritable count
-                          ) throws IOException {
-    if (node == RedBlackTree.NULL) {
-      return 1;
-    }
-    count.set(count.get() + 1);
-    boolean is_red = tree.isRed(node);
-    int left = tree.getLeft(node);
-    int right = tree.getRight(node);
-    if (is_red) {
-      if (tree.isRed(left)) {
-        printTree(tree, "", tree.root);
-        throw new IllegalStateException("Left node of " + node + " is " + left +
-          " and both are red.");
-      }
-      if (tree.isRed(right)) {
-        printTree(tree, "", tree.root);
-        throw new IllegalStateException("Right node of " + node + " is " +
-          right + " and both are red.");
-      }
-    }
-    int left_depth = checkSubtree(tree, left, count);
-    int right_depth = checkSubtree(tree, right, count);
-    if (left_depth != right_depth) {
-      printTree(tree, "", tree.root);
-      throw new IllegalStateException("Lopsided tree at node " + node +
-        " with depths " + left_depth + " and " + right_depth);
-    }
-    if (is_red) {
-      return left_depth;
-    } else {
-      return left_depth + 1;
-    }
-  }
-
-  /**
-   * Checks the validity of the entire tree. Also ensures that the number of
-   * nodes visited is the same as the size of the set.
-   */
-  void checkTree(RedBlackTree tree) throws IOException {
-    IntWritable count = new IntWritable(0);
-    if (tree.isRed(tree.root)) {
-      printTree(tree, "", tree.root);
-      throw new IllegalStateException("root is red");
-    }
-    checkSubtree(tree, tree.root, count);
-    if (count.get() != tree.size) {
-      printTree(tree, "", tree.root);
-      throw new IllegalStateException("Broken tree! visited= " + count.get() +
-        " size=" + tree.size);
-    }
-  }
-
-  void printTree(RedBlackTree tree, String indent, int node
-                ) throws IOException {
-    if (node == RedBlackTree.NULL) {
-      System.err.println(indent + "NULL");
-    } else {
-      System.err.println(indent + "Node " + node + " color " +
-        (tree.isRed(node) ? "red" : "black"));
-      printTree(tree, indent + "  ", tree.getLeft(node));
-      printTree(tree, indent + "  ", tree.getRight(node));
-    }
-  }
-
-  private static class MyVisitor implements StringRedBlackTree.Visitor {
-    private final String[] words;
-    private final int[] order;
-    private final DataOutputBuffer buffer = new DataOutputBuffer();
-    int current = 0;
-
-    MyVisitor(String[] args, int[] order) {
-      words = args;
-      this.order = order;
-    }
-
-    @Override
-    public void visit(StringRedBlackTree.VisitorContext context
-                     ) throws IOException {
-      String word = context.getText().toString();
-      assertEquals("in word " + current, words[current], word);
-      assertEquals("in word " + current, order[current],
-        context.getOriginalPosition());
-      buffer.reset();
-      context.writeBytes(buffer);
-      assertEquals(word, new String(buffer.getData(),0,buffer.getLength()));
-      current += 1;
-    }
-  }
-
-  void checkContents(StringRedBlackTree tree, int[] order,
-                     String... params
-                    ) throws IOException {
-    tree.visit(new MyVisitor(params, order));
-  }
-
-  StringRedBlackTree buildTree(String... params) throws IOException {
-    StringRedBlackTree result = new StringRedBlackTree(1000);
-    for(String word: params) {
-      result.add(word);
-      checkTree(result);
-    }
-    return result;
-  }
-
-  @Test
-  public void test1() throws Exception {
-    StringRedBlackTree tree = new StringRedBlackTree(5);
-    assertEquals(0, tree.getSizeInBytes());
-    checkTree(tree);
-    assertEquals(0, tree.add("owen"));
-    checkTree(tree);
-    assertEquals(1, tree.add("ashutosh"));
-    checkTree(tree);
-    assertEquals(0, tree.add("owen"));
-    checkTree(tree);
-    assertEquals(2, tree.add("alan"));
-    checkTree(tree);
-    assertEquals(2, tree.add("alan"));
-    checkTree(tree);
-    assertEquals(1, tree.add("ashutosh"));
-    checkTree(tree);
-    assertEquals(3, tree.add("greg"));
-    checkTree(tree);
-    assertEquals(4, tree.add("eric"));
-    checkTree(tree);
-    assertEquals(5, tree.add("arun"));
-    checkTree(tree);
-    assertEquals(6, tree.size());
-    checkTree(tree);
-    assertEquals(6, tree.add("eric14"));
-    checkTree(tree);
-    assertEquals(7, tree.add("o"));
-    checkTree(tree);
-    assertEquals(8, tree.add("ziggy"));
-    checkTree(tree);
-    assertEquals(9, tree.add("z"));
-    checkTree(tree);
-    checkContents(tree, new int[]{2,5,1,4,6,3,7,0,9,8},
-      "alan", "arun", "ashutosh", "eric", "eric14", "greg",
-      "o", "owen", "z", "ziggy");
-    assertEquals(32888, tree.getSizeInBytes());
-    // check that adding greg again bumps the count
-    assertEquals(3, tree.add("greg"));
-    assertEquals(41, tree.getCharacterSize());
-    // add some more strings to test the different branches of the
-    // rebalancing
-    assertEquals(10, tree.add("zak"));
-    checkTree(tree);
-    assertEquals(11, tree.add("eric1"));
-    checkTree(tree);
-    assertEquals(12, tree.add("ash"));
-    checkTree(tree);
-    assertEquals(13, tree.add("harry"));
-    checkTree(tree);
-    assertEquals(14, tree.add("john"));
-    checkTree(tree);
-    tree.clear();
-    checkTree(tree);
-    assertEquals(0, tree.getSizeInBytes());
-    assertEquals(0, tree.getCharacterSize());
-  }
-
-  @Test
-  public void test2() throws Exception {
-    StringRedBlackTree tree =
-      buildTree("a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l",
-        "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
-    assertEquals(26, tree.size());
-    checkContents(tree, new int[]{0,1,2, 3,4,5, 6,7,8, 9,10,11, 12,13,14,
-      15,16,17, 18,19,20, 21,22,23, 24,25},
-      "a", "b", "c", "d", "e", "f", "g", "h", "i", "j","k", "l", "m", "n", "o",
-      "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
-  }
-
-  @Test
-  public void test3() throws Exception {
-    StringRedBlackTree tree =
-      buildTree("z", "y", "x", "w", "v", "u", "t", "s", "r", "q", "p", "o", "n",
-        "m", "l", "k", "j", "i", "h", "g", "f", "e", "d", "c", "b", "a");
-    assertEquals(26, tree.size());
-    checkContents(tree, new int[]{25,24,23, 22,21,20, 19,18,17, 16,15,14,
-      13,12,11, 10,9,8, 7,6,5, 4,3,2, 1,0},
-      "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o",
-      "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
-  }
-
-  public static void main(String[] args) throws Exception {
-    TestStringRedBlackTree test = new TestStringRedBlackTree();
-    test.test1();
-    test.test2();
-    test.test3();
-    TestSerializationUtils serUtils = new TestSerializationUtils();
-    serUtils.testDoubles();
-    TestDynamicArray test6 = new TestDynamicArray();
-    test6.testByteArray();
-    test6.testIntArray();
-    TestZlib zlib = new TestZlib();
-    zlib.testCorrupt();
-    zlib.testNoOverflow();
-    TestInStream inStreamTest = new TestInStream();
-    inStreamTest.testUncompressed();
-    inStreamTest.testCompressed();
-    inStreamTest.testCorruptStream();
-    TestRunLengthByteReader rleByte = new TestRunLengthByteReader();
-    rleByte.testUncompressedSeek();
-    rleByte.testCompressedSeek();
-    rleByte.testSkips();
-    TestRunLengthIntegerReader rleInt = new TestRunLengthIntegerReader();
-    rleInt.testUncompressedSeek();
-    rleInt.testCompressedSeek();
-    rleInt.testSkips();
-    TestBitFieldReader bit = new TestBitFieldReader();
-    bit.testUncompressedSeek();
-    bit.testCompressedSeek();
-    bit.testBiggerItems();
-    bit.testSkips();
-    TestOrcFile test1 = new TestOrcFile(false);
-    test1.test1();
-    test1.emptyFile();
-    test1.metaData();
-    test1.testUnionAndTimestamp();
-    test1.columnProjection();
-    test1.testSnappy();
-    test1.testWithoutIndex();
-    test1.testSeek();
-    TestFileDump test2 = new TestFileDump();
-    test2.testDump();
-    TestStreamName test3 = new TestStreamName();
-    test3.test1();
-    TestInputOutputFormat test4 = new TestInputOutputFormat();
-    test4.testInOutFormat();
-    test4.testMROutput();
-    test4.testEmptyFile();
-    test4.testDefaultTypes();
-    TestOrcStruct test5 = new TestOrcStruct();
-    test5.testStruct();
-    test5.testInspectorFromTypeInfo();
-    test5.testUnion();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestTypeDescription.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestTypeDescription.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestTypeDescription.java
index c3095f7..2a82092 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestTypeDescription.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestTypeDescription.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.io.orc;
 
 import static org.junit.Assert.assertEquals;
 
+import org.apache.orc.TypeDescription;
 import org.junit.Test;
 
 public class TestTypeDescription {

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestUnrolledBitPack.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestUnrolledBitPack.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestUnrolledBitPack.java
index 5666cd7..3251731 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestUnrolledBitPack.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestUnrolledBitPack.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.io.LongWritable;
+import org.apache.orc.CompressionKind;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java
index 070e2ab..f5b06db 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.TimestampUtils;
 import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.io.orc.OrcFile.Version;
 import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
@@ -67,6 +66,23 @@ import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hive.common.util.HiveTestUtils;
+import org.apache.orc.BinaryColumnStatistics;
+import org.apache.orc.BooleanColumnStatistics;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.DecimalColumnStatistics;
+import org.apache.orc.DoubleColumnStatistics;
+import org.apache.orc.IntegerColumnStatistics;
+import org.apache.orc.impl.MemoryManager;
+import org.apache.orc.impl.MetadataReader;
+import org.apache.orc.impl.OrcIndex;
+import org.apache.orc.OrcProto;
+import org.apache.orc.OrcUtils;
+import org.apache.orc.StringColumnStatistics;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.StripeStatistics;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.Writer;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -486,7 +502,7 @@ public class TestVectorOrcFile {
     TypeDescription schema = TypeDescription.createTimestamp();
     Writer writer = OrcFile.createWriter(testFilePath,
         OrcFile.writerOptions(conf).setSchema(schema).stripeSize(100000)
-            .bufferSize(10000).version(Version.V_0_11));
+            .bufferSize(10000).version(org.apache.orc.OrcFile.Version.V_0_11));
     List<Timestamp> tslist = Lists.newArrayList();
     tslist.add(Timestamp.valueOf("2037-01-01 00:00:00.000999"));
     tslist.add(Timestamp.valueOf("2003-01-01 00:00:00.000000222"));
@@ -1244,7 +1260,7 @@ public class TestVectorOrcFile {
     Reader reader = OrcFile.createReader(testFilePath,
         OrcFile.readerOptions(conf).filesystem(fs));
     assertEquals(false, reader.rows().hasNext());
-    assertEquals(CompressionKind.NONE, reader.getCompression());
+    assertEquals(CompressionKind.NONE, reader.getCompressionKind());
     assertEquals(0, reader.getNumberOfRows());
     assertEquals(0, reader.getCompressionSize());
     assertEquals(false, reader.getMetadataKeys().iterator().hasNext());
@@ -1743,7 +1759,7 @@ public class TestVectorOrcFile {
     RecordReader rows = reader.rows();
     // get the row index
     MetadataReader meta = ((RecordReaderImpl) rows).getMetadataReader();
-    RecordReaderImpl.Index index =
+    OrcIndex index =
         meta.readRowIndex(reader.getStripes().get(0), null, null, null, null,
             null);
     // check the primitive columns to make sure they have the right number of
@@ -1883,7 +1899,7 @@ public class TestVectorOrcFile {
     }
 
     @Override
-    void addWriter(Path path, long requestedAllocation,
+    public void addWriter(Path path, long requestedAllocation,
                    Callback callback) {
       this.path = path;
       this.lastAllocation = requestedAllocation;
@@ -1891,23 +1907,23 @@ public class TestVectorOrcFile {
     }
 
     @Override
-    synchronized void removeWriter(Path path) {
+    public synchronized void removeWriter(Path path) {
       this.path = null;
       this.lastAllocation = 0;
     }
 
     @Override
-    long getTotalMemoryPool() {
+    public long getTotalMemoryPool() {
       return totalSpace;
     }
 
     @Override
-    double getAllocationScale() {
+    public double getAllocationScale() {
       return rate;
     }
 
     @Override
-    void addedRow(int count) throws IOException {
+    public void addedRow(int count) throws IOException {
       rows += count;
       if (rows % 100 == 0) {
         callback.checkMemory(rate);
@@ -1927,7 +1943,7 @@ public class TestVectorOrcFile {
             .bufferSize(100)
             .rowIndexStride(0)
             .memory(memory)
-            .version(Version.V_0_11));
+            .version(OrcFile.Version.V_0_11));
     assertEquals(testFilePath, memory.path);
     VectorizedRowBatch batch = schema.createRowBatch();
     batch.size = 1;
@@ -1963,7 +1979,7 @@ public class TestVectorOrcFile {
                                          .bufferSize(100)
                                          .rowIndexStride(0)
                                          .memory(memory)
-                                         .version(Version.V_0_12));
+                                         .version(OrcFile.Version.V_0_12));
     VectorizedRowBatch batch = schema.createRowBatch();
     assertEquals(testFilePath, memory.path);
     batch.size = 1;

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
index c739aef..a777b1c 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
@@ -47,7 +47,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.io.BooleanWritable;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.Text;
 import org.junit.Before;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestZlib.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestZlib.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestZlib.java
deleted file mode 100644
index fbb99e3..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestZlib.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.fail;
-
-public class TestZlib {
-
-  @Test
-  public void testNoOverflow() throws Exception {
-    ByteBuffer in = ByteBuffer.allocate(10);
-    ByteBuffer out = ByteBuffer.allocate(10);
-    in.put(new byte[]{1,2,3,4,5,6,7,10});
-    in.flip();
-    CompressionCodec codec = new ZlibCodec();
-    assertEquals(false, codec.compress(in, out, null));
-  }
-
-  @Test
-  public void testCorrupt() throws Exception {
-    ByteBuffer buf = ByteBuffer.allocate(1000);
-    buf.put(new byte[]{127,-128,0,99,98,-1});
-    buf.flip();
-    CompressionCodec codec = new ZlibCodec();
-    ByteBuffer out = ByteBuffer.allocate(1000);
-    try {
-      codec.decompress(buf, out);
-      fail();
-    } catch (IOException ioe) {
-      // EXPECTED
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/serde/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java
deleted file mode 100644
index dd2b1d9..0000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.serde2.io;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.sql.Date;
-import java.util.Calendar;
-import java.util.TimeZone;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.WritableUtils;
-
-
-/**
- * DateWritable
- * Writable equivalent of java.sql.Date.
- *
- * Dates are of the format
- *    YYYY-MM-DD
- *
- */
-public class DateWritable implements WritableComparable<DateWritable> {
-
-  private static final long MILLIS_PER_DAY = TimeUnit.DAYS.toMillis(1);
-
-  // Local time zone.
-  // Java TimeZone has no mention of thread safety. Use thread local instance to be safe.
-  private static final ThreadLocal<TimeZone> LOCAL_TIMEZONE = new ThreadLocal<TimeZone>() {
-    @Override
-    protected TimeZone initialValue() {
-      return Calendar.getInstance().getTimeZone();
-    }
-  };
-
-  // Internal representation is an integer representing day offset from our epoch value 1970-01-01
-  private int daysSinceEpoch = 0;
-
-  /* Constructors */
-  public DateWritable() {
-  }
-
-  public DateWritable(DateWritable d) {
-    set(d);
-  }
-
-  public DateWritable(Date d) {
-    set(d);
-  }
-
-  public DateWritable(int d) {
-    set(d);
-  }
-
-  /**
-   * Set the DateWritable based on the days since epoch date.
-   * @param d integer value representing days since epoch date
-   */
-  public void set(int d) {
-    daysSinceEpoch = d;
-  }
-
-  /**
-   * Set the DateWritable based on the year/month/day of the date in the local timezone.
-   * @param d Date value
-   */
-  public void set(Date d) {
-    if (d == null) {
-      daysSinceEpoch = 0;
-      return;
-    }
-
-    set(dateToDays(d));
-  }
-
-  public void set(DateWritable d) {
-    set(d.daysSinceEpoch);
-  }
-
-  /**
-   *
-   * @return Date value corresponding to the date in the local time zone
-   */
-  public Date get() {
-    return new Date(daysToMillis(daysSinceEpoch));
-  }
-
-  public int getDays() {
-    return daysSinceEpoch;
-  }
-
-  /**
-   *
-   * @return time in seconds corresponding to this DateWritable
-   */
-  public long getTimeInSeconds() {
-    return get().getTime() / 1000;
-  }
-
-  public static Date timeToDate(long l) {
-    return new Date(l * 1000);
-  }
-
-  public static long daysToMillis(int d) {
-    // Convert from day offset to ms in UTC, then apply local timezone offset.
-    long millisUtc = d * MILLIS_PER_DAY;
-    long tmp =  millisUtc - LOCAL_TIMEZONE.get().getOffset(millisUtc);
-    // Between millisUtc and tmp, the time zone offset may have changed due to DST.
-    // Look up the offset again.
-    return millisUtc - LOCAL_TIMEZONE.get().getOffset(tmp);
-  }
-
-  public static int millisToDays(long millisLocal) {
-    long millisUtc = millisLocal + LOCAL_TIMEZONE.get().getOffset(millisLocal);
-    int days;
-    if (millisUtc >= 0L) {
-      days = (int) (millisUtc / MILLIS_PER_DAY);
-    } else {
-      days = (int) ((millisUtc - 86399999) / MILLIS_PER_DAY);
-    }
-    return days;
-  }
-
-  public static int dateToDays(Date d) {
-    // convert to equivalent time in UTC, then get day offset
-    long millisLocal = d.getTime();
-    return millisToDays(millisLocal);
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    daysSinceEpoch = WritableUtils.readVInt(in);
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    WritableUtils.writeVInt(out, daysSinceEpoch);
-  }
-
-  @Override
-  public int compareTo(DateWritable d) {
-    return daysSinceEpoch - d.daysSinceEpoch;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (!(o instanceof DateWritable)) {
-      return false;
-    }
-    return compareTo((DateWritable) o) == 0;
-  }
-
-  @Override
-  public String toString() {
-    return get().toString();
-  }
-
-  @Override
-  public int hashCode() {
-    return daysSinceEpoch;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 0b5c092..c03ae35 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -1045,15 +1045,6 @@ public class Hadoop23Shims extends HadoopShimsSecure {
   }
 
   @Override
-  public DirectDecompressorShim getDirectDecompressor(DirectCompressionType codec) {
-    if(zeroCopy) {
-      return ZeroCopyShims.getDirectDecompressor(codec);
-    }
-    /* not supported */
-    return null;
-  }
-
-  @Override
   public Configuration getConfiguration(org.apache.hadoop.mapreduce.JobContext context) {
     return context.getConfiguration();
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/ZeroCopyShims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/ZeroCopyShims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/ZeroCopyShims.java
index 8de08ad..6ef0467 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/ZeroCopyShims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/ZeroCopyShims.java
@@ -23,13 +23,7 @@ import java.util.EnumSet;
 
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.ReadOption;
-import org.apache.hadoop.hive.shims.HadoopShims.DirectCompressionType;
-import org.apache.hadoop.hive.shims.HadoopShims.DirectDecompressorShim;
 import org.apache.hadoop.io.ByteBufferPool;
-import org.apache.hadoop.io.compress.DirectDecompressor;
-import org.apache.hadoop.io.compress.snappy.SnappyDecompressor.SnappyDirectDecompressor;
-import org.apache.hadoop.io.compress.zlib.ZlibDecompressor.CompressionHeader;
-import org.apache.hadoop.io.compress.zlib.ZlibDecompressor.ZlibDirectDecompressor;
 
 import org.apache.hadoop.hive.shims.HadoopShims.ByteBufferPoolShim;
 import org.apache.hadoop.hive.shims.HadoopShims.ZeroCopyReaderShim;
@@ -89,40 +83,4 @@ class ZeroCopyShims {
     return new ZeroCopyAdapter(in, pool);
   }
 
-  private static final class DirectDecompressorAdapter implements
-      DirectDecompressorShim {
-    private final DirectDecompressor decompressor;
-
-    public DirectDecompressorAdapter(DirectDecompressor decompressor) {
-      this.decompressor = decompressor;
-    }
-
-    public void decompress(ByteBuffer src, ByteBuffer dst) throws IOException {
-      this.decompressor.decompress(src, dst);
-    }
-  }
-
-  public static DirectDecompressorShim getDirectDecompressor(
-      DirectCompressionType codec) {
-    DirectDecompressor decompressor = null;
-    switch (codec) {
-    case ZLIB: {
-      decompressor = new ZlibDirectDecompressor();
-    }
-      break;
-    case ZLIB_NOHEADER: {
-      decompressor = new ZlibDirectDecompressor(CompressionHeader.NO_HEADER, 0);
-    }
-      break;
-    case SNAPPY: {
-      decompressor = new SnappyDirectDecompressor();
-    }
-      break;
-    }
-    if (decompressor != null) {
-      return new DirectDecompressorAdapter(decompressor);
-    }
-    /* not supported */
-    return null;
-  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index f9785ec..7a5a9b5 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -488,19 +488,6 @@ public interface HadoopShims {
     public void releaseBuffer(ByteBuffer buffer);
   }
 
-  public enum DirectCompressionType {
-    NONE,
-    ZLIB_NOHEADER,
-    ZLIB,
-    SNAPPY,
-  };
-
-  public interface DirectDecompressorShim {
-    public void decompress(ByteBuffer src, ByteBuffer dst) throws IOException;
-  }
-
-  public DirectDecompressorShim getDirectDecompressor(DirectCompressionType codec);
-
   /**
    * Get configuration from JobContext
    */

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java b/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
new file mode 100644
index 0000000..86b838c
--- /dev/null
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.common.io.DiskRange;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Disk range information class containing disk ranges and total length.
+ */
+public class DiskRangeInfo {
+  List<DiskRange> diskRanges; // TODO: use DiskRangeList instead
+  long totalLength;
+
+  public DiskRangeInfo(int indexBaseOffset) {
+    this.diskRanges = Lists.newArrayList();
+    // Some data is missing from the stream for PPD uncompressed read (because index offset is
+    // relative to the entire stream and we only read part of stream if RGs are filtered; unlike
+    // with compressed data where PPD only filters CBs, so we always get full CB, and index offset
+    // is relative to CB). To take care of the case when UncompressedStream goes seeking around by
+    // its incorrect (relative to partial stream) index offset, we will increase the length by our
+    // offset-relative-to-the-stream, and also account for it in buffers (see createDiskRangeInfo).
+    // So, index offset now works; as long as noone seeks into this data before the RG (why would
+    // they), everything works. This is hacky... Stream shouldn't depend on having all the data.
+    this.totalLength = indexBaseOffset;
+  }
+
+  public void addDiskRange(DiskRange diskRange) {
+    diskRanges.add(diskRange);
+    totalLength += diskRange.getLength();
+  }
+
+  public List<DiskRange> getDiskRanges() {
+    return diskRanges;
+  }
+
+  public long getTotalLength() {
+    return totalLength;
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java b/storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java
new file mode 100644
index 0000000..dd2b1d9
--- /dev/null
+++ b/storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.io;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.sql.Date;
+import java.util.Calendar;
+import java.util.TimeZone;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableUtils;
+
+
+/**
+ * DateWritable
+ * Writable equivalent of java.sql.Date.
+ *
+ * Dates are of the format
+ *    YYYY-MM-DD
+ *
+ */
+public class DateWritable implements WritableComparable<DateWritable> {
+
+  private static final long MILLIS_PER_DAY = TimeUnit.DAYS.toMillis(1);
+
+  // Local time zone.
+  // Java TimeZone has no mention of thread safety. Use thread local instance to be safe.
+  private static final ThreadLocal<TimeZone> LOCAL_TIMEZONE = new ThreadLocal<TimeZone>() {
+    @Override
+    protected TimeZone initialValue() {
+      return Calendar.getInstance().getTimeZone();
+    }
+  };
+
+  // Internal representation is an integer representing day offset from our epoch value 1970-01-01
+  private int daysSinceEpoch = 0;
+
+  /* Constructors */
+  public DateWritable() {
+  }
+
+  public DateWritable(DateWritable d) {
+    set(d);
+  }
+
+  public DateWritable(Date d) {
+    set(d);
+  }
+
+  public DateWritable(int d) {
+    set(d);
+  }
+
+  /**
+   * Set the DateWritable based on the days since epoch date.
+   * @param d integer value representing days since epoch date
+   */
+  public void set(int d) {
+    daysSinceEpoch = d;
+  }
+
+  /**
+   * Set the DateWritable based on the year/month/day of the date in the local timezone.
+   * @param d Date value
+   */
+  public void set(Date d) {
+    if (d == null) {
+      daysSinceEpoch = 0;
+      return;
+    }
+
+    set(dateToDays(d));
+  }
+
+  public void set(DateWritable d) {
+    set(d.daysSinceEpoch);
+  }
+
+  /**
+   *
+   * @return Date value corresponding to the date in the local time zone
+   */
+  public Date get() {
+    return new Date(daysToMillis(daysSinceEpoch));
+  }
+
+  public int getDays() {
+    return daysSinceEpoch;
+  }
+
+  /**
+   *
+   * @return time in seconds corresponding to this DateWritable
+   */
+  public long getTimeInSeconds() {
+    return get().getTime() / 1000;
+  }
+
+  public static Date timeToDate(long l) {
+    return new Date(l * 1000);
+  }
+
+  public static long daysToMillis(int d) {
+    // Convert from day offset to ms in UTC, then apply local timezone offset.
+    long millisUtc = d * MILLIS_PER_DAY;
+    long tmp =  millisUtc - LOCAL_TIMEZONE.get().getOffset(millisUtc);
+    // Between millisUtc and tmp, the time zone offset may have changed due to DST.
+    // Look up the offset again.
+    return millisUtc - LOCAL_TIMEZONE.get().getOffset(tmp);
+  }
+
+  public static int millisToDays(long millisLocal) {
+    long millisUtc = millisLocal + LOCAL_TIMEZONE.get().getOffset(millisLocal);
+    int days;
+    if (millisUtc >= 0L) {
+      days = (int) (millisUtc / MILLIS_PER_DAY);
+    } else {
+      days = (int) ((millisUtc - 86399999) / MILLIS_PER_DAY);
+    }
+    return days;
+  }
+
+  public static int dateToDays(Date d) {
+    // convert to equivalent time in UTC, then get day offset
+    long millisLocal = d.getTime();
+    return millisToDays(millisLocal);
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    daysSinceEpoch = WritableUtils.readVInt(in);
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    WritableUtils.writeVInt(out, daysSinceEpoch);
+  }
+
+  @Override
+  public int compareTo(DateWritable d) {
+    return daysSinceEpoch - d.daysSinceEpoch;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (!(o instanceof DateWritable)) {
+      return false;
+    }
+    return compareTo((DateWritable) o) == 0;
+  }
+
+  @Override
+  public String toString() {
+    return get().toString();
+  }
+
+  @Override
+  public int hashCode() {
+    return daysSinceEpoch;
+  }
+}


Mime
View raw message