orc-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From majetideepak <...@git.apache.org>
Subject [GitHub] orc pull request #273: ORC-343 Enable C++ writer to support RleV2
Date Mon, 04 Jun 2018 12:51:05 GMT
Github user majetideepak commented on a diff in the pull request:

    https://github.com/apache/orc/pull/273#discussion_r192729045
  
    --- Diff: c++/test/TestRleEncoder.cc ---
    @@ -0,0 +1,243 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +
    +#include <cstdlib>
    +
    +#include "MemoryOutputStream.hh"
    +#include "RLEv1.hh"
    +
    +#include "wrap/orc-proto-wrapper.hh"
    +#include "wrap/gtest-wrapper.h"
    +
    +namespace orc {
    +
    +  const int DEFAULT_MEM_STREAM_SIZE = 1024 * 1024; // 1M
    +
    +  void generateData(
    +                     uint64_t numValues,
    +                     int64_t start,
    +                     int64_t delta,
    +                     bool random,
    +                     int64_t* data,
    +                     uint64_t numNulls = 0,
    +                     char* notNull = nullptr) {
    +    if (numNulls != 0 && notNull != nullptr) {
    +      memset(notNull, 1, numValues);
    +      while (numNulls > 0) {
    +        uint64_t pos = static_cast<uint64_t>(std::rand()) % numValues;
    +        if (notNull[pos]) {
    +          notNull[pos] = static_cast<char>(0);
    +          --numNulls;
    +        }
    +      }
    +    }
    +
    +    for (uint64_t i = 0; i < numValues; ++i) {
    +      if (notNull == nullptr || notNull[i])
    +      {
    +        if (!random) {
    +          data[i] = start + delta * static_cast<int64_t>(i);
    +        } else {
    +          data[i] = std::rand();
    +        }
    +      }
    +    }
    +  }
    +
    +  void decodeAndVerify(
    +                       RleVersion version,
    +                       const MemoryOutputStream& memStream,
    +                       int64_t * data,
    +                       uint64_t numValues,
    +                       const char* notNull,
    +                       bool isSinged) {
    +    std::unique_ptr<RleDecoder> decoder = createRleDecoder(
    +            std::unique_ptr<SeekableArrayInputStream>(new SeekableArrayInputStream(
    +                    memStream.getData(),
    +                    memStream.getLength())),
    +            isSinged, version, *getDefaultPool());
    +
    +    int64_t* decodedData = new int64_t[numValues];
    +    decoder->next(decodedData, numValues, notNull);
    +
    +    for (uint64_t i = 0; i < numValues; ++i) {
    +      if (!notNull || notNull[i]) {
    +        EXPECT_EQ(data[i], decodedData[i]);
    +      }
    +    }
    +
    +    delete [] decodedData;
    +  }
    +
    +  std::unique_ptr<RleEncoder> getEncoder(RleVersion version,
    +                                        MemoryOutputStream& memStream,
    +                                        bool isSigned)
    +  {
    +    MemoryPool * pool = getDefaultPool();
    +
    +    return createRleEncoder(
    +            std::unique_ptr<BufferedOutputStream>(
    +                    new BufferedOutputStream(*pool, &memStream, 500 * 1024, 1024)),
    +            isSigned, version, *pool, true);
    --- End diff --
    
    can we template these tests for `alignedBitpacking =  false`?


---

Mime
View raw message