Mailing-List: contact issues-help@carbondata.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@carbondata.apache.org
From: jackylk <git@git.apache.org>
To: issues@carbondata.apache.org
Reply-To: issues@carbondata.apache.org
References: <git-pr-1566-carbondata@git.apache.org>
In-Reply-To: <git-pr-1566-carbondata@git.apache.org>
Subject: [GitHub] carbondata pull request #1566: [CARBONDATA-1586][Streaming] Support handoff ...
Content-Type: text/plain
Message-Id: <20171127022139.1E94DDF9AE@git1-us-west.apache.org>
Date: Mon, 27 Nov 2017 02:21:39 +0000 (UTC)
archived-at: Mon, 27 Nov 2017 02:21:41 -0000

Github user jackylk commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1566#discussion_r153094868
  
    --- Diff: streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala ---
    @@ -0,0 +1,193 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *    http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +
    +package org.apache.carbondata.streaming
    +
    +import java.text.SimpleDateFormat
    +import java.util
    +import java.util.Date
    +
    +import org.apache.hadoop.conf.Configuration
    +import org.apache.hadoop.mapreduce.{Job, TaskAttemptID, TaskType}
    +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
    +import org.apache.spark.{Partition, SerializableWritable, SparkContext, TaskContext}
    +import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
    +
    +import org.apache.carbondata.core.datastore.block.SegmentProperties
    +import org.apache.carbondata.core.datastore.impl.FileFactory
    +import org.apache.carbondata.core.metadata.schema.table.CarbonTable
    +import org.apache.carbondata.core.scan.result.iterator.RawResultIterator
    +import org.apache.carbondata.core.util.CarbonUtil
    +import org.apache.carbondata.hadoop.{CarbonInputSplit, CarbonProjection}
    +import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
    +import org.apache.carbondata.hadoop.streaming.{CarbonStreamInputFormat, CarbonStreamRecordReader}
    +import org.apache.carbondata.processing.loading.model.CarbonLoadModel
    +import org.apache.carbondata.processing.merger.{CompactionResultSortProcessor, CompactionType}
    +import org.apache.carbondata.spark.HandoffResult
    +import org.apache.carbondata.spark.rdd.CarbonRDD
    +
    +/**
    + * partition of the handoff segment
    + */
    +class HandoffPartition(
    +    val rddId: Int,
    +    val idx: Int,
    +    @transient val inputSplit: CarbonInputSplit
    +) extends Partition {
    +
    +  val split = new SerializableWritable[CarbonInputSplit](inputSplit)
    +
    +  override val index: Int = idx
    +
    +  override def hashCode(): Int = 41 * (41 + rddId) + idx
    +}
    +
    +/**
    + * package the record reader of the handoff segment to RawResultIterator
    + */
    +class StreamingRawResultIterator(
    +    recordReader: CarbonStreamRecordReader
    +) extends RawResultIterator {
    +
    +  override def hasNext: Boolean = {
    +    recordReader.nextKeyValue()
    +  }
    +
    +  override def next(): Array[Object] = {
    +    recordReader
    +      .getCurrentValue
    +      .asInstanceOf[GenericInternalRow]
    +      .values
    +      .asInstanceOf[Array[Object]]
    +  }
    +}
    +
    +/**
    + * execute streaming segment handoff
    + */
    +class StreamHandoffRDD[K, V](
    +    sc: SparkContext,
    +    result: HandoffResult[K, V],
    +    carbonLoadModel: CarbonLoadModel,
    +    handOffSegmentId: String
    +) extends CarbonRDD[(K, V)](sc, Nil) {
    +
    +  private val jobTrackerId: String = {
    +    val formatter = new SimpleDateFormat("yyyyMMddHHmm")
    +    formatter.format(new Date())
    +  }
    +
    +  override def internalCompute(split: Partition,
    +      context: TaskContext): Iterator[(K, V)] = {
    +    carbonLoadModel.setPartitionId("0")
    +    carbonLoadModel.setTaskNo("" + split.index)
    +    val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
    +    val iteratorList = prepareInputIterator(split, carbonTable)
    --- End diff --
    
    add comment to describe it is using raw row


---