drill-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (DRILL-3232) Modify existing vectors to allow type promotion
Date Fri, 30 Oct 2015 04:48:27 GMT

    [ https://issues.apache.org/jira/browse/DRILL-3232?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14981877#comment-14981877
] 

ASF GitHub Bot commented on DRILL-3232:
---------------------------------------

Github user jacques-n commented on a diff in the pull request:

    https://github.com/apache/drill/pull/207#discussion_r43471567
  
    --- Diff: exec/java-exec/src/main/codegen/templates/UnionVector.java ---
    @@ -0,0 +1,479 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +
    +import org.apache.drill.common.types.TypeProtos.MinorType;
    +
    +<@pp.dropOutputFile />
    +<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionVector.java"
/>
    +
    +
    +<#include "/@includes/license.ftl" />
    +
    +package org.apache.drill.exec.vector.complex.impl;
    +
    +<#include "/@includes/vv_imports.ftl" />
    +import java.util.Iterator;
    +import org.apache.drill.exec.vector.complex.impl.ComplexCopier;
    +import org.apache.drill.exec.util.CallBack;
    +
    +/*
    + * This class is generated using freemarker and the ${.template_name} template.
    + */
    +@SuppressWarnings("unused")
    +
    +
    +public class UnionVector implements ValueVector {
    +
    +  private MaterializedField field;
    +  private BufferAllocator allocator;
    +  private Accessor accessor = new Accessor();
    +  private Mutator mutator = new Mutator();
    +  private int valueCount;
    +
    +  private MapVector internalMap;
    +  private SingleMapWriter internalMapWriter;
    +  private UInt1Vector typeVector;
    +
    +  private MapVector mapVector;
    +  private ListVector listVector;
    +  private NullableBigIntVector bigInt;
    +  private NullableVarCharVector varChar;
    +
    +  private FieldReader reader;
    +  private NullableBitVector bit;
    +
    +  private State state = State.INIT;
    +  private int singleType = 0;
    +  private ValueVector singleVector;
    +  private MajorType majorType;
    +
    +  private final CallBack callBack;
    +
    +  private enum State {
    +    INIT, SINGLE, MULTI
    +  }
    +
    +  public UnionVector(MaterializedField field, BufferAllocator allocator, CallBack callBack)
{
    +    this.field = field.clone();
    +    this.allocator = allocator;
    +    internalMap = new MapVector("internal", allocator, callBack);
    +    internalMapWriter = new SingleMapWriter(internalMap, null, true, true);
    +    this.typeVector = internalMap.addOrGet("types", Types.required(MinorType.UINT1),
UInt1Vector.class);
    +    this.field.addChild(internalMap.getField().clone());
    +    this.majorType = field.getType();
    +    this.callBack = callBack;
    +  }
    +
    +  private void updateState(ValueVector v) {
    +    if (state == State.INIT) {
    +      state = State.SINGLE;
    +      singleVector = v;
    +      singleType = v.getField().getType().getMinorType().getNumber();
    +    } else {
    +      state = State.MULTI;
    +      singleVector = null;
    +    }
    +  }
    +
    +  public List<MinorType> getSubTypes() {
    +    return majorType.getSubTypeList();
    +  }
    +
    +  private void addSubType(MinorType type) {
    +    majorType =  MajorType.newBuilder(this.majorType).addSubType(type).build();
    +    if (callBack != null) {
    +      callBack.doWork();
    +    }
    +  }
    +
    +  public boolean isSingleType() {
    +    return state == State.SINGLE && singleType != MinorType.LIST_VALUE;
    +  }
    +
    +  public ValueVector getSingleVector() {
    +    assert state != State.MULTI : "Cannot get single vector when there are multiple types";
    +    assert state != State.INIT : "Cannot get single vector when there are no types";
    +    return singleVector;
    +  }
    +
    +  private static final MajorType MAP_TYPE = Types.optional(MinorType.MAP);
    +
    +  public MapVector getMap() {
    +    if (mapVector == null) {
    +      int vectorCount = internalMap.size();
    +      mapVector = internalMap.addOrGet("map", MAP_TYPE, MapVector.class);
    +      updateState(mapVector);
    +      addSubType(MinorType.MAP);
    +      if (internalMap.size() > vectorCount) {
    +        mapVector.allocateNew();
    +      }
    +    }
    +    return mapVector;
    +  }
    +
    +  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first
/>
    +  <#assign fields = minor.fields!type.fields />
    +  <#assign uncappedName = name?uncap_first/>
    +  <#if !minor.class?starts_with("Decimal")>
    +
    +  private Nullable${name}Vector ${uncappedName}Vector;
    +  private static final MajorType ${name?upper_case}_TYPE = Types.optional(MinorType.${name?upper_case});
    +
    +  public Nullable${name}Vector get${name}Vector() {
    +    if (${uncappedName}Vector == null) {
    +      int vectorCount = internalMap.size();
    +      ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", ${name?upper_case}_TYPE,
Nullable${name}Vector.class);
    +      updateState(${uncappedName}Vector);
    +      addSubType(MinorType.${name?upper_case});
    +      if (internalMap.size() > vectorCount) {
    +        ${uncappedName}Vector.allocateNew();
    +      }
    +    }
    +    return ${uncappedName}Vector;
    +  }
    +
    +  </#if>
    +
    +  </#list></#list>
    +
    +  private static final MajorType LIST_TYPE = Types.optional(MinorType.LIST);
    +
    +  public ListVector getList() {
    +    if (listVector == null) {
    +      int vectorCount = internalMap.size();
    +      listVector = internalMap.addOrGet("list", LIST_TYPE, ListVector.class);
    +      updateState(listVector);
    +      addSubType(MinorType.LIST);
    +      if (internalMap.size() > vectorCount) {
    +        listVector.allocateNew();
    +      }
    +    }
    +    return listVector;
    +  }
    +
    +  public int getTypeValue(int index) {
    +    return typeVector.getAccessor().get(index);
    +  }
    +
    +  public UInt1Vector getTypeVector() {
    +    return typeVector;
    +  }
    +
    +  @Override
    +  public void allocateNew() throws OutOfMemoryRuntimeException {
    +    internalMap.allocateNew();
    +    if (typeVector != null) {
    +      typeVector.zeroVector();
    +    }
    +  }
    +
    +  @Override
    +  public boolean allocateNewSafe() {
    +    boolean safe = internalMap.allocateNewSafe();
    +    if (safe) {
    +      if (typeVector != null) {
    +        typeVector.zeroVector();
    +      }
    +    }
    +    return safe;
    +  }
    +
    +  @Override
    +  public void setInitialCapacity(int numRecords) {
    +  }
    +
    +  @Override
    +  public int getValueCapacity() {
    +    return Math.min(typeVector.getValueCapacity(), internalMap.getValueCapacity());
    +  }
    +
    +  @Override
    +  public void close() {
    +  }
    +
    +  @Override
    +  public void clear() {
    +    internalMap.clear();
    +  }
    +
    +  @Override
    +  public MaterializedField getField() {
    +    return field;
    +  }
    +
    +  @Override
    +  public TransferPair getTransferPair() {
    +    return new TransferImpl(field);
    +  }
    +
    +  @Override
    +  public TransferPair getTransferPair(FieldReference ref) {
    +    return new TransferImpl(field.withPath(ref));
    +  }
    +
    +  @Override
    +  public TransferPair makeTransferPair(ValueVector target) {
    +    return new TransferImpl((UnionVector) target);
    +  }
    +
    +  public void transferTo(UnionVector target) {
    +    internalMap.makeTransferPair(target.internalMap).transfer();
    +    target.valueCount = valueCount;
    +    target.majorType = majorType;
    +  }
    +
    +  public void copyFrom(int inIndex, int outIndex, UnionVector from) {
    +    from.getReader().setPosition(inIndex);
    +    getWriter().setPosition(outIndex);
    +    ComplexCopier copier = new ComplexCopier(from.reader, mutator.writer);
    +    copier.write();
    +  }
    +
    +  public void copyFromSafe(int inIndex, int outIndex, UnionVector from) {
    +    copyFrom(inIndex, outIndex, from);
    +  }
    +
    +  public void addVector(ValueVector v) {
    +    internalMap.putChild(v.getField().getType().getMinorType().name().toLowerCase(),
v);
    +    addSubType(v.getField().getType().getMinorType());
    +  }
    +
    +  private class TransferImpl implements TransferPair {
    +
    +    UnionVector to;
    +
    +    public TransferImpl(MaterializedField field) {
    +      to = new UnionVector(field, allocator, null);
    +    }
    +
    +    public TransferImpl(UnionVector to) {
    +      this.to = to;
    +    }
    +
    +    @Override
    +    public void transfer() {
    +      transferTo(to);
    +    }
    +
    +    @Override
    +    public void splitAndTransfer(int startIndex, int length) {
    +
    +    }
    +
    +    @Override
    +    public ValueVector getTo() {
    +      return to;
    +    }
    +
    +    @Override
    +    public void copyValueSafe(int from, int to) {
    +      this.to.copyFrom(from, to, UnionVector.this);
    +    }
    +  }
    +
    +  @Override
    +  public Accessor getAccessor() {
    +    return accessor;
    +  }
    +
    +  @Override
    +  public Mutator getMutator() {
    +    return mutator;
    +  }
    +
    +  @Override
    +  public FieldReader getReader() {
    +    if (reader == null) {
    +      reader = new UnionReader(this);
    +    }
    +    return reader;
    +  }
    +
    +  public FieldWriter getWriter() {
    +    if (mutator.writer == null) {
    +      mutator.writer = new UnionWriter(this);
    +    }
    +    return mutator.writer;
    +  }
    +
    +  @Override
    +  public UserBitShared.SerializedField getMetadata() {
    +    SerializedField.Builder b = getField() //
    +            .getAsBuilder() //
    +            .setBufferLength(getBufferSize()) //
    +            .setValueCount(valueCount);
    +
    +    b.addChild(internalMap.getMetadata());
    +    return b.build();
    +  }
    +
    +  @Override
    +  public int getBufferSize() {
    +    return internalMap.getBufferSize();
    +  }
    +
    +  @Override
    +  public int getBufferSizeFor(final int valueCount) {
    +    if (valueCount == 0) {
    +      return 0;
    +    }
    +
    +    long bufferSize = 0;
    +    for (final ValueVector v : (Iterable<ValueVector>) this) {
    +      bufferSize += v.getBufferSizeFor(valueCount);
    +    }
    +
    +    return (int) bufferSize;
    +  }
    +
    +  @Override
    +  public DrillBuf[] getBuffers(boolean clear) {
    +    return internalMap.getBuffers(clear);
    +  }
    +
    +  @Override
    +  public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
    +    valueCount = metadata.getValueCount();
    +
    +    internalMap.load(metadata.getChild(0), buffer);
    +  }
    +
    +  @Override
    +  public Iterator<ValueVector> iterator() {
    +    List<ValueVector> vectors = Lists.newArrayList(internalMap.iterator());
    +    vectors.add(typeVector);
    +    return vectors.iterator();
    +  }
    +
    +  public class Accessor extends BaseValueVector.BaseAccessor {
    +
    +
    +    @Override
    +    public Object getObject(int index) {
    +      int type = typeVector.getAccessor().get(index);
    +      switch (type) {
    +      case 0:
    +        return null;
    +      <#list vv.types as type><#list type.minor as minor><#assign name
= minor.class?cap_first />
    +      <#assign fields = minor.fields!type.fields />
    +      <#assign uncappedName = name?uncap_first/>
    +      <#if !minor.class?starts_with("Decimal")>
    +      case MinorType.${name?upper_case}_VALUE:
    +        return get${name}Vector().getAccessor().getObject(index);
    +      </#if>
    +
    +      </#list></#list>
    +      case MinorType.MAP_VALUE:
    +        return getMap().getAccessor().getObject(index);
    +      case MinorType.LIST_VALUE:
    +        return getList().getAccessor().getObject(index);
    +      default:
    +        throw new UnsupportedOperationException("Cannot support type: " + MinorType.valueOf(type));
    +      }
    +    }
    +
    +    public byte[] get(int index) {
    +      return null;
    +    }
    +
    +    public void get(int index, ComplexHolder holder) {
    +    }
    +
    +    public void get(int index, UnionHolder holder) {
    +      if (reader == null) {
    +        reader = new UnionReader(UnionVector.this);
    +      }
    +      reader.setPosition(index);
    +      holder.reader = reader;
    +    }
    +
    +    @Override
    +    public int getValueCount() {
    +      return valueCount;
    +    }
    +
    +    @Override
    +    public boolean isNull(int index) {
    +      return typeVector.getAccessor().get(index) == 0;
    +    }
    +
    +    public int isSet(int index) {
    +      return isNull(index) ? 0 : 1;
    +    }
    +  }
    +
    +  public class Mutator extends BaseValueVector.BaseMutator {
    +
    +    UnionWriter writer;
    +
    +    @Override
    +    public void setValueCount(int valueCount) {
    +      UnionVector.this.valueCount = valueCount;
    +      internalMap.getMutator().setValueCount(valueCount);
    +    }
    +
    +    public void set(int index, byte[] bytes) {
    +    }
    --- End diff --
    
    We need to do something here...


> Modify existing vectors to allow type promotion
> -----------------------------------------------
>
>                 Key: DRILL-3232
>                 URL: https://issues.apache.org/jira/browse/DRILL-3232
>             Project: Apache Drill
>          Issue Type: Sub-task
>          Components: Execution - Codegen, Execution - Data Types, Execution - Relational
Operators, Functions - Drill
>            Reporter: Steven Phillips
>            Assignee: Hanifi Gunes
>             Fix For: 1.3.0
>
>
> Support the ability for existing vectors to be promoted similar to supported implicit
casting rules.
> For example:
> INT > DOUBLE > STRING > EMBEDDED



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message