phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jamestay...@apache.org
Subject [05/51] [partial] Initial commit of master branch from github
Date Mon, 27 Jan 2014 22:15:21 GMT
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/50d523f6/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
new file mode 100644
index 0000000..4ea8a11
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
@@ -0,0 +1,447 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.TreeMap;
+
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import com.google.common.collect.Lists;
+import org.apache.phoenix.compile.ScanRanges;
+import org.apache.phoenix.coprocessor.MetaDataProtocol;
+import org.apache.phoenix.filter.SkipScanFilter;
+import org.apache.phoenix.query.KeyRange;
+import org.apache.phoenix.query.KeyRange.Bound;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.PDataType;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.RowKeySchema;
+
+
+/**
+ * 
+ * Various utilities for scans
+ *
+ * @author jtaylor
+ * @since 0.1
+ */
+public class ScanUtil {
+
+    private ScanUtil() {
+    }
+
+    public static void setTenantId(Scan scan, byte[] tenantId) {
+        scan.setAttribute(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
+    }
+
+    // Use getTenantId and pass in column name to match against
+    // in as PSchema attribute. If column name matches in 
+    // KeyExpressions, set on scan as attribute
+    public static ImmutableBytesWritable getTenantId(Scan scan) {
+        // Create Scan with special aggregation column over which to aggregate
+        byte[] tenantId = scan.getAttribute(PhoenixRuntime.TENANT_ID_ATTRIB);
+        if (tenantId == null) {
+            return null;
+        }
+        return new ImmutableBytesWritable(tenantId);
+    }
+
+    public static Scan newScan(Scan scan) {
+        try {
+            Scan newScan = new Scan(scan);
+            // Clone the underlying family map instead of sharing it between
+            // the existing and cloned Scan (which is the retarded default
+            // behavior).
+            TreeMap<byte [], NavigableSet<byte []>> existingMap = (TreeMap<byte[], NavigableSet<byte[]>>)scan.getFamilyMap();
+            Map<byte [], NavigableSet<byte []>> clonedMap = new TreeMap<byte [], NavigableSet<byte []>>(existingMap);
+            newScan.setFamilyMap(clonedMap);
+            return newScan;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+    /**
+     * Intersects the scan start/stop row with the startKey and stopKey
+     * @param scan
+     * @param startKey
+     * @param stopKey
+     * @return false if the Scan cannot possibly return rows and true otherwise
+     */
+    public static boolean intersectScanRange(Scan scan, byte[] startKey, byte[] stopKey) {
+        return intersectScanRange(scan, startKey, stopKey, false);
+    }
+
+    public static boolean intersectScanRange(Scan scan, byte[] startKey, byte[] stopKey, boolean useSkipScan) {
+        boolean mayHaveRows = false;
+        byte[] existingStartKey = scan.getStartRow();
+        byte[] existingStopKey = scan.getStopRow();
+        if (existingStartKey.length > 0) {
+            if (startKey.length == 0 || Bytes.compareTo(existingStartKey, startKey) > 0) {
+                startKey = existingStartKey;
+            }
+        } else {
+            mayHaveRows = true;
+        }
+        if (existingStopKey.length > 0) {
+            if (stopKey.length == 0 || Bytes.compareTo(existingStopKey, stopKey) < 0) {
+                stopKey = existingStopKey;
+            }
+        } else {
+            mayHaveRows = true;
+        }
+        scan.setStartRow(startKey);
+        scan.setStopRow(stopKey);
+        
+        mayHaveRows = mayHaveRows || Bytes.compareTo(scan.getStartRow(), scan.getStopRow()) < 0;
+        
+        // If the scan is using skip scan filter, intersect and replace the filter.
+        if (mayHaveRows && useSkipScan) {
+            Filter filter = scan.getFilter();
+            if (filter instanceof SkipScanFilter) {
+                SkipScanFilter oldFilter = (SkipScanFilter)filter;
+                SkipScanFilter newFilter = oldFilter.intersect(startKey, stopKey);
+                if (newFilter == null) {
+                    return false;
+                }
+                // Intersect found: replace skip scan with intersected one
+                scan.setFilter(newFilter);
+            } else if (filter instanceof FilterList) {
+                FilterList filterList = (FilterList)filter;
+                Filter firstFilter = filterList.getFilters().get(0);
+                if (firstFilter instanceof SkipScanFilter) {
+                    SkipScanFilter oldFilter = (SkipScanFilter)firstFilter;
+                    SkipScanFilter newFilter = oldFilter.intersect(startKey, stopKey);
+                    if (newFilter == null) {
+                        return false;
+                    }
+                    // Intersect found: replace skip scan with intersected one
+                    List<Filter> allFilters = new ArrayList<Filter>(filterList.getFilters().size());
+                    allFilters.addAll(filterList.getFilters());
+                    allFilters.set(0, newFilter);
+                    scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,allFilters));
+                }
+            }
+        }
+        return mayHaveRows;
+    }
+
+    public static void andFilterAtBeginning(Scan scan, Filter andWithFilter) {
+        if (andWithFilter == null) {
+            return;
+        }
+        Filter filter = scan.getFilter();
+        if (filter == null) {
+            scan.setFilter(andWithFilter); 
+        } else if (filter instanceof FilterList && ((FilterList)filter).getOperator() == FilterList.Operator.MUST_PASS_ALL) {
+            FilterList filterList = (FilterList)filter;
+            List<Filter> allFilters = new ArrayList<Filter>(filterList.getFilters().size() + 1);
+            allFilters.add(andWithFilter);
+            allFilters.addAll(filterList.getFilters());
+            scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,allFilters));
+        } else {
+            scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,Arrays.asList(andWithFilter, filter)));
+        }
+    }
+
+    public static void andFilterAtEnd(Scan scan, Filter andWithFilter) {
+        if (andWithFilter == null) {
+            return;
+        }
+        Filter filter = scan.getFilter();
+        if (filter == null) {
+            scan.setFilter(andWithFilter); 
+        } else if (filter instanceof FilterList && ((FilterList)filter).getOperator() == FilterList.Operator.MUST_PASS_ALL) {
+            FilterList filterList = (FilterList)filter;
+            List<Filter> allFilters = new ArrayList<Filter>(filterList.getFilters().size() + 1);
+            allFilters.addAll(filterList.getFilters());
+            allFilters.add(andWithFilter);
+            scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,allFilters));
+        } else {
+            scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,Arrays.asList(filter, andWithFilter)));
+        }
+    }
+
+    public static void setTimeRange(Scan scan, long ts) {
+        try {
+            scan.setTimeRange(MetaDataProtocol.MIN_TABLE_TIMESTAMP, ts);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static byte[] getMinKey(RowKeySchema schema, List<List<KeyRange>> slots) {
+        return getKey(schema, slots, Bound.LOWER);
+    }
+
+    public static byte[] getMaxKey(RowKeySchema schema, List<List<KeyRange>> slots) {
+        return getKey(schema, slots, Bound.UPPER);
+    }
+
+    private static byte[] getKey(RowKeySchema schema, List<List<KeyRange>> slots, Bound bound) {
+        if (slots.isEmpty()) {
+            return null;
+        }
+        int[] position = new int[slots.size()];
+        int maxLength = 0;
+        for (int i = 0; i < position.length; i++) {
+            position[i] = bound == Bound.LOWER ? 0 : slots.get(i).size()-1;
+            KeyRange range = slots.get(i).get(position[i]);
+            maxLength += range.getRange(bound).length + (schema.getField(i).getDataType().isFixedWidth() ? 0 : 1);
+        }
+        byte[] key = new byte[maxLength];
+        int length = setKey(schema, slots, position, bound, key, 0, 0, position.length);
+        if (length == 0) {
+            return null;
+        }
+        if (length == maxLength) {
+            return key;
+        }
+        byte[] keyCopy = new byte[length];
+        System.arraycopy(key, 0, keyCopy, 0, length);
+        return keyCopy;
+    }
+
+    public static int estimateMaximumKeyLength(RowKeySchema schema, int schemaStartIndex, List<List<KeyRange>> slots) {
+        int maxLowerKeyLength = 0, maxUpperKeyLength = 0;
+        for (int i = 0; i < slots.size(); i++) {
+            int maxLowerRangeLength = 0, maxUpperRangeLength = 0;
+            for (KeyRange range: slots.get(i)) {
+                maxLowerRangeLength = Math.max(maxLowerRangeLength, range.getLowerRange().length); 
+                maxUpperRangeLength = Math.max(maxUpperRangeLength, range.getUpperRange().length);
+            }
+            int trailingByte = (schema.getField(schemaStartIndex).getDataType().isFixedWidth() ||
+                    schemaStartIndex == schema.getFieldCount() - 1 ? 0 : 1);
+            maxLowerKeyLength += maxLowerRangeLength + trailingByte;
+            maxUpperKeyLength += maxUpperKeyLength + trailingByte;
+            schemaStartIndex++;
+        }
+        return Math.max(maxLowerKeyLength, maxUpperKeyLength);
+    }
+
+    /*
+     * Set the key by appending the keyRanges inside slots at positions as specified by the position array.
+     * 
+     * We need to increment part of the key range, or increment the whole key at the end, depending on the
+     * bound we are setting and whether the key range is inclusive or exclusive. The logic for determining
+     * whether to increment or not is:
+     * range/single    boundary       bound      increment
+     *  range          inclusive      lower         no
+     *  range          inclusive      upper         yes, at the end if occurs at any slots.
+     *  range          exclusive      lower         yes
+     *  range          exclusive      upper         no
+     *  single         inclusive      lower         no
+     *  single         inclusive      upper         yes, at the end if it is the last slots.
+     */
+    public static int setKey(RowKeySchema schema, List<List<KeyRange>> slots, int[] position, Bound bound,
+            byte[] key, int byteOffset, int slotStartIndex, int slotEndIndex) {
+        return setKey(schema, slots, position, bound, key, byteOffset, slotStartIndex, slotEndIndex, slotStartIndex);
+    }
+
+    public static int setKey(RowKeySchema schema, List<List<KeyRange>> slots, int[] position, Bound bound,
+            byte[] key, int byteOffset, int slotStartIndex, int slotEndIndex, int schemaStartIndex) {
+        int offset = byteOffset;
+        boolean lastInclusiveUpperSingleKey = false;
+        boolean anyInclusiveUpperRangeKey = false;
+        for (int i = slotStartIndex; i < slotEndIndex; i++) {
+            // Build up the key by appending the bound of each key range
+            // from the current position of each slot. 
+            KeyRange range = slots.get(i).get(position[i]);
+            boolean isFixedWidth = schema.getField(schemaStartIndex++).getDataType().isFixedWidth();
+            /*
+             * If the current slot is unbound then stop if:
+             * 1) setting the upper bound. There's no value in
+             *    continuing because nothing will be filtered.
+             * 2) setting the lower bound when the type is fixed length
+             *    for the same reason. However, if the type is variable width
+             *    continue building the key because null values will be filtered
+             *    since our separator byte will be appended and incremented.
+             */
+            if (  range.isUnbound(bound) &&
+                ( bound == Bound.UPPER || isFixedWidth) ){
+                break;
+            }
+            byte[] bytes = range.getRange(bound);
+            System.arraycopy(bytes, 0, key, offset, bytes.length);
+            offset += bytes.length;
+            /*
+             * We must add a terminator to a variable length key even for the last PK column if
+             * the lower key is non inclusive or the upper key is inclusive. Otherwise, we'd be
+             * incrementing the key value itself, and thus bumping it up too much.
+             */
+            boolean inclusiveUpper = range.isInclusive(bound) && bound == Bound.UPPER;
+            boolean exclusiveLower = !range.isInclusive(bound) && bound == Bound.LOWER;
+            if (!isFixedWidth && ( i < schema.getMaxFields()-1 || inclusiveUpper || exclusiveLower)) {
+                key[offset++] = QueryConstants.SEPARATOR_BYTE;
+            }
+            // If we are setting the upper bound of using inclusive single key, we remember 
+            // to increment the key if we exit the loop after this iteration.
+            // 
+            // We remember to increment the last slot if we are setting the upper bound with an
+            // inclusive range key.
+            //
+            // We cannot combine the two flags together in case for single-inclusive key followed
+            // by the range-exclusive key. In that case, we do not need to increment the end at the
+            // end. But if we combine the two flag, the single inclusive key in the middle of the
+            // key slots would cause the flag to become true.
+            lastInclusiveUpperSingleKey = range.isSingleKey() && inclusiveUpper;
+            anyInclusiveUpperRangeKey |= !range.isSingleKey() && inclusiveUpper;
+            // If we are setting the lower bound with an exclusive range key, we need to bump the
+            // slot up for each key part. For an upper bound, we bump up an inclusive key, but
+            // only after the last key part.
+            if (!range.isSingleKey() && exclusiveLower) {
+                if (!ByteUtil.nextKey(key, offset)) {
+                    // Special case for not being able to increment.
+                    // In this case we return a negative byteOffset to
+                    // remove this part from the key being formed. Since the
+                    // key has overflowed, this means that we should not
+                    // have an end key specified.
+                    return -byteOffset;
+                }
+            }
+        }
+        if (lastInclusiveUpperSingleKey || anyInclusiveUpperRangeKey) {
+            if (!ByteUtil.nextKey(key, offset)) {
+                // Special case for not being able to increment.
+                // In this case we return a negative byteOffset to
+                // remove this part from the key being formed. Since the
+                // key has overflowed, this means that we should not
+                // have an end key specified.
+                return -byteOffset;
+            }
+        }
+        // Remove trailing separator bytes, since the columns may have been added
+        // after the table has data, in which case there won't be a separator
+        // byte.
+        if (bound == Bound.LOWER) {
+            while (schemaStartIndex > 0 && offset > byteOffset && 
+                    !schema.getField(--schemaStartIndex).getDataType().isFixedWidth() && 
+                    key[offset-1] == QueryConstants.SEPARATOR_BYTE) {
+                offset--;
+            }
+        }
+        return offset - byteOffset;
+    }
+
+    public static boolean isAllSingleRowScan(List<List<KeyRange>> ranges, RowKeySchema schema) {
+        if (ranges.size() < schema.getMaxFields()) {
+            return false;
+        }
+        for (int i = 0; i < ranges.size(); i++) {
+            List<KeyRange> orRanges = ranges.get(i);
+            for (KeyRange range: orRanges) {
+                if (!range.isSingleKey()) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Perform a binary lookup on the list of KeyRange for the tightest slot such that the slotBound
+     * of the current slot is higher or equal than the slotBound of our range. 
+     * @return  the index of the slot whose slot bound equals or are the tightest one that is 
+     *          smaller than rangeBound of range, or slots.length if no bound can be found.
+     */
+    public static int searchClosestKeyRangeWithUpperHigherThanPtr(List<KeyRange> slots, ImmutableBytesWritable ptr, int lower) {
+        int upper = slots.size() - 1;
+        int mid;
+        while (lower <= upper) {
+            mid = (lower + upper) / 2;
+            int cmp = slots.get(mid).compareUpperToLowerBound(ptr, true);
+            if (cmp < 0) {
+                lower = mid + 1;
+            } else if (cmp > 0) {
+                upper = mid - 1;
+            } else {
+                return mid;
+            }
+        }
+        mid = (lower + upper) / 2;
+        if (mid == 0 && slots.get(mid).compareUpperToLowerBound(ptr, true) > 0) {
+            return mid;
+        } else {
+            return ++mid;
+        }
+    }
+    
+    public static ScanRanges newScanRanges(List<Mutation> mutations) throws SQLException {
+        List<KeyRange> keys = Lists.newArrayListWithExpectedSize(mutations.size());
+        for (Mutation m : mutations) {
+            keys.add(PDataType.VARBINARY.getKeyRange(m.getRow()));
+        }
+        ScanRanges keyRanges = ScanRanges.create(Collections.singletonList(keys), SchemaUtil.VAR_BINARY_SCHEMA);
+        return keyRanges;
+    }
+
+    public static byte[] nextKey(byte[] key, PTable table, ImmutableBytesWritable ptr) {
+        int pos = 0;
+        RowKeySchema schema = table.getRowKeySchema();
+        int maxOffset = schema.iterator(key, ptr);
+        while (schema.next(ptr, pos, maxOffset) != null) {
+            pos++;
+        }
+        if (!schema.getField(pos-1).getDataType().isFixedWidth()) {
+            byte[] newLowerRange = new byte[key.length + 1];
+            System.arraycopy(key, 0, newLowerRange, 0, key.length);
+            newLowerRange[key.length] = QueryConstants.SEPARATOR_BYTE;
+            key = newLowerRange;
+        } else {
+            key = Arrays.copyOf(key, key.length);
+        }
+        ByteUtil.nextKey(key, key.length);
+        return key;
+    }
+
+    private static final String REVERSED_ATTR = "_reversed_";
+    
+    public static void setReversed(Scan scan) {
+        // TODO: set attribute dynamically here to prevent dependency on newer HBase release
+        scan.setAttribute(REVERSED_ATTR, PDataType.TRUE_BYTES);
+    }
+
+    // Start/stop row must be swapped if scan is being done in reverse
+    public static void swapStartStopRowIfReversed(Scan scan) {
+        if (isReversed(scan)) {
+            byte[] startRow = scan.getStartRow();
+            byte[] stopRow = scan.getStopRow();
+            scan.setStartRow(stopRow);
+            scan.setStopRow(startRow);
+        }
+    }
+
+    public static boolean isReversed(Scan scan) {
+        byte[] reversed = scan.getAttribute(REVERSED_ATTR);
+        return (PDataType.TRUE_BYTES.equals(reversed));
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/50d523f6/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
new file mode 100644
index 0000000..7b070f7
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
@@ -0,0 +1,507 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.hadoop.hbase.index.util.ImmutableBytesPtr;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.exception.SQLExceptionInfo;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.AmbiguousColumnException;
+import org.apache.phoenix.schema.ColumnFamilyNotFoundException;
+import org.apache.phoenix.schema.ColumnModifier;
+import org.apache.phoenix.schema.ColumnNotFoundException;
+import org.apache.phoenix.schema.PColumn;
+import org.apache.phoenix.schema.PColumnFamily;
+import org.apache.phoenix.schema.PDataType;
+import org.apache.phoenix.schema.PDatum;
+import org.apache.phoenix.schema.PMetaData;
+import org.apache.phoenix.schema.PName;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.RowKeySchema;
+import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder;
+import org.apache.phoenix.schema.SaltingUtil;
+
+
+
+/**
+ * 
+ * Static class for various schema-related utilities
+ *
+ * @author jtaylor
+ * @since 0.1
+ */
+public class SchemaUtil {
+    private static final int VAR_LENGTH_ESTIMATE = 10;
+    
+    public static final DataBlockEncoding DEFAULT_DATA_BLOCK_ENCODING = DataBlockEncoding.FAST_DIFF;
+    public static final RowKeySchema VAR_BINARY_SCHEMA = new RowKeySchemaBuilder(1).addField(new PDatum() {
+    
+        @Override
+        public boolean isNullable() {
+            return false;
+        }
+    
+        @Override
+        public PDataType getDataType() {
+            return PDataType.VARBINARY;
+        }
+    
+        @Override
+        public Integer getByteSize() {
+            return null;
+        }
+    
+        @Override
+        public Integer getMaxLength() {
+            return null;
+        }
+    
+        @Override
+        public Integer getScale() {
+            return null;
+        }
+    
+        @Override
+        public ColumnModifier getColumnModifier() {
+            return null;
+        }
+        
+    }, false, null).build();
+    
+    /**
+     * May not be instantiated
+     */
+    private SchemaUtil() {
+    }
+
+    public static boolean isPKColumn(PColumn column) {
+        return column.getFamilyName() == null;
+    }
+    
+    /**
+     * Estimate the max key length in bytes of the PK for a given table
+     * @param table the table
+     * @return the max PK length
+     */
+    public static int estimateKeyLength(PTable table) {
+        int maxKeyLength = 0;
+        // Calculate the max length of a key (each part must currently be of a fixed width)
+        int i = 0;
+        List<PColumn> columns = table.getPKColumns();
+        while (i < columns.size()) {
+            PColumn keyColumn = columns.get(i++);
+            Integer byteSize = keyColumn.getByteSize();
+            maxKeyLength += (byteSize == null) ? VAR_LENGTH_ESTIMATE : byteSize;
+        }
+        return maxKeyLength;
+    }
+
+    /**
+     * Normalize an identifier. If name is surrounded by double quotes,
+     * it is used as-is, otherwise the name is upper caased.
+     * @param name the parsed identifier
+     * @return the normalized identifier
+     */
+    public static String normalizeIdentifier(String name) {
+        if (name == null) {
+            return name;
+        }
+        if (isCaseSensitive(name)) {
+            // Don't upper case if in quotes
+            return name.substring(1, name.length()-1);
+        }
+        return name.toUpperCase();
+    }
+
+    public static boolean isCaseSensitive(String name) {
+        return name.length() > 0 && name.charAt(0)=='"';
+    }
+    
+    public static <T> List<T> concat(List<T> l1, List<T> l2) {
+        int size1 = l1.size();
+        if (size1 == 0) {
+            return l2;
+        }
+        int size2 = l2.size();
+        if (size2 == 0) {
+            return l1;
+        }
+        List<T> l3 = new ArrayList<T>(size1 + size2);
+        l3.addAll(l1);
+        l3.addAll(l2);
+        return l3;
+    }
+
+    public static byte[] getSequenceKey(byte[] tenantId, byte[] schemaName, byte[] sequenceName) {
+        return getTableKey(tenantId, schemaName, sequenceName);
+    }
+
+    public static byte[] getSequenceKey(String tenantId, String schemaName, String sequenceName) {
+        return getTableKey(tenantId, schemaName, sequenceName);
+    }
+
+    /**
+     * Get the key used in the Phoenix metadata row for a table definition
+     * @param schemaName
+     * @param tableName
+     */
+    public static byte[] getTableKey(byte[] tenantId, byte[] schemaName, byte[] tableName) {
+        return ByteUtil.concat(tenantId, QueryConstants.SEPARATOR_BYTE_ARRAY, schemaName, QueryConstants.SEPARATOR_BYTE_ARRAY, tableName);
+    }
+
+    public static byte[] getTableKey(String tenantId, String schemaName, String tableName) {
+        return ByteUtil.concat(tenantId == null  ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(tenantId), QueryConstants.SEPARATOR_BYTE_ARRAY, schemaName == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(schemaName), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(tableName));
+    }
+
+    public static String getTableName(String schemaName, String tableName) {
+        return getName(schemaName,tableName);
+    }
+
+    private static String getName(String optionalQualifier, String name) {
+        if (optionalQualifier == null || optionalQualifier.isEmpty()) {
+            return name;
+        }
+        return optionalQualifier + QueryConstants.NAME_SEPARATOR + name;
+    }
+
+    public static String getTableName(byte[] schemaName, byte[] tableName) {
+        return getName(schemaName, tableName);
+    }
+
+    public static String getColumnDisplayName(byte[] cf, byte[] cq) {
+        return getName(cf == null || cf.length == 0 || Bytes.compareTo(cf, QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES) == 0 ? ByteUtil.EMPTY_BYTE_ARRAY : cf, cq);
+    }
+
+    public static String getColumnDisplayName(String cf, String cq) {
+        return getName(cf == null || cf.isEmpty() || QueryConstants.DEFAULT_COLUMN_FAMILY.equals(cf) ? null : cf, cq);
+    }
+
+    public static String getMetaDataEntityName(String schemaName, String tableName, String familyName, String columnName) {
+        if ((schemaName == null || schemaName.isEmpty()) && (tableName == null || tableName.isEmpty())) {
+            return getName(familyName, columnName);
+        }
+        if ((familyName == null || familyName.isEmpty()) && (columnName == null || columnName.isEmpty())) {
+            return getName(schemaName, tableName);
+        }
+        return getName(getName(schemaName, tableName), getName(familyName, columnName));
+    }
+
+    public static String getColumnName(String familyName, String columnName) {
+        return getName(familyName, columnName);
+    }
+
+    public static byte[] getTableNameAsBytes(String schemaName, String tableName) {
+        if (schemaName == null || schemaName.length() == 0) {
+            return StringUtil.toBytes(tableName);
+        }
+        return getTableNameAsBytes(StringUtil.toBytes(schemaName),StringUtil.toBytes(tableName));
+    }
+
+    public static byte[] getTableNameAsBytes(byte[] schemaName, byte[] tableName) {
+        return getNameAsBytes(schemaName, tableName);
+    }
+
+    private static byte[] getNameAsBytes(byte[] nameOne, byte[] nameTwo) {
+        if (nameOne == null || nameOne.length == 0) {
+            return nameTwo;
+        } else if ((nameTwo == null || nameTwo.length == 0)) {
+            return nameOne;
+        } else {
+            return ByteUtil.concat(nameOne, QueryConstants.NAME_SEPARATOR_BYTES, nameTwo);
+        }
+    }
+
+    public static String getName(byte[] nameOne, byte[] nameTwo) {
+        return Bytes.toString(getNameAsBytes(nameOne,nameTwo));
+    }
+
+    public static int getVarCharLength(byte[] buf, int keyOffset, int maxLength) {
+        return getVarCharLength(buf, keyOffset, maxLength, 1);
+    }
+
+    public static int getVarCharLength(byte[] buf, int keyOffset, int maxLength, int skipCount) {
+        int length = 0;
+        for (int i=0; i<skipCount; i++) {
+            while (length < maxLength && buf[keyOffset+length] != QueryConstants.SEPARATOR_BYTE) {
+                length++;
+            }
+            if (i != skipCount-1) { // skip over the separator if it's not the last one.
+                length++;
+            }
+        }
+        return length;
+    }
+
+    public static int getVarChars(byte[] rowKey, byte[][] rowKeyMetadata) {
+        return getVarChars(rowKey, 0, rowKey.length, 0, rowKeyMetadata);
+    }
+    
+    public static int getVarChars(byte[] rowKey, int colMetaDataLength, byte[][] colMetaData) {
+        return getVarChars(rowKey, 0, rowKey.length, 0, colMetaDataLength, colMetaData);
+    }
+    
+    public static int getVarChars(byte[] rowKey, int keyOffset, int keyLength, int colMetaDataOffset, byte[][] colMetaData) {
+        return getVarChars(rowKey, keyOffset, keyLength, colMetaDataOffset, colMetaData.length, colMetaData);
+    }
+    
+    public static int getVarChars(byte[] rowKey, int keyOffset, int keyLength, int colMetaDataOffset, int colMetaDataLength, byte[][] colMetaData) {
+        int i, offset = keyOffset;
+        for (i = colMetaDataOffset; i < colMetaDataLength && keyLength > 0; i++) {
+            int length = getVarCharLength(rowKey, offset, keyLength);
+            byte[] b = new byte[length];
+            System.arraycopy(rowKey, offset, b, 0, length);
+            offset += length + 1;
+            keyLength -= length + 1;
+            colMetaData[i] = b;
+        }
+        return i;
+    }
+    
+    public static String findExistingColumn(PTable table, List<PColumn> columns) {
+        for (PColumn column : columns) {
+            PName familyName = column.getFamilyName();
+            if (familyName == null) {
+                try {
+                    return table.getPKColumn(column.getName().getString()).getName().getString();
+                } catch (ColumnNotFoundException e) {
+                    continue;
+                }
+            } else {
+                try {
+                    return table.getColumnFamily(familyName.getString()).getColumn(column.getName().getString()).getName().getString();
+                } catch (ColumnFamilyNotFoundException e) {
+                    continue; // Shouldn't happen
+                } catch (ColumnNotFoundException e) {
+                    continue;
+                }
+            }
+        }
+        return null;
+    }
+
+    public static String toString(byte[][] values) {
+        if (values == null) {
+            return "null";
+        }
+        StringBuilder buf = new StringBuilder("[");
+        for (byte[] value : values) {
+            buf.append(Bytes.toStringBinary(value));
+            buf.append(',');
+        }
+        buf.setCharAt(buf.length()-1, ']');
+        return buf.toString();
+    }
+
+    public static String toString(PDataType type, byte[] value) {
+        boolean isString = type.isCoercibleTo(PDataType.VARCHAR);
+        return isString ? ("'" + type.toObject(value).toString() + "'") : type.toObject(value).toString();
+    }
+
+    public static byte[] getEmptyColumnFamily(List<PColumnFamily> families) {
+        return families.isEmpty() ? QueryConstants.EMPTY_COLUMN_BYTES : families.get(0).getName().getBytes();
+    }
+
+    public static ImmutableBytesPtr getEmptyColumnFamilyPtr(List<PColumnFamily> families) {
+        return families.isEmpty() ? QueryConstants.EMPTY_COLUMN_BYTES_PTR : families.get(0)
+                .getName().getBytesPtr();
+    }
+
+    public static boolean isMetaTable(byte[] tableName) {
+        return Bytes.compareTo(tableName, TYPE_TABLE_NAME_BYTES) == 0;
+    }
+    
+    public static boolean isSequenceTable(byte[] tableName) {
+        return Bytes.compareTo(tableName, PhoenixDatabaseMetaData.SEQUENCE_TABLE_NAME_BYTES) == 0;
+    }
+
+    public static boolean isMetaTable(PTable table) {
+        return PhoenixDatabaseMetaData.TYPE_SCHEMA.equals(table.getSchemaName().getString()) && PhoenixDatabaseMetaData.TYPE_TABLE.equals(table.getTableName().getString());
+    }
+    
+    public static boolean isMetaTable(byte[] schemaName, byte[] tableName) {
+        return Bytes.compareTo(schemaName, PhoenixDatabaseMetaData.TYPE_SCHEMA_BYTES) == 0 && Bytes.compareTo(tableName, PhoenixDatabaseMetaData.TYPE_TABLE_BYTES) == 0;
+    }
+    
+    public static boolean isMetaTable(String schemaName, String tableName) {
+        return PhoenixDatabaseMetaData.TYPE_SCHEMA.equals(schemaName) && PhoenixDatabaseMetaData.TYPE_TABLE.equals(tableName);
+    }
+
+    // Given the splits and the rowKeySchema, find out the keys that 
+    public static byte[][] processSplits(byte[][] splits, LinkedHashSet<PColumn> pkColumns, Integer saltBucketNum, boolean defaultRowKeyOrder) throws SQLException {
+        // FIXME: shouldn't this return if splits.length == 0?
+        if (splits == null) return null;
+        // We do not accept user specified splits if the table is salted and we specify defaultRowKeyOrder. In this case,
+        // throw an exception.
+        if (splits.length > 0 && saltBucketNum != null && defaultRowKeyOrder) {
+            throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_SPLITS_ON_SALTED_TABLE).build().buildException();
+        }
+        // If the splits are not specified and table is salted, pre-split the table. 
+        if (splits.length == 0 && saltBucketNum != null) {
+            splits = SaltingUtil.getSalteByteSplitPoints(saltBucketNum);
+        }
+        byte[][] newSplits = new byte[splits.length][];
+        for (int i=0; i<splits.length; i++) {
+            newSplits[i] = processSplit(splits[i], pkColumns); 
+        }
+        return newSplits;
+    }
+
+    // Go through each slot in the schema and try match it with the split byte array. If the split
+    // does not confer to the schema, extends its length to match the schema.
+    private static byte[] processSplit(byte[] split, LinkedHashSet<PColumn> pkColumns) {
+        int pos = 0, offset = 0, maxOffset = split.length;
+        Iterator<PColumn> iterator = pkColumns.iterator();
+        while (pos < pkColumns.size()) {
+            PColumn column = iterator.next();
+            if (column.getDataType().isFixedWidth()) { // Fixed width
+                int length = column.getByteSize();
+                if (maxOffset - offset < length) {
+                    // The split truncates the field. Fill in the rest of the part and any fields that
+                    // are missing after this field.
+                    int fillInLength = length - (maxOffset - offset);
+                    fillInLength += estimatePartLength(pos + 1, iterator);
+                    return ByteUtil.fillKey(split, split.length + fillInLength);
+                }
+                // Account for this field, move to next position;
+                offset += length;
+                pos++;
+            } else { // Variable length
+                // If we are the last slot, then we are done. Nothing needs to be filled in.
+                if (pos == pkColumns.size() - 1) {
+                    break;
+                }
+                while (offset < maxOffset && split[offset] != QueryConstants.SEPARATOR_BYTE) {
+                    offset++;
+                }
+                if (offset == maxOffset) {
+                    // The var-length field does not end with a separator and it's not the last field.
+                    int fillInLength = 1; // SEPARATOR byte for the current var-length slot.
+                    fillInLength += estimatePartLength(pos + 1, iterator);
+                    return ByteUtil.fillKey(split, split.length + fillInLength);
+                }
+                // Move to the next position;
+                offset += 1; // skip separator;
+                pos++;
+            }
+        }
+        return split;
+    }
+
+    // Estimate the key length after pos slot for schema.
+    private static int estimatePartLength(int pos, Iterator<PColumn> iterator) {
+        int length = 0;
+        while (iterator.hasNext()) {
+            PColumn column = iterator.next();
+            if (column.getDataType().isFixedWidth()) {
+                length += column.getByteSize();
+            } else {
+                length += 1; // SEPARATOR byte.
+            }
+        }
+        return length;
+    }
+    
+    public static final String UPGRADE_TO_2_0 = "UpgradeTo20";
+    public static final String UPGRADE_TO_2_1 = "UpgradeTo21";
+
+    public static String getEscapedTableName(String schemaName, String tableName) {
+        if (schemaName == null || schemaName.length() == 0) {
+            return "\"" + tableName + "\"";
+        }
+        return "\"" + schemaName + "\"." + "\"" + tableName + "\"";
+    }
+
+    protected static PhoenixConnection addMetaDataColumn(PhoenixConnection conn, long scn, String columnDef) throws SQLException {
+        String url = conn.getURL();
+        Properties props = conn.getClientInfo();
+        PMetaData metaData = conn.getPMetaData();
+        props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(scn));
+        PhoenixConnection metaConnection = null;
+
+        Statement stmt = null;
+        try {
+            metaConnection = new PhoenixConnection(conn.getQueryServices(), url, props, metaData);
+            try {
+                stmt = metaConnection.createStatement();
+                stmt.executeUpdate("ALTER TABLE SYSTEM.\"TABLE\" ADD IF NOT EXISTS " + columnDef);
+                return metaConnection;
+            } finally {
+                if(stmt != null) {
+                    stmt.close();
+                }
+            }
+        } finally {
+            if(metaConnection != null) {
+                metaConnection.close();
+            }
+        }
+    }
+    
+    public static boolean columnExists(PTable table, String columnName) {
+        try {
+            table.getColumn(columnName);
+            return true;
+        } catch (ColumnNotFoundException e) {
+            return false;
+        } catch (AmbiguousColumnException e) {
+            return true;
+        }
+    }
+    
+    public static String getSchemaNameFromFullName(String tableName) {
+        int index = tableName.indexOf(QueryConstants.NAME_SEPARATOR);
+        if (index < 0) {
+            return ""; 
+        }
+        return tableName.substring(0, index);
+    }
+    
+    public static String getTableNameFromFullName(String tableName) {
+        int index = tableName.indexOf(QueryConstants.NAME_SEPARATOR);
+        if (index < 0) {
+            return tableName; 
+        }
+        return tableName.substring(index+1, tableName.length());
+    }
+
+    public static byte[] getTableKeyFromFullName(String fullTableName) {
+        int index = fullTableName.indexOf(QueryConstants.NAME_SEPARATOR);
+        if (index < 0) {
+            return getTableKey(null, null, fullTableName); 
+        }
+        String schemaName = fullTableName.substring(0, index);
+        String tableName = fullTableName.substring(index+1);
+        return getTableKey(null, schemaName, tableName); 
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/50d523f6/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java
new file mode 100644
index 0000000..819bcc4
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+
+import org.apache.phoenix.exception.PhoenixIOException;
+import org.apache.phoenix.exception.SQLExceptionCode;
+
+
+public class ServerUtil {
+
+    private static final String FORMAT = "ERROR %d (%s): %s";
+    private static final Pattern PATTERN = Pattern.compile("ERROR (\\d+) \\((\\w+)\\): (.*)");
+    private static final Map<Class<? extends Exception>, SQLExceptionCode> errorcodeMap
+        = new HashMap<Class<? extends Exception>, SQLExceptionCode>();
+    static {
+        // Map a normal exception into a corresponding SQLException.
+        errorcodeMap.put(ArithmeticException.class, SQLExceptionCode.SERVER_ARITHMETIC_ERROR);
+    }
+
+    public static void throwIOException(String msg, Throwable t) throws IOException {
+        // First unwrap SQLExceptions if it's root cause is an IOException.
+        if (t instanceof SQLException) {
+            Throwable cause = t.getCause();
+            if (cause instanceof IOException) {
+                t = cause;
+            }
+        }
+        // Throw immediately if DoNotRetryIOException
+        if (t instanceof DoNotRetryIOException) {
+            throw (DoNotRetryIOException)t;
+        } else if (t instanceof IOException) {
+            // If the IOException does not wrap any exception, then bubble it up.
+            Throwable cause = t.getCause();
+            if (cause == null || cause instanceof IOException) {
+                throw (IOException)t;
+            }
+            // Else assume it's been wrapped, so throw as DoNotRetryIOException to prevent client hanging while retrying
+            throw new DoNotRetryIOException(t.getMessage(), cause);
+        } else if (t instanceof SQLException) {
+            // If it's already an SQLException, construct an error message so we can parse and reconstruct on the client side.
+            throw new DoNotRetryIOException(constructSQLErrorMessage((SQLException) t, msg), t);
+        } else {
+            // Not a DoNotRetryIOException, IOException or SQLException. Map the exception type to a general SQLException 
+            // and construct the error message so it can be reconstruct on the client side.
+            //
+            // If no mapping exists, rethrow it as a generic exception.
+            SQLExceptionCode code = errorcodeMap.get(t.getClass());
+            if (code == null) {
+                throw new DoNotRetryIOException(msg + ": " + t.getMessage(), t);
+            } else {
+                throw new DoNotRetryIOException(constructSQLErrorMessage(code, t, msg), t);
+            }
+        }
+    }
+
+    private static String constructSQLErrorMessage(SQLExceptionCode code, Throwable e, String message) {
+        return constructSQLErrorMessage(code.getErrorCode(), code.getSQLState(), code.getMessage() + " " + e.getMessage() + " " + message);
+    }
+
+    private static String constructSQLErrorMessage(SQLException e, String message) {
+        return constructSQLErrorMessage(e.getErrorCode(), e.getSQLState(), e.getMessage() + " " + message);
+    }
+
+    private static String constructSQLErrorMessage(int errorCode, String SQLState, String message) {
+        return String.format(FORMAT, errorCode, SQLState, message);
+    }
+
+    public static SQLException parseServerException(Throwable t) {
+        SQLException e = parseServerExceptionOrNull(t);
+        if (e != null) {
+            return e;
+        }
+        return new PhoenixIOException(t);
+    }
+    
+    public static SQLException parseServerExceptionOrNull(Throwable t) {
+        while (t.getCause() != null) {
+            t = t.getCause();
+        }
+        return parseRemoteException(t);
+    }
+
+    private static SQLException parseRemoteException(Throwable t) {
+        	String message = t.getLocalizedMessage();
+        	if (message != null) {
+            // If the message matches the standard pattern, recover the SQLException and throw it.
+            Matcher matcher = PATTERN.matcher(t.getLocalizedMessage());
+            if (matcher.find()) {
+                int errorCode = Integer.parseInt(matcher.group(1));
+                String sqlState = matcher.group(2);
+                return new SQLException(matcher.group(), sqlState, errorCode, t);
+            }
+        	}
+        return null;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/50d523f6/phoenix-core/src/main/java/org/apache/phoenix/util/SizedUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SizedUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SizedUtil.java
new file mode 100644
index 0000000..62f0816
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SizedUtil.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+
+/**
+ * Utilities for computing an object's size.  All size measurements are in bytes.
+ * Note, all of the sizes here, but especially OBJECT_SIZE and ARRAY_SIZE are estimates and will
+ * depend on the JVM itself (and which JVM, 64bit vs. 32bit, etc).
+ * The current values are based on:
+ * Java HotSpot(TM) 64-Bit Server VM/14.2-b01
+ * 
+ * (Uncomment out and run the main w/ appropriate object to test)
+ * Also, see this link:
+ * https://sites.google.com/a/salesforce.com/development/Home/old-wiki-home-page/i-wish-i-knew#TOC-How-to-measure-the-size-of-a-Java-O
+ * For another way to measure.
+ */
+public class SizedUtil {
+    public static final int POINTER_SIZE = 8; // 64 bit jvm.
+    public static final int OBJECT_SIZE = 16; // measured, see class comment.
+    public static final int ARRAY_SIZE = 24; // measured, see class comment.
+    public static final int CHAR_SIZE = 2;
+    public static final int INT_SIZE = 4;
+    public static final int LONG_SIZE = 8;
+    
+    public static final int MAP_ENTRY_SIZE = OBJECT_SIZE + 3 * POINTER_SIZE + INT_SIZE;
+    public static final int IMMUTABLE_BYTES_WRITABLE_SIZE = OBJECT_SIZE + INT_SIZE * 2 + ARRAY_SIZE;
+    public static final int IMMUTABLE_BYTES_PTR_SIZE = IMMUTABLE_BYTES_WRITABLE_SIZE + INT_SIZE;// Extra is an int field which caches hashcode.
+    public static final int KEY_VALUE_SIZE = 2 * INT_SIZE + LONG_SIZE + 2 * ARRAY_SIZE;
+    public static final int RESULT_SIZE = OBJECT_SIZE +  3 * POINTER_SIZE + IMMUTABLE_BYTES_WRITABLE_SIZE;
+    public static final int INT_OBJECT_SIZE = INT_SIZE + OBJECT_SIZE;
+    public static final int LONG_OBJECT_SIZE = LONG_SIZE + OBJECT_SIZE;
+    public static final int BIG_DECIMAL_SIZE = 
+        OBJECT_SIZE + 2 * INT_SIZE + LONG_SIZE + 2 * POINTER_SIZE +
+        OBJECT_SIZE /* BigInteger */ + 5 * INT_SIZE + ARRAY_SIZE /*mag[]*/ + 2 * INT_SIZE /* est mag[2] */;
+
+    private SizedUtil() {
+    }
+    
+    public static int sizeOfMap(int nRows, int keySize, int valueSize) {
+        return nRows * (
+                SizedUtil.MAP_ENTRY_SIZE + // entry
+                keySize + // key size
+                valueSize); // value size
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/50d523f6/phoenix-core/src/main/java/org/apache/phoenix/util/StringUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/StringUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/StringUtil.java
new file mode 100644
index 0000000..1e9d961
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/StringUtil.java
@@ -0,0 +1,246 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+import org.apache.phoenix.schema.ColumnModifier;
+
+
+public class StringUtil {
+    public static final String EMPTY_STRING = "";
+    // Masks to determine how many bytes are in each character
+    // From http://tools.ietf.org/html/rfc3629#section-3
+    public static final byte SPACE_UTF8 = 0x20;
+    private static final int BYTES_1_MASK = 0xFF << 7; // 0xxxxxxx is a single byte char
+    private static final int BYTES_2_MASK = 0xFF << 5; // 110xxxxx is a double byte char
+    private static final int BYTES_3_MASK = 0xFF << 4; // 1110xxxx is a triple byte char
+    private static final int BYTES_4_MASK = 0xFF << 3; // 11110xxx is a quadruple byte char
+    
+    public static final byte[] MOD_SPACE_UTF8 = new byte[ColumnModifier.values().length];
+    static {
+        for (ColumnModifier columnModifier : ColumnModifier.values()) {
+            MOD_SPACE_UTF8[columnModifier.ordinal()] = columnModifier.apply(new byte[] {SPACE_UTF8}, 0, new byte[1], 0, 1)[0];
+        }
+    }
+
+    public final static char SINGLE_CHAR_WILDCARD = '?';
+    public final static char SINGLE_CHAR_LIKE = '_';
+    public final static char MULTI_CHAR_WILDCARD = '*';
+    public final static char MULTI_CHAR_LIKE = '%';
+    public final static String[] LIKE_ESCAPE_SEQS = new String[]{"\\"+SINGLE_CHAR_LIKE, "\\"+MULTI_CHAR_LIKE};
+    public final static String[] LIKE_UNESCAPED_SEQS = new String[]{""+SINGLE_CHAR_LIKE, ""+MULTI_CHAR_LIKE};
+    
+
+    private StringUtil() {
+    }
+
+    /** Replace instances of character ch in String value with String replacement */
+    public static String replaceChar(String value, char ch, CharSequence replacement) {
+        if (value == null)
+            return null;
+        int i = value.indexOf(ch);
+        if (i == -1)
+            return value; // nothing to do
+
+        // we've got at least one character to replace
+        StringBuilder buf = new StringBuilder(value.length() + 16); // some extra space
+        int j = 0;
+        while (i != -1) {
+            buf.append(value, j, i).append(replacement);
+            j = i + 1;
+            i = value.indexOf(ch, j);
+        }
+        if (j < value.length())
+            buf.append(value, j, value.length());
+        return buf.toString();
+    }
+
+    /**
+     * @return the replacement of all occurrences of src[i] with target[i] in s. Src and target are not regex's so this
+     *         uses simple searching with indexOf()
+     */
+    public static String replace(String s, String[] src, String[] target) {
+        assert src != null && target != null && src.length > 0 && src.length == target.length;
+        if (src.length == 1 && src[0].length() == 1) {
+            return replaceChar(s, src[0].charAt(0), target[0]);
+        }
+        if (s == null)
+            return null;
+        StringBuilder sb = new StringBuilder(s.length());
+        int pos = 0;
+        int limit = s.length();
+        int lastMatch = 0;
+        while (pos < limit) {
+            boolean matched = false;
+            for (int i = 0; i < src.length; i++) {
+                if (s.startsWith(src[i], pos) && src[i].length() > 0) {
+                    // we found a matching pattern - append the acculumation plus the replacement
+                    sb.append(s.substring(lastMatch, pos)).append(target[i]);
+                    pos += src[i].length();
+                    lastMatch = pos;
+                    matched = true;
+                    break;
+                }
+            }
+            if (!matched) {
+                // we didn't match any patterns, so move forward 1 character
+                pos++;
+            }
+        }
+        // see if we found any matches
+        if (lastMatch == 0) {
+            // we didn't match anything, so return the source string
+            return s;
+        }
+        
+        // apppend the trailing portion
+        sb.append(s.substring(lastMatch));
+        
+        return sb.toString();
+    }
+
+    public static int getBytesInChar(byte b, ColumnModifier columnModifier) {
+        if (columnModifier != null) {
+            b = columnModifier.apply(b);
+        }
+        int c = b & 0xff;
+        if ((c & BYTES_1_MASK) == 0)
+            return 1;
+        if ((c & BYTES_2_MASK) == 0xC0)
+            return 2;
+        if ((c & BYTES_3_MASK) == 0xE0)
+            return 3;
+        if ((c & BYTES_4_MASK) == 0xF0)
+            return 4;
+        // Any thing else in the first byte is invalid
+        throw new RuntimeException("Undecodable byte: " + b);
+    }
+
+    public static int calculateUTF8Length(byte[] bytes, int offset, int length, ColumnModifier columnModifier) throws UnsupportedEncodingException {
+        int i = offset, endOffset = offset + length;
+        length = 0;
+        while (i < endOffset) {
+            int charLength = getBytesInChar(bytes[i], columnModifier);
+            i += charLength;
+            length++;
+        }
+        return length;
+    }
+
+    // Given an array of bytes containing encoding utf-8 encoded strings, the offset and a length
+    // parameter, return the actual index into the byte array which would represent a substring
+    // of <length> starting from the character at <offset>. We assume the <offset> is the start
+    // byte of an UTF-8 character.
+    public static int getByteLengthForUtf8SubStr(byte[] bytes, int offset, int length, ColumnModifier columnModifier) throws UnsupportedEncodingException {
+        int byteLength = 0;
+        while(length > 0 && offset + byteLength < bytes.length) {
+            int charLength = getBytesInChar(bytes[offset + byteLength], columnModifier);
+            byteLength += charLength;
+            length--;
+        }
+        return byteLength;
+    }
+
+    public static boolean hasMultiByteChars(String s) {
+        for (int i = 0; i < s.length(); i++) {
+            char c = s.charAt(i);
+            if (c > 0x007F) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public static int getFirstNonBlankCharIdxFromStart(byte[] string, int offset, int length, ColumnModifier columnModifier) {
+        int i = offset;
+        byte space = columnModifier == null ? SPACE_UTF8 : MOD_SPACE_UTF8[columnModifier.ordinal()];
+        for ( ; i < offset + length; i++) {
+            if (string[i] != space) {
+                break;
+            }
+        }
+        return i;
+    }
+
+    public static int getFirstNonBlankCharIdxFromEnd(byte[] string, int offset, int length, ColumnModifier columnModifier) {
+        int i = offset + length - 1;
+        byte space = columnModifier == null ? SPACE_UTF8 : MOD_SPACE_UTF8[columnModifier.ordinal()];
+        for ( ; i >= offset; i--) {
+            if (string[i] != space) {
+                break;
+            }
+         }
+        return i;
+    }
+
+    // A toBytes function backed up HBase's utility function, but would accept null input, in which
+    // case it returns an empty byte array.
+    public static byte[] toBytes(String input) {
+        if (input == null) {
+            return ByteUtil.EMPTY_BYTE_ARRAY;
+        }
+        return Bytes.toBytes(input);
+    }
+
+    public static String escapeLike(String s) {
+        return replace(s, LIKE_UNESCAPED_SEQS, LIKE_ESCAPE_SEQS);
+    }
+
+    public static int getUnpaddedCharLength(byte[] b, int offset, int length, ColumnModifier columnModifier) {
+        return getFirstNonBlankCharIdxFromEnd(b, offset, length, columnModifier) - offset + 1;
+    }
+
+    public static byte[] padChar(byte[] value, int offset, int length, int paddedLength) {
+        byte[] key = new byte[paddedLength];
+        System.arraycopy(value,offset, key, 0, length);
+        Arrays.fill(key, length, paddedLength, SPACE_UTF8);
+        return key;
+    }
+
+    public static byte[] padChar(byte[] value, Integer byteSize) {
+        byte[] newValue = Arrays.copyOf(value, byteSize);
+        if (newValue.length > value.length) {
+            Arrays.fill(newValue, value.length, newValue.length, SPACE_UTF8);
+        }
+        return newValue;
+    }
+    
+    /**
+     * Lame - StringBuilder.equals is retarded.
+     * @param b1
+     * @param b2
+     * @return whether or not the two builders consist the same sequence of characters
+     */
+    public static boolean equals(StringBuilder b1, StringBuilder b2) {
+        if (b1.length() != b2.length()) {
+            return false;
+        }
+        for (int i = 0; i < b1.length(); i++) {
+            if (b1.charAt(i) != b2.charAt(i)) {
+                return false;
+            }
+        }
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/50d523f6/phoenix-core/src/main/java/org/apache/phoenix/util/TrustedByteArrayOutputStream.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/TrustedByteArrayOutputStream.java b/phoenix-core/src/main/java/org/apache/phoenix/util/TrustedByteArrayOutputStream.java
new file mode 100644
index 0000000..f503cb3
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/TrustedByteArrayOutputStream.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+/**
+ * 
+ * Derived version of {@link java.io.ByteArrayOutputStream} that provides access
+ * to underlying byte array buffer so that it doesn't have to be copied
+ *
+ * @author jtaylor
+ * @since 0.1
+ */
+public class TrustedByteArrayOutputStream extends ByteArrayOutputStream {
+    public TrustedByteArrayOutputStream(int initialSize) {
+        super(initialSize);
+    }
+    public byte[] getBuffer() {
+        return buf;
+    }
+    @Override
+    public byte[] toByteArray() {
+        if (buf.length == size()) {
+            return buf;
+        }
+        return super.toByteArray();
+    }
+    @Override
+    public void write(byte[] b) {
+        try {
+            super.write(b);
+        } catch (IOException e) {
+            throw new RuntimeException(e); // Impossible
+        }
+    }
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/50d523f6/phoenix-core/src/main/java/org/apache/phoenix/util/TupleUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/TupleUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/TupleUtil.java
new file mode 100644
index 0000000..9a8608a
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/TupleUtil.java
@@ -0,0 +1,155 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN;
+import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN_FAMILY;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.WritableUtils;
+
+import org.apache.hadoop.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.expression.Expression;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.tuple.Tuple;
+
+
+/**
+ * 
+ * Utilities for Tuple
+ *
+ * @author jtaylor
+ * @since 0.1
+ */
+public class TupleUtil {
+    private TupleUtil() {
+    }
+    
+    public static boolean equals(Tuple t1, Tuple t2, ImmutableBytesWritable ptr) {
+        t1.getKey(ptr);
+        byte[] buf = ptr.get();
+        int offset = ptr.getOffset();
+        int length = ptr.getLength();
+        t2.getKey(ptr);
+        return Bytes.compareTo(buf, offset, length, ptr.get(), ptr.getOffset(), ptr.getLength()) == 0;
+    }
+    
+    public static int compare(Tuple t1, Tuple t2, ImmutableBytesWritable ptr) {
+        return compare(t1, t2, ptr, 0);
+    }
+    
+    public static int compare(Tuple t1, Tuple t2, ImmutableBytesWritable ptr, int keyOffset) {
+        t1.getKey(ptr);
+        byte[] buf = ptr.get();
+        int offset = ptr.getOffset() + keyOffset;
+        int length = ptr.getLength() - keyOffset;
+        t2.getKey(ptr);
+        return Bytes.compareTo(buf, offset, length, ptr.get(), ptr.getOffset() + keyOffset, ptr.getLength() - keyOffset);
+    }
+    
+    /**
+     * Set ptr to point to the value contained in the first KeyValue without
+     * exploding Result into KeyValue array.
+     * @param r
+     * @param ptr
+     */
+    public static void getAggregateValue(Tuple r, ImmutableBytesWritable ptr) {
+        if (r.size() == 1) {
+            KeyValue kv = r.getValue(0); // Just one KV for aggregation
+            if (Bytes.compareTo(SINGLE_COLUMN_FAMILY, 0, SINGLE_COLUMN_FAMILY.length, kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength()) == 0) {
+                if (Bytes.compareTo(SINGLE_COLUMN, 0, SINGLE_COLUMN.length, kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength()) == 0) {
+                    ptr.set(kv.getBuffer(), kv.getValueOffset(), kv.getValueLength());
+                    return;
+                }
+            }
+        }
+        throw new IllegalStateException("Expected single, aggregated KeyValue from coprocessor, but instead received " + r + ". Ensure aggregating coprocessors are loaded correctly on server");
+    }
+    
+    /** Concatenate results evaluated against a list of expressions
+     * 
+     * @param result the tuple for expression evaluation
+     * @param expressions
+     * @return the concatenated byte array as ImmutableBytesWritable
+     * @throws IOException
+     */
+    public static ImmutableBytesPtr getConcatenatedValue(Tuple result, List<Expression> expressions) throws IOException {
+        ImmutableBytesPtr value = new ImmutableBytesPtr(ByteUtil.EMPTY_BYTE_ARRAY);
+        Expression expression = expressions.get(0);
+        boolean evaluated = expression.evaluate(result, value);
+        
+        if (expressions.size() == 1) {
+            if (!evaluated) {
+                value.set(ByteUtil.EMPTY_BYTE_ARRAY);
+            }
+            return value;
+        } else {
+            TrustedByteArrayOutputStream output = new TrustedByteArrayOutputStream(value.getLength() * expressions.size());
+            try {
+                if (evaluated) {
+                    output.write(value.get(), value.getOffset(), value.getLength());
+                }
+                for (int i = 1; i < expressions.size(); i++) {
+                    if (!expression.getDataType().isFixedWidth()) {
+                        output.write(QueryConstants.SEPARATOR_BYTE);
+                    }
+                    expression = expressions.get(i);
+                    // TODO: should we track trailing null values and omit the separator bytes?
+                    if (expression.evaluate(result, value)) {
+                        output.write(value.get(), value.getOffset(), value.getLength());
+                    } else if (i < expressions.size()-1 && expression.getDataType().isFixedWidth()) {
+                        // This should never happen, because any non terminating nullable fixed width type (i.e. INT or LONG) is
+                        // converted to a variable length type (i.e. DECIMAL) to allow an empty byte array to represent null.
+                        throw new DoNotRetryIOException("Non terminating null value found for fixed width expression (" + expression + ") in row: " + result);
+                    }
+                }
+                byte[] outputBytes = output.getBuffer();
+                value.set(outputBytes, 0, output.size());
+                return value;
+            } finally {
+                output.close();
+            }
+        }
+    }
+    
+    public static int write(Tuple result, DataOutput out) throws IOException {
+        int size = 0;
+        for(int i = 0; i < result.size(); i++) {
+            KeyValue kv = result.getValue(i);
+            size += kv.getLength();
+            size += Bytes.SIZEOF_INT; // kv.getLength
+          }
+
+        WritableUtils.writeVInt(out, size);
+        for(int i = 0; i < result.size(); i++) {
+            KeyValue kv = result.getValue(i);
+            out.writeInt(kv.getLength());
+            out.write(kv.getBuffer(), kv.getOffset(), kv.getLength());
+          }
+        return size;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/50d523f6/phoenix-core/src/main/java/overview.html
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/overview.html b/phoenix-core/src/main/java/overview.html
new file mode 100644
index 0000000..b58fa45
--- /dev/null
+++ b/phoenix-core/src/main/java/overview.html
@@ -0,0 +1,25 @@
+<html>
+<body>
+This document is for the JDBC implementation of Phoenix, targeting those who are interested in extending the level of SQL support.
+For users of Phoenix, the programmatic API is <a href="http://docs.oracle.com/javase/1.4.2/docs/api/java/sql/package-summary.html">JDBC</a>.
+For example, the following snippet demonstrates how to query a table:
+
+<pre>
+        Properties props = new Properties();
+        // Ensure the driver is on classpath
+        // Connect through Phoenix to the zookeeper quorum with a host name of myServer
+        Connection conn = DriverManager.getConnection("jdbc:phoenix:myServer", props);
+        try {
+            PreparedStatement statement = conn.prepareStatement("SELECT count(1) FROM product_metrics WHERE organization_id=?");
+            statement.setString(1, orgId);
+            ResultSet rs = statement.executeQuery();
+            if (rs.next()) {
+                System.out.println("Row count of orgId='" + orgId + "' is " + rs.getLong(1));
+            }
+        } finally {
+            conn.close();
+        }
+</pre>
+
+</body>
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/50d523f6/phoenix-core/src/main/resources/java.sql.Driver
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/resources/java.sql.Driver b/phoenix-core/src/main/resources/java.sql.Driver
new file mode 100644
index 0000000..fc51a04
--- /dev/null
+++ b/phoenix-core/src/main/resources/java.sql.Driver
@@ -0,0 +1 @@
+org.apache.phoenix.jdbc.PhoenixDriver

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/50d523f6/phoenix-core/src/site/bin/merge.sh
----------------------------------------------------------------------
diff --git a/phoenix-core/src/site/bin/merge.sh b/phoenix-core/src/site/bin/merge.sh
new file mode 100755
index 0000000..5ac0641
--- /dev/null
+++ b/phoenix-core/src/site/bin/merge.sh
@@ -0,0 +1,10 @@
+current_dir=$(cd $(dirname $0);pwd)
+cd $current_dir
+SITE_TARGET="../../../target/site"
+java -jar merge.jar ../language_reference_source/index.html $SITE_TARGET/language/index.html
+java -jar merge.jar ../language_reference_source/functions.html $SITE_TARGET/language/functions.html
+java -jar merge.jar ../language_reference_source/datatypes.html $SITE_TARGET/language/datatypes.html
+cd $SITE_TARGET
+
+grep -rl class=\"nav-collapse\" . | xargs sed -i 's/class=\"nav-collapse\"/class=\"nav-collapse collapse\"/g';grep -rl class=\"active\" . | xargs sed -i 's/class=\"active\"/class=\"divider\"/g'
+grep -rl "dropdown active" . | xargs sed -i 's/dropdown active/dropdown/g'


Mime
View raw message