phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jamestay...@apache.org
Subject phoenix git commit: PHOENIX-2120 Padding character is not inverted as required for DESC CHAR columns
Date Tue, 21 Jul 2015 00:53:49 GMT
Repository: phoenix
Updated Branches:
  refs/heads/master 936de8815 -> dcf845c25


PHOENIX-2120 Padding character is not inverted as required for DESC CHAR columns


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/dcf845c2
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/dcf845c2
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/dcf845c2

Branch: refs/heads/master
Commit: dcf845c25eb9bd619a999d16ce9e2f548ce7b491
Parents: 936de88
Author: James Taylor <jamestaylor@apache.org>
Authored: Mon Jul 20 17:52:53 2015 -0700
Committer: James Taylor <jamestaylor@apache.org>
Committed: Mon Jul 20 17:53:18 2015 -0700

----------------------------------------------------------------------
 .../apache/phoenix/end2end/LpadFunctionIT.java  |  48 +++-
 .../org/apache/phoenix/compile/KeyPart.java     |   8 +-
 .../apache/phoenix/compile/WhereOptimizer.java  | 132 +++++++---
 .../UngroupedAggregateRegionObserver.java       |  33 ++-
 .../phoenix/expression/LiteralExpression.java   |   3 +-
 .../expression/function/InvertFunction.java     |   8 +-
 .../expression/function/PrefixFunction.java     |  13 +-
 .../expression/function/RTrimFunction.java      |  15 +-
 .../function/RoundDateExpression.java           |  12 +-
 .../function/RoundDecimalExpression.java        |  22 +-
 .../query/ConnectionQueryServicesImpl.java      |   8 +-
 .../org/apache/phoenix/schema/PTableImpl.java   |  25 +-
 .../apache/phoenix/schema/types/PBinary.java    |  28 +-
 .../org/apache/phoenix/schema/types/PChar.java  |  11 +
 .../apache/phoenix/schema/types/PDataType.java  |   1 +
 .../org/apache/phoenix/util/PhoenixRuntime.java |  43 +++-
 .../org/apache/phoenix/util/StringUtil.java     |   7 -
 .../org/apache/phoenix/util/UpgradeUtil.java    | 256 +++++++++++++------
 18 files changed, 495 insertions(+), 178 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/it/java/org/apache/phoenix/end2end/LpadFunctionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LpadFunctionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LpadFunctionIT.java
index 4aa66c2..4070103 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LpadFunctionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LpadFunctionIT.java
@@ -16,6 +16,7 @@
  */
 package org.apache.phoenix.end2end;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -27,8 +28,10 @@ import java.sql.ResultSet;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.TestUtil;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import com.google.common.collect.Lists;
@@ -107,26 +110,59 @@ public class LpadFunctionIT extends BaseHBaseManagedTimeIT {
         testLpad(conn, inputList, length, fillStringList, "pk", expectedOutputList);
     }
 
-    @Ignore
     @Test
     public void testCharPadding() throws Exception {
         ResultSet rs;
         Connection conn = DriverManager.getConnection(getUrl());
+        
+        conn.createStatement().execute("CREATE TABLE t (k CHAR(3) PRIMARY KEY)");
+        conn.createStatement().execute("UPSERT INTO t VALUES('a')");
+        conn.createStatement().execute("UPSERT INTO t VALUES('ab')");
+        conn.commit();
+        rs = conn.createStatement().executeQuery("SELECT * FROM t ORDER BY k");
+        assertTrue(rs.next());
+        assertEquals("a", rs.getString(1));
+        assertTrue(rs.next());
+        assertEquals("ab", rs.getString(1));
+        assertFalse(rs.next());
 
         conn.createStatement().execute("CREATE TABLE tdesc (k CHAR(3) PRIMARY KEY DESC)");
         conn.createStatement().execute("UPSERT INTO tdesc VALUES('a')");
+        conn.createStatement().execute("UPSERT INTO tdesc VALUES('ab')");
         conn.commit();
-        rs = conn.createStatement().executeQuery("SELECT * FROM tdesc");
+        rs = conn.createStatement().executeQuery("SELECT * FROM tdesc ORDER BY k DESC");
+        assertTrue(rs.next());
+        assertEquals("ab", rs.getString(1));
         assertTrue(rs.next());
         assertEquals("a", rs.getString(1));
         assertFalse(rs.next());
+    }
+
+    @Test
+    public void testBinaryPadding() throws Exception {
+        ResultSet rs;
+        Connection conn = DriverManager.getConnection(getUrl());
         
-        conn.createStatement().execute("CREATE TABLE t (k CHAR(3) PRIMARY KEY)");
+        conn.createStatement().execute("CREATE TABLE t (k BINARY(3) PRIMARY KEY)");
         conn.createStatement().execute("UPSERT INTO t VALUES('a')");
+        conn.createStatement().execute("UPSERT INTO t VALUES('ab')");
         conn.commit();
-        rs = conn.createStatement().executeQuery("SELECT * FROM t");
+        rs = conn.createStatement().executeQuery("SELECT * FROM t ORDER BY k");
         assertTrue(rs.next());
-        assertEquals("a", rs.getString(1));
+        assertArrayEquals(ByteUtil.concat(Bytes.toBytes("a"), QueryConstants.SEPARATOR_BYTE_ARRAY, QueryConstants.SEPARATOR_BYTE_ARRAY), rs.getBytes(1));
+        assertTrue(rs.next());
+        assertArrayEquals(ByteUtil.concat(Bytes.toBytes("ab"), QueryConstants.SEPARATOR_BYTE_ARRAY), rs.getBytes(1));
+        assertFalse(rs.next());
+
+        conn.createStatement().execute("CREATE TABLE tdesc (k BINARY(3) PRIMARY KEY DESC)");
+        conn.createStatement().execute("UPSERT INTO tdesc VALUES('a')");
+        conn.createStatement().execute("UPSERT INTO tdesc VALUES('ab')");
+        conn.commit();
+        rs = conn.createStatement().executeQuery("SELECT * FROM tdesc ORDER BY k DESC");
+        assertTrue(rs.next());
+        assertArrayEquals(ByteUtil.concat(Bytes.toBytes("ab"), QueryConstants.SEPARATOR_BYTE_ARRAY), rs.getBytes(1));
+        assertTrue(rs.next());
+        assertArrayEquals(ByteUtil.concat(Bytes.toBytes("a"), QueryConstants.SEPARATOR_BYTE_ARRAY, QueryConstants.SEPARATOR_BYTE_ARRAY), rs.getBytes(1));
         assertFalse(rs.next());
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java
index 4eb53d3..55385f2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java
@@ -20,10 +20,10 @@ package org.apache.phoenix.compile;
 import java.util.List;
 
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
-
 import org.apache.phoenix.expression.Expression;
 import org.apache.phoenix.query.KeyRange;
 import org.apache.phoenix.schema.PColumn;
+import org.apache.phoenix.schema.PTable;
 
 /**
  * 
@@ -74,4 +74,10 @@ public interface KeyPart {
      * @return the primary key column for this key part
      */
     public PColumn getColumn();
+    
+    /**
+     * Gets the table metadata object associated with this key part
+     * @return the table for this key part
+     */
+    public PTable getTable();
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
index 332f293..a270f12 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
@@ -63,7 +63,7 @@ import org.apache.phoenix.schema.SaltingUtil;
 import org.apache.phoenix.schema.SortOrder;
 import org.apache.phoenix.schema.ValueSchema.Field;
 import org.apache.phoenix.schema.tuple.Tuple;
-import org.apache.phoenix.schema.types.PArrayDataType;
+import org.apache.phoenix.schema.types.PBinary;
 import org.apache.phoenix.schema.types.PChar;
 import org.apache.phoenix.schema.types.PDataType;
 import org.apache.phoenix.schema.types.PVarbinary;
@@ -322,6 +322,14 @@ public class WhereOptimizer {
         }
     }
     
+    private static int computeUnpaddedLength(byte[] b, byte padByte) {
+        int len = b.length;
+        while (len > 0 && b[len - 1] == padByte) {
+            len--;
+        }
+        return len;                                       
+    }
+    
     // Special hack for PHOENIX-2067 to change the constant array to match
     // the separators used for descending, variable length arrays.
     // Note that there'd already be a coerce expression around the constant
@@ -335,27 +343,60 @@ public class WhereOptimizer {
             PTable table = context.getCurrentTable().getTable();
             // Constants are always build with rowKeyOptimizable as true, using the correct separators
             // We only need to do this conversion if we have a table that has not yet been converted.
-            if (type != null && type.isArrayType() && column.getSortOrder() == SortOrder.DESC && !PArrayDataType.arrayBaseType(type).isFixedWidth() && !table.rowKeyOrderOptimizable()) {
-                ImmutableBytesWritable ptr = context.getTempPtr();
-                List<KeyRange> newKeyRanges = Lists.newArrayListWithExpectedSize(keyRanges.size());
-                for (KeyRange keyRange : keyRanges) {
-                    byte[] lower = keyRange.getLowerRange();
-                    if (!keyRange.lowerUnbound()) {
-                        ptr.set(lower);;
-                        type.coerceBytes(ptr, null, type, null, null, SortOrder.DESC, null, null, SortOrder.DESC, false);
-                        lower = ByteUtil.copyKeyBytesIfNecessary(ptr);
+            if (type != null && !table.rowKeyOrderOptimizable()) {
+                if (type.isArrayType() && column.getSortOrder() == SortOrder.DESC) {
+                    ImmutableBytesWritable ptr = context.getTempPtr();
+                    List<KeyRange> newKeyRanges = Lists.newArrayListWithExpectedSize(keyRanges.size());
+                    for (KeyRange keyRange : keyRanges) {
+                        byte[] lower = keyRange.getLowerRange();
+                        if (!keyRange.lowerUnbound()) {
+                            ptr.set(lower);
+                            type.coerceBytes(ptr, null, type, null, null, SortOrder.DESC, null, null, SortOrder.DESC, false);
+                            lower = ByteUtil.copyKeyBytesIfNecessary(ptr);
+                        }
+                        byte[] upper = keyRange.getUpperRange();
+                        if (!keyRange.upperUnbound()) {
+                            ptr.set(upper);
+                            type.coerceBytes(ptr, null, type, null, null, SortOrder.DESC, null, null, SortOrder.DESC, false);
+                            upper = ByteUtil.copyKeyBytesIfNecessary(ptr);
+                        }
+                        keyRange = KeyRange.getKeyRange(lower, keyRange.isLowerInclusive(), upper, keyRange.isUpperInclusive());
+                        newKeyRanges.add(keyRange);
                     }
-                    byte[] upper = keyRange.getUpperRange();
-                    if (!keyRange.upperUnbound()) {
-                        ptr.set(upper);;
-                        type.coerceBytes(ptr, null, type, null, null, SortOrder.DESC, null, null, SortOrder.DESC, false);
-                        upper = ByteUtil.copyKeyBytesIfNecessary(ptr);
+                    return newKeyRanges;
+                } else if (type == PBinary.INSTANCE || (column.getSortOrder() == SortOrder.DESC && type == PChar.INSTANCE)) {
+                    // Since the table has not been upgraded, we need to replace the correct trailing byte back to the incorrect
+                    // trailing byte so that we form the correct start/stop key
+                    List<KeyRange> newKeyRanges = Lists.newArrayListWithExpectedSize(keyRanges.size());
+                    byte byteToReplace;
+                    if (type == PBinary.INSTANCE) {
+                        byteToReplace = column.getSortOrder() == SortOrder.ASC ? QueryConstants.SEPARATOR_BYTE : QueryConstants.DESC_SEPARATOR_BYTE;
+                    } else {
+                        byteToReplace = StringUtil.INVERTED_SPACE_UTF8;
+                    }
+                    for (KeyRange keyRange : keyRanges) {
+                        byte[] lower = keyRange.getLowerRange();
+                        if (!keyRange.lowerUnbound()) {
+                            int len = computeUnpaddedLength(lower, byteToReplace);
+                            if (len != lower.length) {
+                                lower = Arrays.copyOf(lower, len);
+                                lower = StringUtil.padChar(lower, lower.length);
+                            }
+                        }
+                        byte[] upper = keyRange.getUpperRange();
+                        if (!keyRange.upperUnbound()) {
+                            int len = computeUnpaddedLength(upper, byteToReplace);
+                            if (len != upper.length) {
+                                upper = Arrays.copyOf(upper, len);
+                                upper = StringUtil.padChar(upper, upper.length);
+                            }
+                        }
+                        keyRange = KeyRange.getKeyRange(lower, keyRange.isLowerInclusive(), upper, keyRange.isUpperInclusive());
+                        newKeyRanges.add(keyRange);
                     }
-                    keyRange = KeyRange.getKeyRange(lower, keyRange.isLowerInclusive(), upper, keyRange.isUpperInclusive());
-                    newKeyRanges.add(keyRange);
+                    return newKeyRanges;
                 }
                 
-                return newKeyRanges;
             }
         }
         return keyRanges;
@@ -532,7 +573,7 @@ public class WhereOptimizer {
             List<Expression> extractNodes = extractNode == null || slot.getKeyPart().getExtractNodes().isEmpty()
                   ? Collections.<Expression>emptyList()
                   : Collections.<Expression>singletonList(extractNode);
-            return new SingleKeySlot(new BaseKeyPart(slot.getKeyPart().getColumn(), extractNodes), slot.getPKPosition(), slot.getPKSpan(), keyRanges, minMaxRange, slot.getOrderPreserving());
+            return new SingleKeySlot(new BaseKeyPart(table, slot.getKeyPart().getColumn(), extractNodes), slot.getPKPosition(), slot.getPKSpan(), keyRanges, minMaxRange, slot.getOrderPreserving());
         }
 
         private KeySlots newKeyParts(KeySlot slot, List<Expression> extractNodes, List<KeyRange> keyRanges, KeyRange minMaxRange) {
@@ -540,7 +581,7 @@ public class WhereOptimizer {
                 return EMPTY_KEY_SLOTS;
             }
             
-            return new SingleKeySlot(new BaseKeyPart(slot.getKeyPart().getColumn(), extractNodes), slot.getPKPosition(), slot.getPKSpan(), keyRanges, minMaxRange, slot.getOrderPreserving());
+            return new SingleKeySlot(new BaseKeyPart(table, slot.getKeyPart().getColumn(), extractNodes), slot.getPKPosition(), slot.getPKSpan(), keyRanges, minMaxRange, slot.getOrderPreserving());
         }
 
         private KeySlots newRowValueConstructorKeyParts(RowValueConstructorExpression rvc, List<KeySlots> childSlots) {
@@ -648,6 +689,11 @@ public class WhereOptimizer {
                 public PColumn getColumn() {
                     return childPart.getColumn();
                 }
+
+                @Override
+                public PTable getTable() {
+                    return childPart.getTable();
+                }
             }, slot.getPKPosition(), slot.getKeyRanges());
         }
 
@@ -724,7 +770,7 @@ public class WhereOptimizer {
 
             if (!minMaxExtractNodes.isEmpty()) {
                 if (keySlot[initPosition] == null) {
-                    keySlot[initPosition] = new KeySlot(new BaseKeyPart(table.getPKColumns().get(initPosition), minMaxExtractNodes), initPosition, 1, EVERYTHING_RANGES, null);
+                    keySlot[initPosition] = new KeySlot(new BaseKeyPart(table, table.getPKColumns().get(initPosition), minMaxExtractNodes), initPosition, 1, EVERYTHING_RANGES, null);
                 } else {
                     keySlot[initPosition] = keySlot[initPosition].concatExtractNodes(minMaxExtractNodes);
                 }
@@ -841,7 +887,7 @@ public class WhereOptimizer {
                 slotRanges = Collections.emptyList();
             }
             if (theSlot == null) {
-                theSlot = new KeySlot(new BaseKeyPart(table.getPKColumns().get(initialPos), slotExtractNodes), initialPos, 1, EVERYTHING_RANGES, null);
+                theSlot = new KeySlot(new BaseKeyPart(table, table.getPKColumns().get(initialPos), slotExtractNodes), initialPos, 1, EVERYTHING_RANGES, null);
             } else if (minMaxRange != KeyRange.EMPTY_RANGE && !slotExtractNodes.isEmpty()) {
                 theSlot = theSlot.concatExtractNodes(slotExtractNodes);
             }
@@ -926,7 +972,7 @@ public class WhereOptimizer {
         @Override
         public KeySlots visit(RowKeyColumnExpression node) {
             PColumn column = table.getPKColumns().get(node.getPosition());
-            return new SingleKeySlot(new BaseKeyPart(column, Collections.<Expression>singletonList(node)), node.getPosition(), 1, EVERYTHING_RANGES);
+            return new SingleKeySlot(new BaseKeyPart(table, column, Collections.<Expression>singletonList(node)), node.getPosition(), 1, EVERYTHING_RANGES);
         }
 
         @Override
@@ -997,7 +1043,8 @@ public class WhereOptimizer {
             KeySlots childSlots = childParts.get(0);
             KeySlot childSlot = childSlots.iterator().next();
             final String startsWith = node.getLiteralPrefix();
-            byte[] key = PVarchar.INSTANCE.toBytes(startsWith, node.getChildren().get(0).getSortOrder());
+            SortOrder sortOrder = node.getChildren().get(0).getSortOrder();
+            byte[] key = PVarchar.INSTANCE.toBytes(startsWith, sortOrder);
             // If the expression is an equality expression against a fixed length column
             // and the key length doesn't match the column length, the expression can
             // never be true.
@@ -1014,8 +1061,15 @@ public class WhereOptimizer {
             byte[] upperRange = ByteUtil.nextKey(key);
             Integer columnFixedLength = column.getMaxLength();
             if (type.isFixedWidth() && columnFixedLength != null) {
-                lowerRange = StringUtil.padChar(lowerRange, columnFixedLength);
-                upperRange = StringUtil.padChar(upperRange, columnFixedLength);
+                if (table.rowKeyOrderOptimizable()) {
+                    // Always use minimum byte to fill as otherwise our key is bigger
+                    // that it should be when the sort order is descending.
+                    lowerRange = type.pad(lowerRange, columnFixedLength, SortOrder.ASC);
+                    upperRange = type.pad(upperRange, columnFixedLength, SortOrder.ASC);
+                } else { // TODO: remove broken logic once tables are required to have been upgraded for PHOENIX-2067 and PHOENIX-2120
+                    lowerRange = StringUtil.padChar(lowerRange, columnFixedLength);
+                    upperRange = StringUtil.padChar(upperRange, columnFixedLength);
+                }
             }
             KeyRange keyRange = type.getKeyRange(lowerRange, true, upperRange, false);
             // Only extract LIKE expression if pattern ends with a wildcard and everything else was extracted
@@ -1117,7 +1171,7 @@ public class WhereOptimizer {
 
             public final KeySlot concatExtractNodes(List<Expression> extractNodes) {
                 return new KeySlot(
-                        new BaseKeyPart(this.getKeyPart().getColumn(),
+                        new BaseKeyPart(this.getKeyPart().getTable(), this.getKeyPart().getColumn(),
                                     SchemaUtil.concat(this.getKeyPart().getExtractNodes(),extractNodes)),
                         this.getPKPosition(),
                         this.getPKSpan(),
@@ -1135,7 +1189,7 @@ public class WhereOptimizer {
                         return null;
                     }
                     return new KeySlot(
-                            new BaseKeyPart(this.getKeyPart().getColumn(),
+                            new BaseKeyPart(this.getKeyPart().getTable(), this.getKeyPart().getColumn(),
                                         SchemaUtil.concat(this.getKeyPart().getExtractNodes(),
                                                           that.getKeyPart().getExtractNodes())),
                             this.getPKPosition(),
@@ -1181,7 +1235,7 @@ public class WhereOptimizer {
                             return null;
                         }
                         return new KeySlot(
-                                new BaseKeyPart(this.getKeyPart().getColumn(),
+                                new BaseKeyPart(this.getKeyPart().getTable(), this.getKeyPart().getColumn(),
                                             SchemaUtil.concat(this.getKeyPart().getExtractNodes(),
                                                               that.getKeyPart().getExtractNodes())),
                                 this.getPKPosition(),
@@ -1272,10 +1326,12 @@ public class WhereOptimizer {
                 return ByteUtil.getKeyRange(key, op, type);
             }
 
+            private final PTable table;
             private final PColumn column;
             private final List<Expression> nodes;
 
-            private BaseKeyPart(PColumn column, List<Expression> nodes) {
+            private BaseKeyPart(PTable table, PColumn column, List<Expression> nodes) {
+                this.table = table;
                 this.column = column;
                 this.nodes = nodes;
             }
@@ -1289,9 +1345,14 @@ public class WhereOptimizer {
             public PColumn getColumn() {
                 return column;
             }
+
+            @Override
+            public PTable getTable() {
+                return table;
+            }
         }
         
-        private  class RowValueConstructorKeyPart implements KeyPart {
+        private class RowValueConstructorKeyPart implements KeyPart {
             private final RowValueConstructorExpression rvc;
             private final PColumn column;
             private final List<Expression> nodes;
@@ -1319,7 +1380,13 @@ public class WhereOptimizer {
             public PColumn getColumn() {
                 return column;
             }
-           @Override
+
+            @Override
+            public PTable getTable() {
+                return table;
+            }
+
+            @Override
             public KeyRange getKeyRange(CompareOp op, Expression rhs) {
                // With row value constructors, we need to convert the operator for any transformation we do on individual values
                // to prevent keys from being increased to the next key as would be done for fixed width values. The next key is
@@ -1446,7 +1513,6 @@ public class WhereOptimizer {
                 byte[] key = ByteUtil.copyKeyBytesIfNecessary(ptr);
                 return ByteUtil.getKeyRange(key, op, PVarbinary.INSTANCE);
             }
-
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index 788a342..a7e3e44 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -85,7 +85,8 @@ import org.apache.phoenix.schema.SortOrder;
 import org.apache.phoenix.schema.ValueSchema.Field;
 import org.apache.phoenix.schema.stats.StatisticsCollector;
 import org.apache.phoenix.schema.tuple.MultiKeyValueTuple;
-import org.apache.phoenix.schema.types.PArrayDataType;
+import org.apache.phoenix.schema.types.PBinary;
+import org.apache.phoenix.schema.types.PChar;
 import org.apache.phoenix.schema.types.PDataType;
 import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.IndexUtil;
@@ -95,6 +96,7 @@ import org.apache.phoenix.util.MetaDataUtil;
 import org.apache.phoenix.util.ScanUtil;
 import org.apache.phoenix.util.SchemaUtil;
 import org.apache.phoenix.util.ServerUtil;
+import org.apache.phoenix.util.StringUtil;
 import org.apache.phoenix.util.TimeKeeper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -296,11 +298,30 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
                                         break;
                                     }
                                     Field field = schema.getField(i);
-                                    // Special case for re-writing DESC ARRAY, as the actual byte value needs to change in this case
-                                    if (field.getDataType().isArrayType() && field.getSortOrder() == SortOrder.DESC && !PArrayDataType.arrayBaseType(field.getDataType()).isFixedWidth()) {
-                                        field.getDataType().coerceBytes(ptr, null, field.getDataType(),
-                                                field.getMaxLength(), field.getScale(), field.getSortOrder(), 
-                                                field.getMaxLength(), field.getScale(), field.getSortOrder(), true); // force to use correct separator byte
+                                    if (field.getSortOrder() == SortOrder.DESC) {
+                                        // Special case for re-writing DESC ARRAY, as the actual byte value needs to change in this case
+                                        if (field.getDataType().isArrayType()) {
+                                            field.getDataType().coerceBytes(ptr, null, field.getDataType(),
+                                                    field.getMaxLength(), field.getScale(), field.getSortOrder(), 
+                                                    field.getMaxLength(), field.getScale(), field.getSortOrder(), true); // force to use correct separator byte
+                                        }
+                                        // Special case for re-writing DESC CHAR or DESC BINARY, to force the re-writing of trailing space characters
+                                        else if (field.getDataType() == PChar.INSTANCE || field.getDataType() == PBinary.INSTANCE) {
+                                            int len = ptr.getLength();
+                                            while (len > 0 && ptr.get()[ptr.getOffset() + len - 1] == StringUtil.SPACE_UTF8) {
+                                                len--;
+                                            }
+                                            ptr.set(ptr.get(), ptr.getOffset(), len);
+                                        }
+                                    } else if (field.getDataType() == PBinary.INSTANCE) {
+                                        // Remove trailing space characters so that the setValues call below will replace them
+                                        // with the correct zero byte character. Note this is somewhat dangerous as these
+                                        // could be legit, but I don't know what the alternative is.
+                                        int len = ptr.getLength();
+                                        while (len > 0 && ptr.get()[ptr.getOffset() + len - 1] == StringUtil.SPACE_UTF8) {
+                                            len--;
+                                        }
+                                        ptr.set(ptr.get(), ptr.getOffset(), len);                                        
                                     }
                                     values[i] = ptr.copyBytes();
                                 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/main/java/org/apache/phoenix/expression/LiteralExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/LiteralExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/LiteralExpression.java
index 26c076c..c1faf66 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/LiteralExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/LiteralExpression.java
@@ -37,7 +37,6 @@ import org.apache.phoenix.schema.types.PTimestamp;
 import org.apache.phoenix.schema.types.PVarchar;
 import org.apache.phoenix.schema.types.PhoenixArray;
 import org.apache.phoenix.util.ByteUtil;
-import org.apache.phoenix.util.StringUtil;
 
 import com.google.common.base.Preconditions;
 
@@ -174,7 +173,7 @@ public class LiteralExpression extends BaseTerminalExpression {
         byte[] b = type.toBytes(value, sortOrder);
         if (type == PVarchar.INSTANCE || type == PChar.INSTANCE) {
             if (type == PChar.INSTANCE && maxLength != null  && b.length < maxLength) {
-                b = StringUtil.padChar(b, maxLength);
+                b = type.pad(b, maxLength, sortOrder);
             } else if (value != null) {
                 maxLength = ((String)value).length();
             }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/main/java/org/apache/phoenix/expression/function/InvertFunction.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/InvertFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/InvertFunction.java
index 3dcbf7c..6a3e2a1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/InvertFunction.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/InvertFunction.java
@@ -28,9 +28,10 @@ import org.apache.phoenix.parse.FunctionParseNode.Argument;
 import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction;
 import org.apache.phoenix.query.KeyRange;
 import org.apache.phoenix.schema.PColumn;
-import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.SortOrder;
 import org.apache.phoenix.schema.tuple.Tuple;
+import org.apache.phoenix.schema.types.PDataType;
 
 @BuiltInFunction(name = InvertFunction.NAME, args = { @Argument() })
 public class InvertFunction extends ScalarFunction {
@@ -107,6 +108,11 @@ public class InvertFunction extends ScalarFunction {
             public PColumn getColumn() {
                 return childPart.getColumn();
             }
+
+            @Override
+            public PTable getTable() {
+                return childPart.getTable();
+            }
         };
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/main/java/org/apache/phoenix/expression/function/PrefixFunction.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/PrefixFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/PrefixFunction.java
index 3373df7..111c8b3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/PrefixFunction.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/PrefixFunction.java
@@ -26,9 +26,10 @@ import org.apache.phoenix.compile.KeyPart;
 import org.apache.phoenix.expression.Expression;
 import org.apache.phoenix.query.KeyRange;
 import org.apache.phoenix.schema.PColumn;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.SortOrder;
 import org.apache.phoenix.schema.types.PDataType;
 import org.apache.phoenix.util.ByteUtil;
-import org.apache.phoenix.util.StringUtil;
 
 abstract public class PrefixFunction extends ScalarFunction {
     public PrefixFunction() {
@@ -84,16 +85,22 @@ abstract public class PrefixFunction extends ScalarFunction {
                     return childPart.getKeyRange(op, rhs);
                 }
                 Integer length = getColumn().getMaxLength();
+                SortOrder sortOrder = getColumn().getSortOrder();
                 if (type.isFixedWidth() && length != null) {
                     if (lowerRange != KeyRange.UNBOUND) {
-                        lowerRange = StringUtil.padChar(lowerRange, length);
+                        lowerRange = type.pad(lowerRange, length, sortOrder);
                     }
                     if (upperRange != KeyRange.UNBOUND) {
-                        upperRange = StringUtil.padChar(upperRange, length);
+                        upperRange = type.pad(upperRange, length, sortOrder);
                     }
                 }
                 return KeyRange.getKeyRange(lowerRange, lowerInclusive, upperRange, false);
             }
+
+            @Override
+            public PTable getTable() {
+                return childPart.getTable();
+            }
         };
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RTrimFunction.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RTrimFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RTrimFunction.java
index 911ed19..aa632e2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RTrimFunction.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RTrimFunction.java
@@ -29,10 +29,11 @@ import org.apache.phoenix.parse.FunctionParseNode.Argument;
 import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction;
 import org.apache.phoenix.query.KeyRange;
 import org.apache.phoenix.schema.PColumn;
-import org.apache.phoenix.schema.types.PDataType;
-import org.apache.phoenix.schema.types.PVarchar;
+import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.SortOrder;
 import org.apache.phoenix.schema.tuple.Tuple;
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PVarchar;
 import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.StringUtil;
 
@@ -123,12 +124,13 @@ public class RTrimFunction extends ScalarFunction {
                     return childPart.getKeyRange(op, rhs);
                 }
                 Integer length = getColumn().getMaxLength();
+                SortOrder sortOrder = getColumn().getSortOrder();
                 if (type.isFixedWidth() && length != null) {
                     if (lowerRange != KeyRange.UNBOUND) {
-                        lowerRange = StringUtil.padChar(lowerRange, length);
+                        lowerRange = type.pad(lowerRange, length, sortOrder);
                     }
                     if (upperRange != KeyRange.UNBOUND) {
-                        upperRange = StringUtil.padChar(upperRange, length);
+                        upperRange = type.pad(upperRange, length, sortOrder);
                     }
                 }
                 return KeyRange.getKeyRange(lowerRange, lowerInclusive, upperRange, false);
@@ -143,6 +145,11 @@ public class RTrimFunction extends ScalarFunction {
             public PColumn getColumn() {
                 return childPart.getColumn();
             }
+
+            @Override
+            public PTable getTable() {
+                return childPart.getTable();
+            }
         };
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDateExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDateExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDateExpression.java
index d8fa1dc..e410e34 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDateExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDateExpression.java
@@ -33,13 +33,14 @@ import org.apache.phoenix.expression.Determinism;
 import org.apache.phoenix.expression.Expression;
 import org.apache.phoenix.expression.LiteralExpression;
 import org.apache.phoenix.query.KeyRange;
-import org.apache.phoenix.schema.types.PInteger;
 import org.apache.phoenix.schema.PColumn;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.tuple.Tuple;
 import org.apache.phoenix.schema.types.PDataType;
 import org.apache.phoenix.schema.types.PDataType.PDataCodec;
+import org.apache.phoenix.schema.types.PInteger;
 import org.apache.phoenix.schema.types.PVarchar;
-import org.apache.phoenix.schema.SortOrder;
-import org.apache.phoenix.schema.tuple.Tuple;
 import org.apache.phoenix.util.ByteUtil;
 
 import com.google.common.collect.Lists;
@@ -255,6 +256,11 @@ public class RoundDateExpression extends ScalarFunction {
                     return childPart.getKeyRange(op, rhs);
                 }
             }
+
+            @Override
+            public PTable getTable() {
+                return childPart.getTable();
+            }
         };
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDecimalExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDecimalExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDecimalExpression.java
index e81650f..7f44082 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDecimalExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDecimalExpression.java
@@ -23,26 +23,27 @@ import java.io.IOException;
 import java.math.BigDecimal;
 import java.math.RoundingMode;
 import java.sql.SQLException;
+import java.util.Collections;
 import java.util.List;
 
+import org.apache.hadoop.hbase.filter.CompareFilter;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.io.WritableUtils;
+import org.apache.phoenix.compile.KeyPart;
 import org.apache.phoenix.expression.Determinism;
 import org.apache.phoenix.expression.Expression;
 import org.apache.phoenix.expression.LiteralExpression;
-import org.apache.phoenix.schema.types.PDecimal;
+import org.apache.phoenix.query.KeyRange;
 import org.apache.phoenix.schema.IllegalDataException;
-import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.schema.PColumn;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.tuple.Tuple;
 import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PDecimal;
+import org.apache.phoenix.schema.types.PInteger;
 import org.apache.phoenix.schema.types.PLong;
-import org.apache.phoenix.schema.tuple.Tuple;
 
 import com.google.common.collect.Lists;
-import java.util.Collections;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.phoenix.compile.KeyPart;
-import org.apache.phoenix.query.KeyRange;
-import org.apache.phoenix.schema.PColumn;
 
 /**
  *
@@ -274,6 +275,11 @@ public class RoundDecimalExpression extends ScalarFunction {
                 // otherwise, rounding has not affected the operator, so return normally
                 return rounded;
             }
+
+            @Override
+            public PTable getTable() {
+                return childPart.getTable();
+            }
         };
     }
     

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index a17e28a..dede98d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -1983,9 +1983,13 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                                     props.remove(PhoenixRuntime.TENANT_ID_ATTRIB);
                                     PhoenixConnection conn = new PhoenixConnection(ConnectionQueryServicesImpl.this, metaConnection.getURL(), props, metaConnection.getMetaDataCache());
                                     try {
-                                        Set<String> tablesNeedingUpgrade = UpgradeUtil.getPhysicalTablesWithDescVarLengthRowKey(conn);
+                                        List<String> tablesNeedingUpgrade = UpgradeUtil.getPhysicalTablesWithDescRowKey(conn);
                                         if (!tablesNeedingUpgrade.isEmpty()) {
-                                            logger.warn("The following tables require upgrade due to a bug causing the row key to be incorrect for descending columns (PHOENIX-2067):\n" + Joiner.on(' ').join(tablesNeedingUpgrade));
+                                            logger.warn("The following tables require upgrade due to a bug causing the row key to be incorrect for descending columns and ascending BINARY columns (PHOENIX-2067 and PHOENIX-2120):\n" + Joiner.on(' ').join(tablesNeedingUpgrade) + "\nTo upgrade issue the \"bin/psql.py -u\" command.");
+                                        }
+                                        List<String> unsupportedTables = UpgradeUtil.getPhysicalTablesWithDescVarbinaryRowKey(conn);
+                                        if (!unsupportedTables.isEmpty()) {
+                                            logger.warn("The following tables use an unsupported VARBINARY DESC construct and need to be changed:\n" + Joiner.on(' ').join(unsupportedTables));
                                         }
                                     } catch (Exception ex) {
                                         logger.error("Unable to determine tables requiring upgrade due to PHOENIX-2067", ex);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index 1756c2f..521fb42 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -54,6 +54,8 @@ import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder;
 import org.apache.phoenix.schema.stats.GuidePostsInfo;
 import org.apache.phoenix.schema.stats.PTableStats;
 import org.apache.phoenix.schema.stats.PTableStatsImpl;
+import org.apache.phoenix.schema.types.PBinary;
+import org.apache.phoenix.schema.types.PChar;
 import org.apache.phoenix.schema.types.PDataType;
 import org.apache.phoenix.schema.types.PVarchar;
 import org.apache.phoenix.util.ByteUtil;
@@ -125,8 +127,8 @@ public class PTableImpl implements PTable {
     private IndexType indexType;
     private PTableStats tableStats = PTableStats.EMPTY_STATS;
     private int baseColumnCount;
-    private boolean hasDescVarLengthColumns;
-    private boolean rowKeyOrderOptimizable;
+    private boolean rowKeyOrderOptimizable; // TODO: remove when required that tables have been upgrade for PHOENIX-2067
+    private boolean hasColumnsRequiringUpgrade; // TODO: remove when required that tables have been upgrade for PHOENIX-2067
 
     public PTableImpl() {
         this.indexes = Collections.emptyList();
@@ -405,7 +407,8 @@ public class PTableImpl implements PTable {
         for (PColumn column : allColumns) {
             PName familyName = column.getFamilyName();
             if (familyName == null) {
-                hasDescVarLengthColumns |= (column.getSortOrder() == SortOrder.DESC && !column.getDataType().isFixedWidth());
+                hasColumnsRequiringUpgrade |= (column.getSortOrder() == SortOrder.DESC && (!column.getDataType().isFixedWidth() || column.getDataType() == PChar.INSTANCE || column.getDataType() == PBinary.INSTANCE))
+                        || (column.getSortOrder() == SortOrder.ASC && column.getDataType() == PBinary.INSTANCE && column.getMaxLength() != null && column.getMaxLength() > 1);
             	pkColumns.add(column);
             }
             if (familyName == null) {
@@ -547,8 +550,16 @@ public class PTableImpl implements PTable {
                     throw new ConstraintViolationException(name.getString() + "." + column.getName().getString() + " may not be null");
                 }
                 Integer	maxLength = column.getMaxLength();
-                if (maxLength != null && type.isFixedWidth() && byteValue.length <= maxLength) {
-                    byteValue = StringUtil.padChar(byteValue, maxLength);
+                if (maxLength != null && type.isFixedWidth() && byteValue.length < maxLength) {
+                    if (rowKeyOrderOptimizable()) {
+                        key.set(byteValue);
+                        type.pad(key, maxLength, sortOrder);
+                        byteValue = ByteUtil.copyKeyBytesIfNecessary(key);
+                    } else {
+                        // TODO: remove this incorrect code and move StringUtil.padChar() to TestUtil
+                        // once we require tables to have been upgraded
+                        byteValue = StringUtil.padChar(byteValue, maxLength);
+                    }
                 } else if (maxLength != null && byteValue.length > maxLength) {
                     throw new DataExceedsCapacityException(name.getString() + "." + column.getName().getString() + " may not exceed " + maxLength + " bytes (" + SchemaUtil.toString(type, byteValue) + ")");
                 }
@@ -733,7 +744,7 @@ public class PTableImpl implements PTable {
                         HConstants.EMPTY_BYTE_ARRAY : byteValue);
                 Integer	maxLength = column.getMaxLength();
             	if (!isNull && type.isFixedWidth() && maxLength != null) {
-    				if (ptr.getLength() <= maxLength) {
+    				if (ptr.getLength() < maxLength) {
                         type.pad(ptr, maxLength, column.getSortOrder());
                     } else if (ptr.getLength() > maxLength) {
                         throw new DataExceedsCapacityException(name.getString() + "." + column.getName().getString() + " may not exceed " + maxLength + " bytes (" + type.toObject(byteValue) + ")");
@@ -1131,6 +1142,6 @@ public class PTableImpl implements PTable {
 
     @Override
     public boolean rowKeyOrderOptimizable() {
-        return rowKeyOrderOptimizable || !hasDescVarLengthColumns;
+        return rowKeyOrderOptimizable || !hasColumnsRequiringUpgrade;
     }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PBinary.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PBinary.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PBinary.java
index b397554..8e0c4b5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PBinary.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PBinary.java
@@ -19,11 +19,13 @@ package org.apache.phoenix.schema.types;
 
 import java.sql.Types;
 import java.text.Format;
+import java.util.Arrays;
 
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.util.Base64;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.exception.DataExceedsCapacityException;
+import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.schema.SortOrder;
 
 public class PBinary extends PDataType<byte[]> {
@@ -35,29 +37,43 @@ public class PBinary extends PDataType<byte[]> {
   }
 
   @Override
+  public byte[] pad(byte[] b, Integer maxLength, SortOrder sortOrder) {
+      if (b == null || b.length >= maxLength) {
+          return b;
+        }
+        byte[] newBytes = new byte[maxLength];
+        System.arraycopy(b, 0, newBytes, 0, b.length);
+        if (sortOrder == SortOrder.DESC) {
+            Arrays.fill(newBytes, b.length, maxLength, QueryConstants.DESC_SEPARATOR_BYTE);
+        }
+        return newBytes;
+  }
+  
+  @Override
   public void pad(ImmutableBytesWritable ptr, Integer maxLength, SortOrder sortOrder) {
     if (ptr.getLength() >= maxLength) {
       return;
     }
     byte[] newBytes = new byte[maxLength];
     System.arraycopy(ptr.get(), ptr.getOffset(), newBytes, 0, ptr.getLength());
+    if (sortOrder == SortOrder.DESC) {
+        Arrays.fill(newBytes, ptr.getLength(), maxLength, QueryConstants.DESC_SEPARATOR_BYTE);
+    }
     ptr.set(newBytes);
   }
 
   @Override
   public Object pad(Object object, Integer maxLength) {
     byte[] b = (byte[]) object;
-    if (b == null) {
-      return new byte[maxLength];
-    }
-    if (b.length == maxLength) {
+    int length = (b == null ? 0 : b.length);
+    if (length == maxLength) {
       return object;
     }
-    if (b.length > maxLength) {
+    if (length > maxLength) {
       throw new DataExceedsCapacityException(this, maxLength, null);
     }
     byte[] newBytes = new byte[maxLength];
-    System.arraycopy(b, 0, newBytes, 0, b.length);
+    System.arraycopy(b, 0, newBytes, 0, length);
 
     return newBytes;
   }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PChar.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PChar.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PChar.java
index c7cc1c1..6b26197 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PChar.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PChar.java
@@ -53,6 +53,17 @@ public class PChar extends PDataType<String> {
     }
 
     @Override
+    public byte[] pad(byte[] b, Integer maxLength, SortOrder sortOrder) {
+      if (b == null || b.length >= maxLength) {
+        return b;
+      }
+      byte[] newBytes = new byte[maxLength];
+      System.arraycopy(b, 0, newBytes, 0, b.length);
+      Arrays.fill(newBytes, b.length, maxLength, sortOrder == SortOrder.ASC ? StringUtil.SPACE_UTF8 : StringUtil.INVERTED_SPACE_UTF8);
+      return newBytes;
+    }
+
+    @Override
     public Object pad(Object object, Integer maxLength) {
       String s = (String) object;
       if (s == null) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PDataType.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PDataType.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PDataType.java
index 2c91dc5..43bab0e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PDataType.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PDataType.java
@@ -1147,6 +1147,7 @@ public abstract class PDataType<T> implements DataType<T>, Comparable<PDataType<
     }
 
     public void pad(ImmutableBytesWritable ptr, Integer maxLength, SortOrder sortOrder) {}
+    public byte[] pad(byte[] b, Integer maxLength, SortOrder sortOrder) { return b; }
 
     public static PDataType arrayBaseType(PDataType arrayType) {
         Preconditions.checkArgument(arrayType.isArrayType(), "Not a phoenix array type");

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java
index 1a2019d..c99d47e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java
@@ -193,17 +193,28 @@ public class PhoenixRuntime {
                     .unwrap(PhoenixConnection.class);
 
             if (execCmd.isUpgrade()) {
+                if (conn.getClientInfo(PhoenixRuntime.CURRENT_SCN_ATTRIB) != null) {
+                    throw new SQLException("May not specify the CURRENT_SCN property when upgrading");
+                }
+                if (conn.getClientInfo(PhoenixRuntime.TENANT_ID_ATTRIB) != null) {
+                    throw new SQLException("May not specify the TENANT_ID_ATTRIB property when upgrading");
+                }
                 if (execCmd.getInputFiles().isEmpty()) {
-                    Set<String> tablesNeedingUpgrade = UpgradeUtil.getPhysicalTablesWithDescVarLengthRowKey(conn);
+                    List<String> tablesNeedingUpgrade = UpgradeUtil.getPhysicalTablesWithDescRowKey(conn);
                     if (tablesNeedingUpgrade.isEmpty()) {
-                        String msg = "No tables are required to be upgraded due to incorrect row key order for descending, variable length columsn (PHOENIX-2067)";
+                        String msg = "No tables are required to be upgraded due to incorrect row key order (PHOENIX-2067 and PHOENIX-2120)";
                         System.out.println(msg);
                     } else {
-                        String msg = "The following tables require upgrade due to a bug causing the row key to be incorrect for descending columns (PHOENIX-2067):\n" + Joiner.on(' ').join(tablesNeedingUpgrade);
+                        String msg = "The following tables require upgrade due to a bug causing the row key to be incorrectly ordered (PHOENIX-2067 and PHOENIX-2120):\n" + Joiner.on(' ').join(tablesNeedingUpgrade);
+                        System.out.println("WARNING: " + msg);
+                    }
+                    List<String> unsupportedTables = UpgradeUtil.getPhysicalTablesWithDescVarbinaryRowKey(conn);
+                    if (!unsupportedTables.isEmpty()) {
+                        String msg = "The following tables use an unsupported VARBINARY DESC construct and need to be changed:\n" + Joiner.on(' ').join(unsupportedTables);
                         System.out.println("WARNING: " + msg);
                     }
                 } else {
-                    UpgradeUtil.upgradeDescVarLengthRowKeys(conn, execCmd.getInputFiles());
+                    UpgradeUtil.upgradeDescVarLengthRowKeys(conn, execCmd.getInputFiles(), execCmd.isBypassUpgrade());
                 }
             } else {
                 for (String inputFile : execCmd.getInputFiles()) {
@@ -476,6 +487,7 @@ public class PhoenixRuntime {
         private boolean strict;
         private List<String> inputFiles;
         private boolean isUpgrade;
+        private boolean isBypassUpgrade;
 
         /**
          * Factory method to build up an {@code ExecutionCommand} based on supplied parameters.
@@ -502,9 +514,18 @@ public class PhoenixRuntime {
                     "Define the array element separator, defaults to ':'");
             Option upgradeOption = new Option("u", "upgrade", false, "Upgrades tables specified as arguments " +
                     "by rewriting them with the correct row key for descending columns. If no arguments are " +
-                    "specified, then tables that need to be upgraded will be displayed. " +
+                    "specified, then tables that need to be upgraded will be displayed without being upgraded. " +
+                    "Use the -b option to bypass the rewrite if you know that your data does not need to be upgrade. " +
+                    "This would only be the case if you have not relied on auto padding for BINARY and CHAR data, " +
+                    "but instead have always provided data up to the full max length of the column. See PHOENIX-2067 " +
+                    "and PHOENIX-2120 for more information. " +
                     "Note that " + QueryServices.THREAD_TIMEOUT_MS_ATTRIB + " and hbase.regionserver.lease.period " +
                     "parameters must be set very high to prevent timeouts when upgrading.");
+            Option bypassUpgradeOption = new Option("b", "bypass-upgrade", false,
+                    "Used in conjunction with the -u option to bypass the rewrite during upgrade if you know that your data does not need to be upgrade. " +
+                    "This would only be the case if you have not relied on auto padding for BINARY and CHAR data, " +
+                    "but instead have always provided data up to the full max length of the column. See PHOENIX-2067 " +
+                    "and PHOENIX-2120 for more information. ");
             Options options = new Options();
             options.addOption(tableOption);
             options.addOption(headerOption);
@@ -514,6 +535,7 @@ public class PhoenixRuntime {
             options.addOption(escapeCharacterOption);
             options.addOption(arrayValueSeparatorOption);
             options.addOption(upgradeOption);
+            options.addOption(bypassUpgradeOption);
 
             CommandLineParser parser = new PosixParser();
             CommandLine cmdLine = null;
@@ -558,6 +580,13 @@ public class PhoenixRuntime {
                 execCmd.isUpgrade = true;
             }
 
+            if (cmdLine.hasOption(bypassUpgradeOption.getOpt())) {
+                if (!execCmd.isUpgrade()) {
+                    usageError("The bypass-upgrade option may only be used in conjunction with the -u option", options);
+                }
+                execCmd.isBypassUpgrade = true;
+            }
+
 
             List<String> argList = Lists.newArrayList(cmdLine.getArgList());
             if (argList.isEmpty()) {
@@ -649,6 +678,10 @@ public class PhoenixRuntime {
         public boolean isUpgrade() {
             return isUpgrade;
         }
+
+        public boolean isBypassUpgrade() {
+            return isBypassUpgrade;
+        }
     }
     
     /**

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/main/java/org/apache/phoenix/util/StringUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/StringUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/StringUtil.java
index 89ae43b..4d3a36f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/StringUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/StringUtil.java
@@ -268,13 +268,6 @@ public class StringUtil {
         return getFirstNonBlankCharIdxFromEnd(b, offset, length, sortOrder) - offset + 1;
     }
 
-    public static byte[] padChar(byte[] value, int offset, int length, int paddedLength) {
-        byte[] key = new byte[paddedLength];
-        System.arraycopy(value,offset, key, 0, length);
-        Arrays.fill(key, length, paddedLength, SPACE_UTF8);
-        return key;
-    }
-
     public static byte[] padChar(byte[] value, Integer byteSize) {
         byte[] newValue = Arrays.copyOf(value, byteSize);
         if (newValue.length > value.length) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dcf845c2/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
index b4fcef8..2ab4f8b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
@@ -79,11 +79,14 @@ import org.apache.phoenix.schema.PTable.LinkType;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.SaltingUtil;
 import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.types.PBinary;
 import org.apache.phoenix.schema.types.PBoolean;
+import org.apache.phoenix.schema.types.PChar;
 import org.apache.phoenix.schema.types.PDataType;
 import org.apache.phoenix.schema.types.PDecimal;
 import org.apache.phoenix.schema.types.PInteger;
 import org.apache.phoenix.schema.types.PLong;
+import org.apache.phoenix.schema.types.PVarbinary;
 import org.apache.phoenix.schema.types.PVarchar;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -878,10 +881,17 @@ public class UpgradeUtil {
         return otherTables;
     }
     
-    // Return all types that are not fixed width that may need upgrading due to PHOENIX-2067
+    // Return all types that are descending and either:
+    // 1) variable length, which includes all array types (PHOENIX-2067)
+    // 2) fixed length with padding (PHOENIX-2120)
     // We exclude VARBINARY as we no longer support DESC for it.
     private static String getAffectedDataTypes() {
-        StringBuilder buf = new StringBuilder("(" + PVarchar.INSTANCE.getSqlType() + "," + PDecimal.INSTANCE.getSqlType() + ",");
+        StringBuilder buf = new StringBuilder("(" 
+                + PVarchar.INSTANCE.getSqlType() + "," +
+                + PChar.INSTANCE.getSqlType() + "," +
+                + PBinary.INSTANCE.getSqlType() + "," +
+                + PDecimal.INSTANCE.getSqlType() + ","
+                );
         for (PDataType type : PDataType.values()) {
             if (type.isArrayType()) {
                 buf.append(type.getSqlType());
@@ -891,20 +901,46 @@ public class UpgradeUtil {
         buf.setCharAt(buf.length()-1, ')');
         return buf.toString();
     }
+    
+    
     /**
-     * Identify the tables that need to be upgraded due to PHOENIX-2067
+     * Identify the tables that are DESC VARBINARY as this is no longer supported
      */
-    public static Set<String> getPhysicalTablesWithDescVarLengthRowKey(PhoenixConnection conn) throws SQLException {
-        // First query finds column rows of tables that need to be upgraded.
-        // We cannot tell if the column is from a table, view, or index however.
-        ResultSet rs = conn.createStatement().executeQuery(
-                "SELECT TENANT_ID,TABLE_SCHEM,TABLE_NAME\n" + 
+    public static List<String> getPhysicalTablesWithDescVarbinaryRowKey(PhoenixConnection conn) throws SQLException {
+        String query = "SELECT TENANT_ID,TABLE_SCHEM,TABLE_NAME\n" + 
                 "FROM SYSTEM.CATALOG cat1\n" + 
                 "WHERE COLUMN_NAME IS NOT NULL\n" + 
                 "AND COLUMN_FAMILY IS NULL\n" + 
                 "AND SORT_ORDER = " + SortOrder.DESC.getSystemValue() + "\n" + 
-                "AND DATA_TYPE IN " + getAffectedDataTypes() + "\n" +
-                "GROUP BY TENANT_ID,TABLE_SCHEM,TABLE_NAME");
+                "AND DATA_TYPE = " + PVarbinary.INSTANCE.getSqlType() + "\n" +
+                "GROUP BY TENANT_ID,TABLE_SCHEM,TABLE_NAME";
+        return getPhysicalTablesWithDescRowKey(query, conn);
+    }
+    
+    /**
+     * Identify the tables that need to be upgraded due to PHOENIX-2067 and PHOENIX-2120
+     */
+    public static List<String> getPhysicalTablesWithDescRowKey(PhoenixConnection conn) throws SQLException {
+        String query = "SELECT TENANT_ID,TABLE_SCHEM,TABLE_NAME\n" + 
+                "FROM SYSTEM.CATALOG cat1\n" + 
+                "WHERE COLUMN_NAME IS NOT NULL\n" + 
+                "AND COLUMN_FAMILY IS NULL\n" + 
+                "AND ( ( SORT_ORDER = " + SortOrder.DESC.getSystemValue() + "\n" + 
+                "        AND DATA_TYPE IN " + getAffectedDataTypes() + ")\n" +
+                "    OR ( SORT_ORDER = " + SortOrder.ASC.getSystemValue() + "\n" + 
+                "         AND DATA_TYPE = " + PBinary.INSTANCE.getSqlType() + "\n" +
+                "         AND COLUMN_SIZE > 1 ) )\n" +
+                "GROUP BY TENANT_ID,TABLE_SCHEM,TABLE_NAME";
+        return getPhysicalTablesWithDescRowKey(query, conn);
+    }
+    
+    /**
+     * Identify the tables that need to be upgraded due to PHOENIX-2067
+     */
+    private static List<String> getPhysicalTablesWithDescRowKey(String query, PhoenixConnection conn) throws SQLException {
+        // First query finds column rows of tables that need to be upgraded.
+        // We cannot tell if the column is from a table, view, or index however.
+        ResultSet rs = conn.createStatement().executeQuery(query);
         Set<String> physicalTables = Sets.newHashSetWithExpectedSize(1024);
         List<String> remainingTableNames = addPhysicalTables(conn, rs, PTableType.INDEX, physicalTables);
         if (!remainingTableNames.isEmpty()) {
@@ -931,71 +967,129 @@ public class UpgradeUtil {
                  addPhysicalTables(conn, rs, PTableType.TABLE, physicalTables);
              }
         }
-        return physicalTables;
+        List<String> sortedPhysicalTables = new ArrayList<String>(physicalTables);
+        Collections.sort(sortedPhysicalTables);
+        return sortedPhysicalTables;
     }
 
-    private static void upgradeDescVarLengthRowKeys(PhoenixConnection upgradeConn, PhoenixConnection globalConn, String schemaName, String tableName, boolean isTable) throws SQLException {
-        String escapedTableName = SchemaUtil.getEscapedTableName(schemaName, tableName);
-        String tenantInfo = "";
-        PName tenantId = PName.EMPTY_NAME;
-        if (upgradeConn.getTenantId() != null) {
-            tenantId = upgradeConn.getTenantId();
-            tenantInfo = " for tenant " + tenantId.getString();
+    private static void upgradeDescVarLengthRowKeys(PhoenixConnection upgradeConn, PhoenixConnection globalConn, String schemaName, String tableName, boolean isTable, boolean bypassUpgrade) throws SQLException {
+        String physicalName = SchemaUtil.getTableName(schemaName, tableName);
+        long currentTime = System.currentTimeMillis();
+        String snapshotName = physicalName + "_" + currentTime;
+        HBaseAdmin admin = null;
+        if (isTable && !bypassUpgrade) {
+            admin = globalConn.getQueryServices().getAdmin();
         }
-        String msg = "Starting upgrade of " + escapedTableName + tenantInfo + "...";
-        System.out.println(msg);
-        logger.info(msg);
-        ResultSet rs = upgradeConn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ count(*) FROM " + escapedTableName);
-        rs.next(); // Run query
-        List<String> tableNames = Lists.newArrayListWithExpectedSize(1024);
-        tableNames.add(tenantId == PName.EMPTY_NAME ? null : tenantId.getString());
-        tableNames.add(schemaName);
-        tableNames.add(tableName);
-        // Find views to mark as upgraded
-        if (isTable) {
-            String physicalName = SchemaUtil.getTableName(schemaName, tableName);
-            String query =
-                    "SELECT TENANT_ID,TABLE_SCHEM,TABLE_NAME\n" + 
-                    "FROM SYSTEM.CATALOG\n" + 
-                    "WHERE COLUMN_NAME IS NULL\n" + 
-                    "AND COLUMN_FAMILY = '" + physicalName + "'" +
-                    "AND LINK_TYPE = " + LinkType.PHYSICAL_TABLE.getSerializedValue();
-            rs = globalConn.createStatement().executeQuery(query);
-            while (rs.next()) {
-                tableNames.add(rs.getString(1));
-                tableNames.add(rs.getString(2));
-                tableNames.add(rs.getString(3));
+        boolean restoreSnapshot = false;
+        boolean success = false;
+        try {
+            if (isTable && !bypassUpgrade) {
+                String msg = "Taking snapshot of physical table " + physicalName + " prior to upgrade...";
+                System.out.println(msg);
+                logger.info(msg);
+                admin.disableTable(physicalName);
+                admin.snapshot(snapshotName, physicalName);
+                admin.enableTable(physicalName);
+                restoreSnapshot = true;
+            }
+            String escapedTableName = SchemaUtil.getEscapedTableName(schemaName, tableName);
+            String tenantInfo = "";
+            PName tenantId = PName.EMPTY_NAME;
+            if (upgradeConn.getTenantId() != null) {
+                tenantId = upgradeConn.getTenantId();
+                tenantInfo = " for tenant " + tenantId.getString();
+            }
+            String msg = "Starting upgrade of " + escapedTableName + tenantInfo + "...";
+            System.out.println(msg);
+            logger.info(msg);
+            ResultSet rs;
+            if (!bypassUpgrade) {
+                rs = upgradeConn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ count(*) FROM " + escapedTableName);
+                rs.next(); // Run query
+            }
+            List<String> tableNames = Lists.newArrayListWithExpectedSize(1024);
+            tableNames.add(tenantId == PName.EMPTY_NAME ? null : tenantId.getString());
+            tableNames.add(schemaName);
+            tableNames.add(tableName);
+            // Find views to mark as upgraded
+            if (isTable) {
+                String query =
+                        "SELECT TENANT_ID,TABLE_SCHEM,TABLE_NAME\n" + 
+                        "FROM SYSTEM.CATALOG\n" + 
+                        "WHERE COLUMN_NAME IS NULL\n" + 
+                        "AND COLUMN_FAMILY = '" + physicalName + "'" +
+                        "AND LINK_TYPE = " + LinkType.PHYSICAL_TABLE.getSerializedValue();
+                rs = globalConn.createStatement().executeQuery(query);
+                while (rs.next()) {
+                    tableNames.add(rs.getString(1));
+                    tableNames.add(rs.getString(2));
+                    tableNames.add(rs.getString(3));
+                }
+            }
+            // Mark the table and views as upgraded now
+            for (int i = 0; i < tableNames.size(); i += 3) {
+                String theTenantId = tableNames.get(i);
+                String theSchemaName = tableNames.get(i+1);
+                String theTableName = tableNames.get(i+2);
+                globalConn.createStatement().execute("UPSERT INTO " + PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME + 
+                    " (" + PhoenixDatabaseMetaData.TENANT_ID + "," +
+                           PhoenixDatabaseMetaData.TABLE_SCHEM + "," +
+                           PhoenixDatabaseMetaData.TABLE_NAME + "," +
+                           MetaDataEndpointImpl.ROW_KEY_ORDER_OPTIMIZABLE + " BOOLEAN"
+                   + ") VALUES (" +
+                           "'" + (theTenantId == null ? StringUtil.EMPTY_STRING : theTenantId) + "'," +
+                           "'" + (theSchemaName == null ? StringUtil.EMPTY_STRING : theSchemaName) + "'," +
+                           "'" + theTableName + "'," +
+                           "TRUE)");
+            }
+            globalConn.commit();
+            for (int i = 0; i < tableNames.size(); i += 3) {
+                String theTenantId = tableNames.get(i);
+                String theSchemaName = tableNames.get(i+1);
+                String theTableName = tableNames.get(i+2);
+                globalConn.getQueryServices().clearTableFromCache(
+                        theTenantId == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(theTenantId),
+                        theSchemaName == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(schemaName),
+                        Bytes.toBytes(theTableName), HConstants.LATEST_TIMESTAMP);
+            }
+            success = true;
+            msg = "Completed upgrade of " + escapedTableName + tenantInfo;
+            System.out.println(msg);
+            logger.info(msg);
+        } catch (Exception e) {
+            logger.error("Exception during upgrade of " + physicalName + ":", e);
+        } finally {
+            boolean restored = false;
+            try {
+                if (!success && restoreSnapshot) {
+                    admin.disableTable(physicalName);
+                    admin.restoreSnapshot(snapshotName, false);
+                    admin.enableTable(physicalName);
+                    String msg = "Restored snapshot of " + physicalName + " due to failure of upgrade";
+                    System.out.println(msg);
+                    logger.info(msg);
+                }
+                restored = true;
+            } catch (Exception e) {
+                logger.warn("Unable to restoring snapshot " + snapshotName + " after failed upgrade", e);
+            } finally {
+                try {
+                    if (restoreSnapshot && restored) {
+                        admin.deleteSnapshot(snapshotName);
+                    }
+                } catch (Exception e) {
+                    logger.warn("Unable to delete snapshot " + snapshotName + " after upgrade:", e);
+                } finally {
+                    try {
+                        if (admin != null) {
+                            admin.close();
+                        }
+                    } catch (IOException e) {
+                        logger.warn("Unable to close admin after upgrade:", e);
+                    }
+                }
             }
         }
-        // Mark the table and views as upgraded now
-        for (int i = 0; i < tableNames.size(); i += 3) {
-            String theTenantId = tableNames.get(i);
-            String theSchemaName = tableNames.get(i+1);
-            String theTableName = tableNames.get(i+2);
-            globalConn.createStatement().execute("UPSERT INTO " + PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME + 
-                " (" + PhoenixDatabaseMetaData.TENANT_ID + "," +
-                       PhoenixDatabaseMetaData.TABLE_SCHEM + "," +
-                       PhoenixDatabaseMetaData.TABLE_NAME + "," +
-                       MetaDataEndpointImpl.ROW_KEY_ORDER_OPTIMIZABLE + " BOOLEAN"
-               + ") VALUES (" +
-                       "'" + (theTenantId == null ? StringUtil.EMPTY_STRING : theTenantId) + "'," +
-                       "'" + (theSchemaName == null ? StringUtil.EMPTY_STRING : theSchemaName) + "'," +
-                       "'" + theTableName + "'," +
-                       "TRUE)");
-        }
-        globalConn.commit();
-        for (int i = 0; i < tableNames.size(); i += 3) {
-            String theTenantId = tableNames.get(i);
-            String theSchemaName = tableNames.get(i+1);
-            String theTableName = tableNames.get(i+2);
-            globalConn.getQueryServices().clearTableFromCache(
-                    theTenantId == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(theTenantId),
-                    theSchemaName == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(schemaName),
-                    Bytes.toBytes(theTableName), HConstants.LATEST_TIMESTAMP);
-        }
-        msg = "Completed upgrade of " + escapedTableName + tenantInfo;
-        System.out.println(msg);
-        logger.info(msg);
     }
     
     private static boolean isInvalidTableToUpgrade(PTable table) throws SQLException {
@@ -1007,16 +1101,10 @@ public class UpgradeUtil {
      * Upgrade tables and their indexes due to a bug causing descending row keys to have a row key that
      * prevents them from being sorted correctly (PHOENIX-2067).
      */
-    public static void upgradeDescVarLengthRowKeys(PhoenixConnection conn, List<String> tablesToUpgrade) throws SQLException {
+    public static void upgradeDescVarLengthRowKeys(PhoenixConnection conn, List<String> tablesToUpgrade, boolean bypassUpgrade) throws SQLException {
         if (tablesToUpgrade.isEmpty()) {
             return;
         }
-        if (conn.getClientInfo(PhoenixRuntime.CURRENT_SCN_ATTRIB) != null) {
-            throw new SQLException("May not specify the CURRENT_SCN property when upgrading");
-        }
-        if (conn.getClientInfo(PhoenixRuntime.TENANT_ID_ATTRIB) != null) {
-            throw new SQLException("May not specify the TENANT_ID_ATTRIB property when upgrading");
-        }
         List<PTable> tablesNeedingUpgrading = Lists.newArrayListWithExpectedSize(tablesToUpgrade.size());
         List<String> invalidTables = Lists.newArrayListWithExpectedSize(tablesToUpgrade.size());
         for (String fullTableName : tablesToUpgrade) {
@@ -1042,23 +1130,23 @@ public class UpgradeUtil {
                 boolean wasUpgraded = false;
                 if (!table.rowKeyOrderOptimizable()) {
                     wasUpgraded = true;
-                    upgradeDescVarLengthRowKeys(upgradeConn, conn, table.getSchemaName().getString(), table.getTableName().getString(), true);
+                    upgradeDescVarLengthRowKeys(upgradeConn, conn, table.getSchemaName().getString(), table.getTableName().getString(), true, bypassUpgrade);
                 }
                 
                 // Upgrade global indexes
                 for (PTable index : table.getIndexes()) {
                     if (!index.rowKeyOrderOptimizable() && index.getIndexType() != IndexType.LOCAL) {
                         wasUpgraded = true;
-                        upgradeDescVarLengthRowKeys(upgradeConn, conn, index.getSchemaName().getString(), index.getTableName().getString(), false);
+                        upgradeDescVarLengthRowKeys(upgradeConn, conn, index.getSchemaName().getString(), index.getTableName().getString(), false, bypassUpgrade);
                     }
                 }
                 
                 String sharedViewIndexName = Bytes.toString(MetaDataUtil.getViewIndexPhysicalName(table.getName().getBytes()));
                 // Upgrade view indexes
-                wasUpgraded |= upgradeSharedIndex(upgradeConn, conn, sharedViewIndexName);
+                wasUpgraded |= upgradeSharedIndex(upgradeConn, conn, sharedViewIndexName, bypassUpgrade);
                 String sharedLocalIndexName = Bytes.toString(MetaDataUtil.getLocalIndexPhysicalName(table.getName().getBytes()));
-                // Upgrade view indexes
-                wasUpgraded |= upgradeSharedIndex(upgradeConn, conn, sharedLocalIndexName);
+                // Upgrade local indexes
+                wasUpgraded |= upgradeSharedIndex(upgradeConn, conn, sharedLocalIndexName, bypassUpgrade);
                 
                 if (!wasUpgraded) {
                     System.out.println("Upgrade not required for this table or its indexes: " + table.getName().getString());
@@ -1074,7 +1162,7 @@ public class UpgradeUtil {
      * physical table.
      * @return true if any upgrades were performed and false otherwise.
      */
-    private static boolean upgradeSharedIndex(PhoenixConnection upgradeConn, PhoenixConnection globalConn, String physicalName) throws SQLException {
+    private static boolean upgradeSharedIndex(PhoenixConnection upgradeConn, PhoenixConnection globalConn, String physicalName, boolean bypassUpgrade) throws SQLException {
         String query =
                 "SELECT TENANT_ID,TABLE_SCHEM,TABLE_NAME\n" + 
                 "FROM SYSTEM.CATALOG cat1\n" + 
@@ -1105,7 +1193,7 @@ public class UpgradeUtil {
             PTable table = PhoenixRuntime.getTable(conn, fullTableName);
             String tableTenantId = table.getTenantId() == null ? null : table.getTenantId().getString();
             if (Objects.equal(lastTenantId, tableTenantId) && !table.rowKeyOrderOptimizable()) {
-                upgradeDescVarLengthRowKeys(upgradeConn, globalConn, table.getSchemaName().getString(), table.getTableName().getString(), false);
+                upgradeDescVarLengthRowKeys(upgradeConn, globalConn, table.getSchemaName().getString(), table.getTableName().getString(), false, bypassUpgrade);
                 wasUpgraded = true;
             }
         }


Mime
View raw message