phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rajeshb...@apache.org
Subject [2/2] phoenix git commit: PHOENIX-3170 Remove the futuretask from the list if StaleRegionBoundaryCacheException is thrown while initializing the scanners(Rajeshbabu)
Date Tue, 13 Sep 2016 10:59:14 GMT
PHOENIX-3170 Remove the futuretask from the list if StaleRegionBoundaryCacheException is thrown
while initializing the scanners(Rajeshbabu)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b6bac814
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b6bac814
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b6bac814

Branch: refs/heads/4.8-HBase-1.2
Commit: b6bac8149f97911b46e7dbd6bc0d5f36a28c4faf
Parents: 8786a3d 15d35e2
Author: Rajeshbabu Chintaguntla <rajeshbabu@apache.org>
Authored: Tue Sep 13 16:37:51 2016 +0530
Committer: Rajeshbabu Chintaguntla <rajeshbabu@apache.org>
Committed: Tue Sep 13 16:37:51 2016 +0530

----------------------------------------------------------------------
 KEYS                                            |  58 ++
 bin/queryserver.py                              |   4 +-
 phoenix-assembly/pom.xml                        |   2 +-
 phoenix-client/pom.xml                          |   2 +-
 phoenix-core/pom.xml                            |   7 +-
 .../phoenix/end2end/AggregateQueryIT.java       |  36 +
 .../org/apache/phoenix/end2end/DateTimeIT.java  | 758 ++++++++++++++++++-
 .../phoenix/end2end/DistinctPrefixFilterIT.java |  20 +-
 .../org/apache/phoenix/end2end/IndexToolIT.java |   3 +-
 .../phoenix/end2end/LikeExpressionIT.java       | 152 +++-
 .../apache/phoenix/end2end/RenewLeaseIT.java    |  90 +++
 .../apache/phoenix/end2end/StoreNullsIT.java    |  40 +-
 .../apache/phoenix/end2end/UnnestArrayIT.java   |  18 +-
 .../apache/phoenix/end2end/UpsertSelectIT.java  |   2 +-
 .../end2end/index/AsyncImmutableIndexIT.java    |  73 +-
 .../end2end/index/AsyncIndexAutoBuildIT.java    |  51 --
 .../end2end/index/AsyncIndexDisabledIT.java     |  78 ++
 .../end2end/index/AsyncIndexRegularBuildIT.java |  62 --
 .../end2end/index/AsyncIndexTestUtil.java       |  58 --
 .../index/ChildViewsUseParentViewIndexIT.java   | 254 +++++++
 .../apache/phoenix/end2end/index/IndexIT.java   |  39 +
 .../phoenix/end2end/index/IndexMetadataIT.java  |  40 +
 .../phoenix/end2end/index/MutableIndexIT.java   |  25 +-
 phoenix-core/src/main/antlr3/PhoenixSQL.g       |   5 +-
 .../IndexHalfStoreFileReaderGenerator.java      |   1 -
 .../org/apache/phoenix/cache/GlobalCache.java   |   4 +-
 .../cache/aggcache/SpillableGroupByCache.java   |   5 +-
 .../phoenix/compile/CreateTableCompiler.java    |   7 +-
 .../phoenix/compile/ExpressionCompiler.java     |  32 +-
 .../apache/phoenix/compile/OrderByCompiler.java |   3 +
 .../compile/PostLocalIndexDDLCompiler.java      |  15 +-
 .../coprocessor/BaseScannerRegionObserver.java  | 126 ++-
 .../coprocessor/DelegateRegionScanner.java      |   2 +-
 .../phoenix/coprocessor/GroupByCache.java       |   5 +-
 .../GroupedAggregateRegionObserver.java         |  15 +-
 .../coprocessor/HashJoinRegionScanner.java      |   5 +-
 .../coprocessor/MetaDataEndpointImpl.java       |  14 +-
 .../phoenix/coprocessor/MetaDataProtocol.java   |  34 +-
 .../coprocessor/MetaDataRegionObserver.java     | 111 +--
 .../phoenix/coprocessor/ScanRegionObserver.java |   9 +-
 .../UngroupedAggregateRegionObserver.java       |  34 +
 .../phoenix/exception/SQLExceptionCode.java     |   1 +
 .../apache/phoenix/execute/AggregatePlan.java   |   2 +-
 .../apache/phoenix/execute/BaseQueryPlan.java   |   7 +-
 .../apache/phoenix/execute/MutationState.java   | 343 +++++----
 .../expression/ByteBasedLikeExpression.java     |   4 +-
 .../expression/StringBasedLikeExpression.java   |   3 +-
 .../ByteBasedRegexpReplaceFunction.java         |   3 +-
 .../function/ByteBasedRegexpSplitFunction.java  |   3 +-
 .../function/ByteBasedRegexpSubstrFunction.java |   3 +-
 .../expression/function/CeilDateExpression.java |   3 +
 .../function/CeilTimestampExpression.java       |   3 +
 .../expression/function/DateScalarFunction.java |  54 ++
 .../expression/function/DayOfMonthFunction.java |   4 +-
 .../expression/function/HourFunction.java       |   4 +-
 .../expression/function/MinuteFunction.java     |   4 +-
 .../expression/function/MonthFunction.java      |   4 +-
 .../function/RoundDateExpression.java           |   3 +
 .../function/RoundDecimalExpression.java        |   2 +-
 .../function/RoundJodaDateExpression.java       |   3 +
 .../expression/function/SecondFunction.java     |   4 +-
 .../StringBasedRegexpReplaceFunction.java       |   3 +-
 .../StringBasedRegexpSubstrFunction.java        |   3 +-
 .../expression/function/ToDateFunction.java     |  10 +-
 .../expression/function/WeekFunction.java       |   4 +-
 .../expression/function/YearFunction.java       |   4 +-
 .../hbase/index/covered/IndexMetaData.java      |   9 +-
 .../hbase/index/covered/NonTxIndexBuilder.java  |   6 +-
 .../apache/phoenix/index/IndexMaintainer.java   |  16 +-
 .../phoenix/index/PhoenixIndexMetaData.java     |  15 +-
 .../index/PhoenixTransactionalIndexer.java      |  89 ++-
 .../phoenix/iterate/BaseResultIterators.java    |   8 +-
 .../iterate/MergeSortResultIterator.java        |   2 +-
 .../apache/phoenix/jdbc/PhoenixConnection.java  |  55 +-
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |   2 +-
 .../org/apache/phoenix/jdbc/PhoenixDriver.java  |  10 +-
 .../phoenix/jdbc/PhoenixEmbeddedDriver.java     | 104 ++-
 .../phoenix/jdbc/PhoenixPreparedStatement.java  |  26 +-
 .../phoenix/mapreduce/PhoenixInputFormat.java   |   2 -
 .../phoenix/mapreduce/index/IndexToolUtil.java  |   4 -
 .../index/automation/PhoenixMRJobSubmitter.java |  22 +-
 .../apache/phoenix/optimize/QueryOptimizer.java |  21 +-
 .../org/apache/phoenix/parse/SQLParser.java     |   2 +-
 .../query/ConnectionQueryServicesImpl.java      | 368 ++++++---
 .../query/ConnectionlessQueryServicesImpl.java  |  51 +-
 .../query/DelegateConnectionQueryServices.java  |  38 +-
 .../apache/phoenix/query/MetaDataMutated.java   |  19 +-
 .../apache/phoenix/query/QueryConstants.java    |  23 +-
 .../org/apache/phoenix/query/QueryServices.java |  12 +-
 .../phoenix/query/QueryServicesOptions.java     |  21 +-
 .../apache/phoenix/schema/MetaDataClient.java   | 197 ++---
 .../org/apache/phoenix/schema/PMetaData.java    |   4 +-
 .../apache/phoenix/schema/PMetaDataCache.java   | 221 ++++++
 .../apache/phoenix/schema/PMetaDataImpl.java    | 343 ++-------
 .../phoenix/schema/PSynchronizedMetaData.java   | 249 ++++++
 .../java/org/apache/phoenix/schema/PTable.java  |  20 +-
 .../org/apache/phoenix/schema/PTableImpl.java   | 108 ++-
 .../org/apache/phoenix/schema/PTableRef.java    |  56 +-
 .../apache/phoenix/schema/PTableRefFactory.java |  52 ++
 .../apache/phoenix/schema/PTableRefImpl.java    |  39 +
 .../phoenix/schema/SerializedPTableRef.java     |  47 ++
 .../schema/SerializedPTableRefFactory.java      |  37 +
 .../org/apache/phoenix/schema/SortOrder.java    |  11 +
 .../phoenix/schema/stats/StatisticsScanner.java |  72 +-
 .../apache/phoenix/schema/types/PDataType.java  |  19 +-
 .../org/apache/phoenix/schema/types/PDate.java  | 283 +++----
 .../apache/phoenix/schema/types/PDecimal.java   |   6 +-
 .../org/apache/phoenix/schema/types/PLong.java  | 519 ++++++-------
 .../org/apache/phoenix/schema/types/PTime.java  |   2 +-
 .../apache/phoenix/schema/types/PTimestamp.java |  44 +-
 .../phoenix/schema/types/PUnsignedDate.java     | 259 ++++---
 .../phoenix/schema/types/PUnsignedLong.java     | 317 ++++----
 .../phoenix/schema/types/PUnsignedTime.java     |   2 +-
 .../schema/types/PUnsignedTimestamp.java        | 152 +---
 .../java/org/apache/phoenix/util/DateUtil.java  |  23 +-
 .../org/apache/phoenix/util/ExpressionUtil.java |  15 +-
 .../apache/phoenix/util/InstanceResolver.java   |   7 +
 .../java/org/apache/phoenix/util/ScanUtil.java  |   5 +-
 .../apache/phoenix/util/TransactionUtil.java    |  11 +-
 .../org/apache/phoenix/util/UpgradeUtil.java    |  12 +-
 .../phoenix/util/csv/CsvUpsertExecutor.java     |   8 +-
 .../compile/CreateTableCompilerTest.java        |  47 ++
 .../phoenix/expression/LikeExpressionTest.java  |  14 +
 .../RoundFloorCeilExpressionsTest.java          |  14 +
 .../phoenix/jdbc/SecureUserConnectionsTest.java | 382 ++++++++++
 .../apache/phoenix/parse/QueryParserTest.java   |  10 +
 .../phoenix/query/QueryServicesTestImpl.java    |   4 +-
 .../phoenix/schema/PMetaDataImplTest.java       | 102 +--
 .../schema/stats/StatisticsScannerTest.java     | 144 ++++
 .../phoenix/schema/types/PDataTypeTest.java     |  18 +
 .../src/test/resources/log4j.properties         |   2 +
 phoenix-flume/pom.xml                           |   2 +-
 phoenix-hive/pom.xml                            |   2 +-
 phoenix-pherf/pom.xml                           |   2 +-
 phoenix-pig/pom.xml                             |   2 +-
 phoenix-queryserver-client/pom.xml              |   9 +-
 phoenix-queryserver/pom.xml                     |   9 +-
 .../apache/phoenix/queryserver/server/Main.java |  63 +-
 .../server/PhoenixDoAsCallbackTest.java         |  89 +++
 phoenix-server/pom.xml                          |   2 +-
 phoenix-spark/pom.xml                           |   2 +-
 phoenix-spark/src/it/resources/setup.sql        |   6 +
 .../apache/phoenix/spark/PhoenixSparkIT.scala   |  49 ++
 .../apache/phoenix/spark/PhoenixRelation.scala  |  26 +-
 phoenix-tracing-webapp/pom.xml                  |   2 +-
 pom.xml                                         |  20 +-
 146 files changed, 5556 insertions(+), 2311 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/b6bac814/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index eda59d1,a7c6fde..c3fa40f
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@@ -187,45 -187,39 +187,72 @@@ public class UngroupedAggregateRegionOb
                m.setAttribute(PhoenixIndexCodec.INDEX_UUID, indexUUID);
            }
        }
+       
        Mutation[] mutationArray = new Mutation[mutations.size()];
+       // When memstore size reaches blockingMemstoreSize we are waiting 3 seconds for the
+       // flush happen which decrease the memstore size and then writes allowed on the region.
+       for (int i = 0; region.getMemstoreSize() > blockingMemstoreSize && i <
30; i++) {
+           try {
+               checkForRegionClosing();
+               Thread.sleep(100);
+           } catch (InterruptedException e) {
+               Thread.currentThread().interrupt();
+               throw new IOException(e);
+           }
+       }
        // TODO: should we use the one that is all or none?
        logger.debug("Committing bactch of " + mutations.size() + " mutations for " + region.getRegionInfo().getTable().getNameAsString());
 -      region.batchMutate(mutations.toArray(mutationArray), HConstants.NO_NONCE, HConstants.NO_NONCE);
 +      try {
 +          region.batchMutate(mutations.toArray(mutationArray), HConstants.NO_NONCE, HConstants.NO_NONCE);
 +      } catch (RegionTooBusyException rtbe) {
 +            // When memstore size reaches blockingMemstoreSize we are waiting 3 seconds
for the
 +            // flush happen which decrease the memstore size and then writes allowed on
the region.
 +            for (int i = 0; region.getMemstoreSize() > blockingMemstoreSize &&
i < 30; i++) {
 +                try {
 +                    checkForRegionClosing();
 +                    Thread.sleep(100);
 +                } catch (InterruptedException e) {
 +                    Thread.currentThread().interrupt();
 +                    throw new IOException(e);
 +                }
 +            }
 +            if (region.getMemstoreSize() > blockingMemstoreSize) {
 +                throw rtbe;
 +            }
 +            region.batchMutate(mutationArray, HConstants.NO_NONCE, HConstants.NO_NONCE);
 +      }
 +    }
 +
 +    /**
 +     * There is a chance that region might be closing while running balancer/move/merge.
In this
 +     * case if the memstore size reaches blockingMemstoreSize better to fail query because
there is
 +     * a high chance that flush might not proceed and memstore won't be freed up.
 +     * @throws IOException
 +     */
 +    private void checkForRegionClosing() throws IOException {
 +        synchronized (lock) {
 +            if(isRegionClosing) {
 +                lock.notifyAll();
 +                throw new IOException("Region is getting closed. Not allowing to write to
avoid possible deadlock.");
 +            }
 +        }
      }
  
+     /**
+      * There is a chance that region might be closing while running balancer/move/merge.
In this
+      * case if the memstore size reaches blockingMemstoreSize better to fail query because
there is
+      * a high chance that flush might not proceed and memstore won't be freed up.
+      * @throws IOException
+      */
+     private void checkForRegionClosing() throws IOException {
+         synchronized (lock) {
+             if(isRegionClosing) {
+                 lock.notifyAll();
+                 throw new IOException("Region is getting closed. Not allowing to write to
avoid possible deadlock.");
+             }
+         }
+     }
+ 
      public static void serializeIntoScan(Scan scan) {
          scan.setAttribute(BaseScannerRegionObserver.UNGROUPED_AGG, QueryConstants.TRUE);
      }
@@@ -354,8 -348,10 +381,15 @@@
          }
  
          /**
++<<<<<<< HEAD
 +         * Upper bound of memstore size allowed for region. Updates will be blocked until
the flush
 +         * happen if the memstore reaches this threshold.
++=======
+          * Slow down the writes if the memstore size more than
+          * (hbase.hregion.memstore.block.multiplier - 1) times hbase.hregion.memstore.flush.size
+          * bytes. This avoids flush storm to hdfs for cases like index building where reads
and
+          * write happen to all the table regions in the server.
++>>>>>>> 15d35e25b554fb35cc0b0f676f34176dbe5d384c
           */
          final long blockingMemStoreSize = flushSize * (
                  conf.getLong(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER,

http://git-wip-us.apache.org/repos/asf/phoenix/blob/b6bac814/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index d0ade72,8b9adfd..2685b93
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@@ -31,6 -31,6 +31,7 @@@ import java.io.EOFException
  import java.sql.SQLException;
  import java.util.ArrayList;
  import java.util.Collections;
++import java.util.Iterator;
  import java.util.List;
  import java.util.Map;
  import java.util.Map.Entry;
@@@ -751,7 -751,7 +752,9 @@@ public abstract class BaseResultIterato
              boolean clearedCache = false;
              for (List<Pair<Scan,Future<PeekingResultIterator>>> future
: reverseIfNecessary(futures,isReverse)) {
                  List<PeekingResultIterator> concatIterators = Lists.newArrayListWithExpectedSize(future.size());
--                for (Pair<Scan,Future<PeekingResultIterator>> scanPair : reverseIfNecessary(future,isReverse))
{
++                Iterator<Pair<Scan, Future<PeekingResultIterator>>> scanPairItr
= reverseIfNecessary(future,isReverse).iterator();
++                while (scanPairItr.hasNext()) {
++                    Pair<Scan,Future<PeekingResultIterator>> scanPair = scanPairItr.next();
                      try {
                          long timeOutForScan = maxQueryEndTime - System.currentTimeMillis();
                          if (timeOutForScan < 0) {
@@@ -773,6 -773,6 +776,7 @@@
                          try { // Rethrow as SQLException
                              throw ServerUtil.parseServerException(e);
                          } catch (StaleRegionBoundaryCacheException e2) {
++                            scanPairItr.remove();
                              // Catch only to try to recover from region boundary cache being
out of date
                              if (!clearedCache) { // Clear cache once so that we rejigger
job based on new boundaries
                                  services.clearTableRegionCache(physicalTableName);


Mime
View raw message