Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E3EFE1859E for ; Fri, 18 Mar 2016 19:02:08 +0000 (UTC) Received: (qmail 9519 invoked by uid 500); 18 Mar 2016 19:02:08 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 9381 invoked by uid 500); 18 Mar 2016 19:02:08 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 9162 invoked by uid 99); 18 Mar 2016 19:02:07 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 18 Mar 2016 19:02:07 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A83BEE00FF; Fri, 18 Mar 2016 19:02:07 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: misty@apache.org To: commits@hbase.apache.org Date: Fri, 18 Mar 2016 19:02:13 -0000 Message-Id: <0965bdaba5ec42c8b63b725b7ca4e6ea@git.apache.org> In-Reply-To: <439e24c72fb548fe8f0d041a45e29cbe@git.apache.org> References: <439e24c72fb548fe8f0d041a45e29cbe@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [07/23] hbase-site git commit: Published site at 3a6d683d63089c1986e68e531939df7328e58300. http://git-wip-us.apache.org/repos/asf/hbase-site/blob/07fa263b/testdevapidocs/src-html/org/apache/hadoop/hbase/TestPartialResultsFromClientSide.html ---------------------------------------------------------------------- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/TestPartialResultsFromClientSide.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/TestPartialResultsFromClientSide.html index 9166af1..d1d6220 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/TestPartialResultsFromClientSide.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/TestPartialResultsFromClientSide.html @@ -25,819 +25,1030 @@ 017 */ 018package org.apache.hadoop.hbase; 019 -020import static org.junit.Assert.assertEquals; -021import static org.junit.Assert.assertFalse; -022import static org.junit.Assert.assertTrue; -023import static org.junit.Assert.fail; -024 -025import java.io.IOException; -026import java.util.ArrayList; -027import java.util.LinkedHashSet; -028import java.util.List; -029import java.util.Set; -030 -031import org.apache.commons.logging.Log; -032import org.apache.commons.logging.LogFactory; -033import org.apache.hadoop.hbase.client.ClientScanner; -034import org.apache.hadoop.hbase.client.Delete; -035import org.apache.hadoop.hbase.client.Put; -036import org.apache.hadoop.hbase.client.Result; -037import org.apache.hadoop.hbase.client.ResultScanner; -038import org.apache.hadoop.hbase.client.Scan; -039import org.apache.hadoop.hbase.client.Table; -040import org.apache.hadoop.hbase.filter.ColumnPrefixFilter; -041import org.apache.hadoop.hbase.filter.ColumnRangeFilter; -042import org.apache.hadoop.hbase.filter.Filter; -043import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; -044import org.apache.hadoop.hbase.filter.FirstKeyValueMatchingQualifiersFilter; -045import org.apache.hadoop.hbase.filter.RandomRowFilter; -046import org.apache.hadoop.hbase.testclassification.MediumTests; -047import org.apache.hadoop.hbase.util.Bytes; -048import org.junit.AfterClass; -049import org.junit.BeforeClass; -050import org.junit.Test; -051import org.junit.experimental.categories.Category; -052 -053/** -054 * These tests are focused on testing how partial results appear to a client. Partial results are -055 * {@link Result}s that contain only a portion of a row's complete list of cells. Partial results -056 * are formed when the server breaches its maximum result size when trying to service a client's RPC -057 * request. It is the responsibility of the scanner on the client side to recognize when partial -058 * results have been returned and to take action to form the complete results. -059 * <p> -060 * Unless the flag {@link Scan#setAllowPartialResults(boolean)} has been set to true, the caller of -061 * {@link ResultScanner#next()} should never see partial results. -062 */ -063@Category(MediumTests.class) -064public class TestPartialResultsFromClientSide { -065 private static final Log LOG = LogFactory.getLog(TestPartialResultsFromClientSide.class); -066 -067 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); +020import static org.junit.Assert.assertArrayEquals; +021import static org.junit.Assert.assertEquals; +022import static org.junit.Assert.assertFalse; +023import static org.junit.Assert.assertTrue; +024import static org.junit.Assert.fail; +025 +026import java.io.IOException; +027import java.util.ArrayList; +028import java.util.LinkedHashSet; +029import java.util.List; +030import java.util.Set; +031 +032import org.apache.commons.logging.Log; +033import org.apache.commons.logging.LogFactory; +034import org.apache.hadoop.hbase.client.ClientScanner; +035import org.apache.hadoop.hbase.client.Delete; +036import org.apache.hadoop.hbase.client.Put; +037import org.apache.hadoop.hbase.client.Result; +038import org.apache.hadoop.hbase.client.ResultScanner; +039import org.apache.hadoop.hbase.client.Scan; +040import org.apache.hadoop.hbase.client.Table; +041import org.apache.hadoop.hbase.filter.ColumnPrefixFilter; +042import org.apache.hadoop.hbase.filter.ColumnRangeFilter; +043import org.apache.hadoop.hbase.filter.Filter; +044import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +045import org.apache.hadoop.hbase.filter.FirstKeyValueMatchingQualifiersFilter; +046import org.apache.hadoop.hbase.filter.RandomRowFilter; +047import org.apache.hadoop.hbase.testclassification.MediumTests; +048import org.apache.hadoop.hbase.util.Bytes; +049import org.apache.hadoop.hbase.util.Pair; +050import org.junit.AfterClass; +051import org.junit.BeforeClass; +052import org.junit.Test; +053import org.junit.experimental.categories.Category; +054 +055/** +056 * These tests are focused on testing how partial results appear to a client. Partial results are +057 * {@link Result}s that contain only a portion of a row's complete list of cells. Partial results +058 * are formed when the server breaches its maximum result size when trying to service a client's RPC +059 * request. It is the responsibility of the scanner on the client side to recognize when partial +060 * results have been returned and to take action to form the complete results. +061 * <p> +062 * Unless the flag {@link Scan#setAllowPartialResults(boolean)} has been set to true, the caller of +063 * {@link ResultScanner#next()} should never see partial results. +064 */ +065@Category(MediumTests.class) +066public class TestPartialResultsFromClientSide { +067 private static final Log LOG = LogFactory.getLog(TestPartialResultsFromClientSide.class); 068 -069 private static Table TABLE = null; -070 -071 /** -072 * Table configuration -073 */ -074 private static TableName TABLE_NAME = TableName.valueOf("testTable"); -075 -076 private static int NUM_ROWS = 5; -077 private static byte[] ROW = Bytes.toBytes("testRow"); -078 private static byte[][] ROWS = HTestConst.makeNAscii(ROW, NUM_ROWS); -079 -080 // Should keep this value below 10 to keep generation of expected kv's simple. If above 10 then -081 // table/row/cf1/... will be followed by table/row/cf10/... instead of table/row/cf2/... which -082 // breaks the simple generation of expected kv's -083 private static int NUM_FAMILIES = 10; -084 private static byte[] FAMILY = Bytes.toBytes("testFamily"); -085 private static byte[][] FAMILIES = HTestConst.makeNAscii(FAMILY, NUM_FAMILIES); -086 -087 private static int NUM_QUALIFIERS = 10; -088 private static byte[] QUALIFIER = Bytes.toBytes("testQualifier"); -089 private static byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, NUM_QUALIFIERS); -090 -091 private static int VALUE_SIZE = 1024; -092 private static byte[] VALUE = Bytes.createMaxByteArray(VALUE_SIZE); -093 -094 private static int NUM_COLS = NUM_FAMILIES * NUM_QUALIFIERS; +069 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); +070 private final static int MINICLUSTER_SIZE = 5; +071 private static Table TABLE = null; +072 +073 /** +074 * Table configuration +075 */ +076 private static TableName TABLE_NAME = TableName.valueOf("testTable"); +077 +078 private static int NUM_ROWS = 5; +079 private static byte[] ROW = Bytes.toBytes("testRow"); +080 private static byte[][] ROWS = HTestConst.makeNAscii(ROW, NUM_ROWS); +081 +082 // Should keep this value below 10 to keep generation of expected kv's simple. If above 10 then +083 // table/row/cf1/... will be followed by table/row/cf10/... instead of table/row/cf2/... which +084 // breaks the simple generation of expected kv's +085 private static int NUM_FAMILIES = 10; +086 private static byte[] FAMILY = Bytes.toBytes("testFamily"); +087 private static byte[][] FAMILIES = HTestConst.makeNAscii(FAMILY, NUM_FAMILIES); +088 +089 private static int NUM_QUALIFIERS = 10; +090 private static byte[] QUALIFIER = Bytes.toBytes("testQualifier"); +091 private static byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, NUM_QUALIFIERS); +092 +093 private static int VALUE_SIZE = 1024; +094 private static byte[] VALUE = Bytes.createMaxByteArray(VALUE_SIZE); 095 -096 // Approximation of how large the heap size of cells in our table. Should be accessed through -097 // getCellHeapSize(). -098 private static long CELL_HEAP_SIZE = -1; -099 -100 @BeforeClass -101 public static void setUpBeforeClass() throws Exception { -102 TEST_UTIL.startMiniCluster(3); -103 TABLE = createTestTable(TABLE_NAME, ROWS, FAMILIES, QUALIFIERS, VALUE); -104 } -105 -106 static Table createTestTable(TableName name, byte[][] rows, byte[][] families, -107 byte[][] qualifiers, byte[] cellValue) throws IOException { -108 Table ht = TEST_UTIL.createTable(name, families); -109 List<Put> puts = createPuts(rows, families, qualifiers, cellValue); -110 ht.put(puts); -111 -112 return ht; -113 } +096 private static int NUM_COLS = NUM_FAMILIES * NUM_QUALIFIERS; +097 +098 // Approximation of how large the heap size of cells in our table. Should be accessed through +099 // getCellHeapSize(). +100 private static long CELL_HEAP_SIZE = -1; +101 +102 @BeforeClass +103 public static void setUpBeforeClass() throws Exception { +104 TEST_UTIL.startMiniCluster(MINICLUSTER_SIZE); +105 TEST_UTIL.getAdmin().setBalancerRunning(false, true); +106 TABLE = createTestTable(TABLE_NAME, ROWS, FAMILIES, QUALIFIERS, VALUE); +107 } +108 +109 static Table createTestTable(TableName name, byte[][] rows, byte[][] families, +110 byte[][] qualifiers, byte[] cellValue) throws IOException { +111 Table ht = TEST_UTIL.createTable(name, families); +112 List<Put> puts = createPuts(rows, families, qualifiers, cellValue); +113 ht.put(puts); 114 -115 @AfterClass -116 public static void tearDownAfterClass() throws Exception { -117 TEST_UTIL.shutdownMiniCluster(); -118 } -119 -120 /** -121 * Ensure that the expected key values appear in a result returned from a scanner that is -122 * combining partial results into complete results -123 * @throws Exception -124 */ -125 @Test -126 public void testExpectedValuesOfPartialResults() throws Exception { -127 testExpectedValuesOfPartialResults(false); -128 testExpectedValuesOfPartialResults(true); -129 } -130 -131 public void testExpectedValuesOfPartialResults(boolean reversed) throws Exception { -132 Scan partialScan = new Scan(); -133 partialScan.setMaxVersions(); -134 // Max result size of 1 ensures that each RPC request will return a single cell. The scanner -135 // will need to reconstruct the results into a complete result before returning to the caller -136 partialScan.setMaxResultSize(1); -137 partialScan.setReversed(reversed); -138 ResultScanner partialScanner = TABLE.getScanner(partialScan); -139 -140 final int startRow = reversed ? ROWS.length - 1 : 0; -141 final int endRow = reversed ? -1 : ROWS.length; -142 final int loopDelta = reversed ? -1 : 1; -143 String message; -144 -145 for (int row = startRow; row != endRow; row = row + loopDelta) { -146 message = "Ensuring the expected keyValues are present for row " + row; -147 List<Cell> expectedKeyValues = createKeyValuesForRow(ROWS[row], FAMILIES, QUALIFIERS, VALUE); -148 Result result = partialScanner.next(); -149 assertFalse(result.isPartial()); -150 verifyResult(result, expectedKeyValues, message); -151 } -152 -153 partialScanner.close(); -154 } +115 return ht; +116 } +117 +118 @AfterClass +119 public static void tearDownAfterClass() throws Exception { +120 TEST_UTIL.shutdownMiniCluster(); +121 } +122 +123 /** +124 * Ensure that the expected key values appear in a result returned from a scanner that is +125 * combining partial results into complete results +126 * @throws Exception +127 */ +128 @Test +129 public void testExpectedValuesOfPartialResults() throws Exception { +130 testExpectedValuesOfPartialResults(false); +131 testExpectedValuesOfPartialResults(true); +132 } +133 +134 public void testExpectedValuesOfPartialResults(boolean reversed) throws Exception { +135 Scan partialScan = new Scan(); +136 partialScan.setMaxVersions(); +137 // Max result size of 1 ensures that each RPC request will return a single cell. The scanner +138 // will need to reconstruct the results into a complete result before returning to the caller +139 partialScan.setMaxResultSize(1); +140 partialScan.setReversed(reversed); +141 ResultScanner partialScanner = TABLE.getScanner(partialScan); +142 +143 final int startRow = reversed ? ROWS.length - 1 : 0; +144 final int endRow = reversed ? -1 : ROWS.length; +145 final int loopDelta = reversed ? -1 : 1; +146 String message; +147 +148 for (int row = startRow; row != endRow; row = row + loopDelta) { +149 message = "Ensuring the expected keyValues are present for row " + row; +150 List<Cell> expectedKeyValues = createKeyValuesForRow(ROWS[row], FAMILIES, QUALIFIERS, VALUE); +151 Result result = partialScanner.next(); +152 assertFalse(result.isPartial()); +153 verifyResult(result, expectedKeyValues, message); +154 } 155 -156 /** -157 * Ensure that we only see Results marked as partial when the allowPartial flag is set -158 * @throws Exception -159 */ -160 @Test -161 public void testAllowPartialResults() throws Exception { -162 Scan scan = new Scan(); -163 scan.setAllowPartialResults(true); -164 scan.setMaxResultSize(1); -165 ResultScanner scanner = TABLE.getScanner(scan); -166 Result result = scanner.next(); -167 -168 assertTrue(result != null); -169 assertTrue(result.isPartial()); -170 assertTrue(result.rawCells() != null); -171 assertTrue(result.rawCells().length == 1); -172 -173 scanner.close(); -174 -175 scan.setAllowPartialResults(false); -176 scanner = TABLE.getScanner(scan); -177 result = scanner.next(); -178 -179 assertTrue(result != null); -180 assertTrue(!result.isPartial()); -181 assertTrue(result.rawCells() != null); -182 assertTrue(result.rawCells().length == NUM_COLS); -183 -184 scanner.close(); -185 } +156 partialScanner.close(); +157 } +158 +159 /** +160 * Ensure that we only see Results marked as partial when the allowPartial flag is set +161 * @throws Exception +162 */ +163 @Test +164 public void testAllowPartialResults() throws Exception { +165 Scan scan = new Scan(); +166 scan.setAllowPartialResults(true); +167 scan.setMaxResultSize(1); +168 ResultScanner scanner = TABLE.getScanner(scan); +169 Result result = scanner.next(); +170 +171 assertTrue(result != null); +172 assertTrue(result.isPartial()); +173 assertTrue(result.rawCells() != null); +174 assertTrue(result.rawCells().length == 1); +175 +176 scanner.close(); +177 +178 scan.setAllowPartialResults(false); +179 scanner = TABLE.getScanner(scan); +180 result = scanner.next(); +181 +182 assertTrue(result != null); +183 assertTrue(!result.isPartial()); +184 assertTrue(result.rawCells() != null); +185 assertTrue(result.rawCells().length == NUM_COLS); 186 -187 /** -188 * Ensure that the results returned from a scanner that retrieves all results in a single RPC call -189 * matches the results that are returned from a scanner that must incrementally combine partial -190 * results into complete results. A variety of scan configurations can be tested -191 * @throws Exception -192 */ -193 @Test -194 public void testEquivalenceOfScanResults() throws Exception { -195 Scan oneShotScan = new Scan(); -196 oneShotScan.setMaxResultSize(Long.MAX_VALUE); -197 -198 Scan partialScan = new Scan(oneShotScan); -199 partialScan.setMaxResultSize(1); +187 scanner.close(); +188 } +189 +190 /** +191 * Ensure that the results returned from a scanner that retrieves all results in a single RPC call +192 * matches the results that are returned from a scanner that must incrementally combine partial +193 * results into complete results. A variety of scan configurations can be tested +194 * @throws Exception +195 */ +196 @Test +197 public void testEquivalenceOfScanResults() throws Exception { +198 Scan oneShotScan = new Scan(); +199 oneShotScan.setMaxResultSize(Long.MAX_VALUE); 200 -201 testEquivalenceOfScanResults(TABLE, oneShotScan, partialScan); -202 } +201 Scan partialScan = new Scan(oneShotScan); +202 partialScan.setMaxResultSize(1); 203 -204 public void testEquivalenceOfScanResults(Table table, Scan scan1, Scan scan2) throws Exception { -205 ResultScanner scanner1 = table.getScanner(scan1); -206 ResultScanner scanner2 = table.getScanner(scan2); -207 -208 Result r1 = null; -209 Result r2 = null; -210 int count = 0; -211 -212 while ((r1 = scanner1.next()) != null) { -213 r2 = scanner2.next(); +204 testEquivalenceOfScanResults(TABLE, oneShotScan, partialScan); +205 } +206 +207 public void testEquivalenceOfScanResults(Table table, Scan scan1, Scan scan2) throws Exception { +208 ResultScanner scanner1 = table.getScanner(scan1); +209 ResultScanner scanner2 = table.getScanner(scan2); +210 +211 Result r1 = null; +212 Result r2 = null; +213 int count = 0; 214 -215 assertTrue(r2 != null); -216 compareResults(r1, r2, "Comparing result #" + count); -217 count++; -218 } -219 -220 r2 = scanner2.next(); -221 assertTrue("r2: " + r2 + " Should be null", r2 == null); +215 while ((r1 = scanner1.next()) != null) { +216 r2 = scanner2.next(); +217 +218 assertTrue(r2 != null); +219 compareResults(r1, r2, "Comparing result #" + count); +220 count++; +221 } 222 -223 scanner1.close(); -224 scanner2.close(); -225 } -226 -227 /** -228 * Order of cells in partial results matches the ordering of cells from complete results -229 * @throws Exception -230 */ -231 @Test -232 public void testOrderingOfCellsInPartialResults() throws Exception { -233 Scan scan = new Scan(); -234 -235 for (int col = 1; col <= NUM_COLS; col++) { -236 scan.setMaxResultSize(getResultSizeForNumberOfCells(col)); -237 testOrderingOfCellsInPartialResults(scan); -238 -239 // Test again with a reversed scanner -240 scan.setReversed(true); -241 testOrderingOfCellsInPartialResults(scan); -242 } -243 } -244 -245 public void testOrderingOfCellsInPartialResults(final Scan basePartialScan) throws Exception { -246 // Scan that retrieves results in pieces (partials). By setting allowPartialResults to be true -247 // the results will NOT be reconstructed and instead the caller will see the partial results -248 // returned by the server -249 Scan partialScan = new Scan(basePartialScan); -250 partialScan.setAllowPartialResults(true); -251 ResultScanner partialScanner = TABLE.getScanner(partialScan); -252 -253 // Scan that retrieves all table results in single RPC request -254 Scan oneShotScan = new Scan(basePartialScan); -255 oneShotScan.setMaxResultSize(Long.MAX_VALUE); -256 oneShotScan.setCaching(ROWS.length); -257 ResultScanner oneShotScanner = TABLE.getScanner(oneShotScan); -258 -259 Result oneShotResult = oneShotScanner.next(); -260 Result partialResult = null; -261 int iterationCount = 0; -262 -263 while (oneShotResult != null && oneShotResult.rawCells() != null) { -264 List<Cell> aggregatePartialCells = new ArrayList<Cell>(); -265 do { -266 partialResult = partialScanner.next(); -267 assertTrue("Partial Result is null. iteration: " + iterationCount, partialResult != null); -268 assertTrue("Partial cells are null. iteration: " + iterationCount, -269 partialResult.rawCells() != null); -270 -271 for (Cell c : partialResult.rawCells()) { -272 aggregatePartialCells.add(c); -273 } -274 } while (partialResult.isPartial()); -275 -276 assertTrue("Number of cells differs. iteration: " + iterationCount, -277 oneShotResult.rawCells().length == aggregatePartialCells.size()); -278 final Cell[] oneShotCells = oneShotResult.rawCells(); -279 for (int cell = 0; cell < oneShotCells.length; cell++) { -280 Cell oneShotCell = oneShotCells[cell]; -281 Cell partialCell = aggregatePartialCells.get(cell); -282 -283 assertTrue("One shot cell was null", oneShotCell != null); -284 assertTrue("Partial cell was null", partialCell != null); -285 assertTrue("Cell differs. oneShotCell:" + oneShotCell + " partialCell:" + partialCell, -286 oneShotCell.equals(partialCell)); -287 } -288 -289 oneShotResult = oneShotScanner.next(); -290 iterationCount++; -291 } -292 -293 assertTrue(partialScanner.next() == null); -294 -295 partialScanner.close(); -296 oneShotScanner.close(); -297 } -298 -299 /** -300 * Setting the max result size allows us to control how many cells we expect to see on each call -301 * to next on the scanner. Test a variety of different sizes for correctness -302 * @throws Exception -303 */ -304 @Test -305 public void testExpectedNumberOfCellsPerPartialResult() throws Exception { -306 Scan scan = new Scan(); -307 testExpectedNumberOfCellsPerPartialResult(scan); -308 -309 scan.setReversed(true); +223 r2 = scanner2.next(); +224 assertTrue("r2: " + r2 + " Should be null", r2 == null); +225 +226 scanner1.close(); +227 scanner2.close(); +228 } +229 +230 /** +231 * Order of cells in partial results matches the ordering of cells from complete results +232 * @throws Exception +233 */ +234 @Test +235 public void testOrderingOfCellsInPartialResults() throws Exception { +236 Scan scan = new Scan(); +237 +238 for (int col = 1; col <= NUM_COLS; col++) { +239 scan.setMaxResultSize(getResultSizeForNumberOfCells(col)); +240 testOrderingOfCellsInPartialResults(scan); +241 +242 // Test again with a reversed scanner +243 scan.setReversed(true); +244 testOrderingOfCellsInPartialResults(scan); +245 } +246 } +247 +248 public void testOrderingOfCellsInPartialResults(final Scan basePartialScan) throws Exception { +249 // Scan that retrieves results in pieces (partials). By setting allowPartialResults to be true +250 // the results will NOT be reconstructed and instead the caller will see the partial results +251 // returned by the server +252 Scan partialScan = new Scan(basePartialScan); +253 partialScan.setAllowPartialResults(true); +254 ResultScanner partialScanner = TABLE.getScanner(partialScan); +255 +256 // Scan that retrieves all table results in single RPC request +257 Scan oneShotScan = new Scan(basePartialScan); +258 oneShotScan.setMaxResultSize(Long.MAX_VALUE); +259 oneShotScan.setCaching(ROWS.length); +260 ResultScanner oneShotScanner = TABLE.getScanner(oneShotScan); +261 +262 Result oneShotResult = oneShotScanner.next(); +263 Result partialResult = null; +264 int iterationCount = 0; +265 +266 while (oneShotResult != null && oneShotResult.rawCells() != null) { +267 List<Cell> aggregatePartialCells = new ArrayList<Cell>(); +268 do { +269 partialResult = partialScanner.next(); +270 assertTrue("Partial Result is null. iteration: " + iterationCount, partialResult != null); +271 assertTrue("Partial cells are null. iteration: " + iterationCount, +272 partialResult.rawCells() != null); +273 +274 for (Cell c : partialResult.rawCells()) { +275 aggregatePartialCells.add(c); +276 } +277 } while (partialResult.isPartial()); +278 +279 assertTrue("Number of cells differs. iteration: " + iterationCount, +280 oneShotResult.rawCells().length == aggregatePartialCells.size()); +281 final Cell[] oneShotCells = oneShotResult.rawCells(); +282 for (int cell = 0; cell < oneShotCells.length; cell++) { +283 Cell oneShotCell = oneShotCells[cell]; +284 Cell partialCell = aggregatePartialCells.get(cell); +285 +286 assertTrue("One shot cell was null", oneShotCell != null); +287 assertTrue("Partial cell was null", partialCell != null); +288 assertTrue("Cell differs. oneShotCell:" + oneShotCell + " partialCell:" + partialCell, +289 oneShotCell.equals(partialCell)); +290 } +291 +292 oneShotResult = oneShotScanner.next(); +293 iterationCount++; +294 } +295 +296 assertTrue(partialScanner.next() == null); +297 +298 partialScanner.close(); +299 oneShotScanner.close(); +300 } +301 +302 /** +303 * Setting the max result size allows us to control how many cells we expect to see on each call +304 * to next on the scanner. Test a variety of different sizes for correctness +305 * @throws Exception +306 */ +307 @Test +308 public void testExpectedNumberOfCellsPerPartialResult() throws Exception { +309 Scan scan = new Scan(); 310 testExpectedNumberOfCellsPerPartialResult(scan); -311 } -312 -313 public void testExpectedNumberOfCellsPerPartialResult(Scan baseScan) throws Exception { -314 for (int expectedCells = 1; expectedCells <= NUM_COLS; expectedCells++) { -315 testExpectedNumberOfCellsPerPartialResult(baseScan, expectedCells); -316 } -317 } -318 -319 public void testExpectedNumberOfCellsPerPartialResult(Scan baseScan, int expectedNumberOfCells) -320 throws Exception { +311 +312 scan.setReversed(true); +313 testExpectedNumberOfCellsPerPartialResult(scan); +314 } +315 +316 public void testExpectedNumberOfCellsPerPartialResult(Scan baseScan) throws Exception { +317 for (int expectedCells = 1; expectedCells <= NUM_COLS; expectedCells++) { +318 testExpectedNumberOfCellsPerPartialResult(baseScan, expectedCells); +319 } +320 } 321 -322 if (LOG.isInfoEnabled()) LOG.info("groupSize:" + expectedNumberOfCells); -323 -324 // Use the cellHeapSize to set maxResultSize such that we know how many cells to expect back -325 // from the call. The returned results should NOT exceed expectedNumberOfCells but may be less -326 // than it in cases where expectedNumberOfCells is not an exact multiple of the number of -327 // columns in the table. -328 Scan scan = new Scan(baseScan); -329 scan.setAllowPartialResults(true); -330 scan.setMaxResultSize(getResultSizeForNumberOfCells(expectedNumberOfCells)); -331 -332 ResultScanner scanner = TABLE.getScanner(scan); -333 Result result = null; -334 byte[] prevRow = null; -335 while ((result = scanner.next()) != null) { -336 assertTrue(result.rawCells() != null); -337 -338 // Cases when cell count won't equal expectedNumberOfCells: -339 // 1. Returned result is the final result needed to form the complete result for that row -340 // 2. It is the first result we have seen for that row and thus may have been fetched as -341 // the last group of cells that fit inside the maxResultSize -342 assertTrue( -343 "Result's cell count differed from expected number. result: " + result, -344 result.rawCells().length == expectedNumberOfCells || !result.isPartial() -345 || !Bytes.equals(prevRow, result.getRow())); -346 prevRow = result.getRow(); -347 } -348 -349 scanner.close(); -350 } +322 public void testExpectedNumberOfCellsPerPartialResult(Scan baseScan, int expectedNumberOfCells) +323 throws Exception { +324 +325 if (LOG.isInfoEnabled()) LOG.info("groupSize:" + expectedNumberOfCells); +326 +327 // Use the cellHeapSize to set maxResultSize such that we know how many cells to expect back +328 // from the call. The returned results should NOT exceed expectedNumberOfCells but may be less +329 // than it in cases where expectedNumberOfCells is not an exact multiple of the number of +330 // columns in the table. +331 Scan scan = new Scan(baseScan); +332 scan.setAllowPartialResults(true); +333 scan.setMaxResultSize(getResultSizeForNumberOfCells(expectedNumberOfCells)); +334 +335 ResultScanner scanner = TABLE.getScanner(scan); +336 Result result = null; +337 byte[] prevRow = null; +338 while ((result = scanner.next()) != null) { +339 assertTrue(result.rawCells() != null); +340 +341 // Cases when cell count won't equal expectedNumberOfCells: +342 // 1. Returned result is the final result needed to form the complete result for that row +343 // 2. It is the first result we have seen for that row and thus may have been fetched as +344 // the last group of cells that fit inside the maxResultSize +345 assertTrue( +346 "Result's cell count differed from expected number. result: " + result, +347 result.rawCells().length == expectedNumberOfCells || !result.isPartial() +348 || !Bytes.equals(prevRow, result.getRow())); +349 prevRow = result.getRow(); +350 } 351 -352 /** -353 * @return The approximate heap size of a cell in the test table. All cells should have -354 * approximately the same heap size, so the value is cached to avoid repeating the -355 * calculation -356 * @throws Exception -357 */ -358 private long getCellHeapSize() throws Exception { -359 if (CELL_HEAP_SIZE == -1) { -360 // Do a partial scan that will return a single result with a single cell -361 Scan scan = new Scan(); -362 scan.setMaxResultSize(1); -363 scan.setAllowPartialResults(true); -364 ResultScanner scanner = TABLE.getScanner(scan); -365 -366 Result result = scanner.next(); -367 -368 assertTrue(result != null); -369 assertTrue(result.rawCells() != null); -370 assertTrue(result.rawCells().length == 1); -371 -372 CELL_HEAP_SIZE = CellUtil.estimatedHeapSizeOf(result.rawCells()[0]); -373 if (LOG.isInfoEnabled()) LOG.info("Cell heap size: " + CELL_HEAP_SIZE); -374 scanner.close(); -375 } -376 -377 return CELL_HEAP_SIZE; -378 } +352 scanner.close(); +353 } +354 +355 /** +356 * @return The approximate heap size of a cell in the test table. All cells should have +357 * approximately the same heap size, so the value is cached to avoid repeating the +358 * calculation +359 * @throws Exception +360 */ +361 private long getCellHeapSize() throws Exception { +362 if (CELL_HEAP_SIZE == -1) { +363 // Do a partial scan that will return a single result with a single cell +364 Scan scan = new Scan(); +365 scan.setMaxResultSize(1); +366 scan.setAllowPartialResults(true); +367 ResultScanner scanner = TABLE.getScanner(scan); +368 +369 Result result = scanner.next(); +370 +371 assertTrue(result != null); +372 assertTrue(result.rawCells() != null); +373 assertTrue(result.rawCells().length == 1); +374 +375 CELL_HEAP_SIZE = CellUtil.estimatedHeapSizeOf(result.rawCells()[0]); +376 if (LOG.isInfoEnabled()) LOG.info("Cell heap size: " + CELL_HEAP_SIZE); +377 scanner.close(); +378 } 379 -380 /** -381 * @param numberOfCells -382 * @return the result size that should be used in {@link Scan#setMaxResultSize(long)} if you want -383 * the server to return exactly numberOfCells cells -384 * @throws Exception -385 */ -386 private long getResultSizeForNumberOfCells(int numberOfCells) throws Exception { -387 return getCellHeapSize() * numberOfCells; -388 } -389 -390 /** -391 * Test various combinations of batching and partial results for correctness -392 */ -393 @Test -394 public void testPartialResultsAndBatch() throws Exception { -395 for (int batch = 1; batch <= NUM_COLS / 4; batch++) { -396 for (int cellsPerPartial = 1; cellsPerPartial <= NUM_COLS / 4; cellsPerPartial++) { -397 testPartialResultsAndBatch(batch, cellsPerPartial); -398 } -399 } -400 } -401 -402 public void testPartialResultsAndBatch(final int batch, final int cellsPerPartialResult) -403 throws Exception { -404 if (LOG.isInfoEnabled()) { -405 LOG.info("batch: " + batch + " cellsPerPartialResult: " + cellsPerPartialResult); -406 } -407 -408 Scan scan = new Scan(); -409 scan.setMaxResultSize(getResultSizeForNumberOfCells(cellsPerPartialResult)); -410 scan.setBatch(batch); -411 ResultScanner scanner = TABLE.getScanner(scan); -412 Result result = scanner.next(); -413 int repCount = 0; -414 -415 while ((result = scanner.next()) != null) { -416 assertTrue(result.rawCells() != null); +380 return CELL_HEAP_SIZE; +381 } +382 +383 /** +384 * @param numberOfCells +385 * @return the result size that should be used in {@link Scan#setMaxResultSize(long)} if you want +386 * the server to return exactly numberOfCells cells +387 * @throws Exception +388 */ +389 private long getResultSizeForNumberOfCells(int numberOfCells) throws Exception { +390 return getCellHeapSize() * numberOfCells; +391 } +392 +393 /** +394 * Test various combinations of batching and partial results for correctness +395 */ +396 @Test +397 public void testPartialResultsAndBatch() throws Exception { +398 for (int batch = 1; batch <= NUM_COLS / 4; batch++) { +399 for (int cellsPerPartial = 1; cellsPerPartial <= NUM_COLS / 4; cellsPerPartial++) { +400 testPartialResultsAndBatch(batch, cellsPerPartial); +401 } +402 } +403 } +404 +405 public void testPartialResultsAndBatch(final int batch, final int cellsPerPartialResult) +406 throws Exception { +407 if (LOG.isInfoEnabled()) { +408 LOG.info("batch: " + batch + " cellsPerPartialResult: " + cellsPerPartialResult); +409 } +410 +411 Scan scan = new Scan(); +412 scan.setMaxResultSize(getResultSizeForNumberOfCells(cellsPerPartialResult)); +413 scan.setBatch(batch); +414 ResultScanner scanner = TABLE.getScanner(scan); +415 Result result = scanner.next(); +416 int repCount = 0; 417 -418 if (result.isPartial()) { -419 final String error = -420 "Cells:" + result.rawCells().length + " Batch size:" + batch -421 + " cellsPerPartialResult:" + cellsPerPartialResult + " rep:" + repCount; -422 assertTrue(error, result.rawCells().length <= Math.min(batch, cellsPerPartialResult)); -423 } else { -424 assertTrue(result.rawCells().length <= batch); -425 } -426 repCount++; -427 } -428 -429 scanner.close(); -430 } +418 while ((result = scanner.next()) != null) { +419 assertTrue(result.rawCells() != null); +420 +421 if (result.isPartial()) { +422 final String error = +423 "Cells:" + result.rawCells().length + " Batch size:" + batch +424 + " cellsPerPartialResult:" + cellsPerPartialResult + " rep:" + repCount; +425 assertTrue(error, result.rawCells().length <= Math.min(batch, cellsPerPartialResult)); +426 } else { +427 assertTrue(result.rawCells().length <= batch); +428 } +429 repCount++; +430 } 431 -432 /** -433 * Test the method {@link Result#createCompleteResult(List, Result)} -434 * @throws Exception -435 */ -436 @Test -437 public void testPartialResultsReassembly() throws Exception { -438 Scan scan = new Scan(); -439 testPartialResultsReassembly(scan); -440 scan.setReversed(true); -441 testPartialResultsReassembly(scan); -442 } -443 -444 public void testPartialResultsReassembly(Scan scanBase) throws Exception { -445 Scan partialScan = new Scan(scanBase); -446 partialScan.setMaxResultSize(1); -447 partialScan.setAllowPartialResults(true); -448 ResultScanner partialScanner = TABLE.getScanner(partialScan); -449 -450 Scan oneShotScan = new Scan(scanBase); -451 oneShotScan.setMaxResultSize(Long.MAX_VALUE); -452 ResultScanner oneShotScanner = TABLE.getScanner(oneShotScan); -453 -454 ArrayList<Result> partials = new ArrayList<>(); -455 for (int i = 0; i < NUM_ROWS; i++) { -456 Result partialResult = null; -457 Result completeResult = null; -458 Result oneShotResult = null; -459 partials.clear(); -460 -461 do { -462 partialResult = partialScanner.next(); -463 partials.add(partialResult); -464 } while (partialResult != null && partialResult.isPartial()); -465 -466 completeResult = Result.createCompleteResult(partials); -467 oneShotResult = oneShotScanner.next(); +432 scanner.close(); +433 } +434 +435 /** +436 * Test the method {@link Result#createCompleteResult(List)} +437 * @throws Exception +438 */ +439 @Test +440 public void testPartialResultsReassembly() throws Exception { +441 Scan scan = new Scan(); +442 testPartialResultsReassembly(scan); +443 scan.setReversed(true); +444 testPartialResultsReassembly(scan); +445 } +446 +447 public void testPartialResultsReassembly(Scan scanBase) throws Exception { +448 Scan partialScan = new Scan(scanBase); +449 partialScan.setMaxResultSize(1); +450 partialScan.setAllowPartialResults(true); +451 ResultScanner partialScanner = TABLE.getScanner(partialScan); +452 +453 Scan oneShotScan = new Scan(scanBase); +454 oneShotScan.setMaxResultSize(Long.MAX_VALUE); +455 ResultScanner oneShotScanner = TABLE.getScanner(oneShotScan); +456 +457 ArrayList<Result> partials = new ArrayList<>(); +458 for (int i = 0; i < NUM_ROWS; i++) { +459 Result partialResult = null; +460 Result completeResult = null; +461 Result oneShotResult = null; +462 partials.clear(); +463 +464 do { +465 partialResult = partialScanner.next(); +466 partials.add(partialResult); +467 } while (partialResult != null && partialResult.isPartial()); 468 -469 compareResults(completeResult, oneShotResult, null); -470 } +469 completeResult = Result.createCompleteResult(partials); +470 oneShotResult = oneShotScanner.next(); 471 -472 assertTrue(oneShotScanner.next() == null); -473 assertTrue(partialScanner.next() == null); +472 compareResults(completeResult, oneShotResult, null); +473 } 474 -475 oneShotScanner.close(); -476 partialScanner.close(); -477 } -478 -479 /** -480 * When reconstructing the complete result from its partials we ensure that the row of each<