hbase-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Jamal (Updated) (JIRA)" <j...@apache.org>
Subject [jira] [Updated] (HBASE-5247) ColumnPaginationFilter doesn't return the requested number of columns
Date Sun, 22 Jan 2012 17:30:40 GMT

     [ https://issues.apache.org/jira/browse/HBASE-5247?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Jamal updated HBASE-5247:
-------------------------

    Description: The ColumnPaginationFilter doesn't return the correct number of columns when
the limit is less than the total number of columns.  I don't know much about the internals
of Hbase, but it appears that the problem is more re-producible when a table is split accross
at least 2 regions.  I have created a unit test which can re-produce the issue which is now
attached.  (was: The ColumnPaginationFilter doesn't return the correct number of columns when
the limit is less than the total number of columns.  I don't know much about the internals
of Hbase, but it appears that the problem is more re-producible when a table is split accross
at least 2 regions.  I have created a unit test which can re-produce the issue which is below.
 Sorry, I didn't see a place to upload an attachment.  To run it, just create a standard maven
project with the following 2 files:


pom.xml:
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<groupId>org.sandbox</groupId>
	<artifactId>pagination</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<properties>
		<spring.version>3.0.6.RELEASE</spring.version>
	</properties>
	<dependencies>
		<dependency>
			<groupId>org.springframework</groupId>
			<artifactId>spring-context</artifactId>
			<version>${spring.version}</version>
			<exclusions>
				<exclusion>
					<groupId>commons-logging</groupId>
					<artifactId>commons-logging</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<dependency>
			<groupId>org.apache.hbase</groupId>
			<artifactId>hbase</artifactId>
			<version>0.90.4</version>
		</dependency>
		<dependency>
			<groupId>org.springframework</groupId>
			<artifactId>spring-test</artifactId>
			<version>${spring.version}</version>
			<scope>test</scope>
		</dependency>
		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<version>4.8.2</version>
			<scope>test</scope>
		</dependency>
	</dependencies>
	<repositories>
		<repository>
			<id>com.springsource.repository.maven.release</id>
			<url>http://maven.springframework.org/release/</url>
			<snapshots>
				<enabled>false</enabled>
			</snapshots>
		</repository>
	</repositories>
</project>


PaginationTest.java:
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.codec.digest.DigestUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.ColumnPaginationFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.test.context.TestExecutionListeners;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

@RunWith(SpringJUnit4ClassRunner.class)
@TestExecutionListeners({})
public class PaginationTest {
	private byte[] TABLE_NAME = Bytes.toBytes("pagination-test-table");
	private Configuration conf = HBaseConfiguration.create();
	private HTable table;
	private HBaseAdmin admin;
	private byte[] COLUMN_FAMILY = Bytes.toBytes("stuff");
	byte[] rowKey = Bytes.toBytes("row5");
	private Logger logger = LoggerFactory.getLogger(getClass());
	private int rowCount = 5000, columnCount = 12;

	@Before
	public void setup() throws IOException, InterruptedException {
		admin = new HBaseAdmin(conf);

		if (!admin.isTableAvailable(TABLE_NAME)) {

			HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
			HColumnDescriptor columnDesc = new HColumnDescriptor(COLUMN_FAMILY);
			desc.addFamily(columnDesc);
			admin.createTable(desc);
		}

		table = new HTable(conf, TABLE_NAME);

		for (int z = 0; z < 3; z++) {
			List<Put> puts = new ArrayList<Put>(columnCount);
			for (int j = 0; j < rowCount; j++) {

				byte[] rowKey = Bytes.toBytes("row" + j);
				Put put = new Put(rowKey);
				for (int i = 0; i < columnCount; i++) {

					put.add(COLUMN_FAMILY,
							Bytes.toBytes(DigestUtils.md5Hex("something:" + i)),
							Bytes.toBytes(i));

				}
				puts.add(put);
			}

			table.put(puts);
			table.flushCommits();
			admin.split(TABLE_NAME);

			// Wait for split to "finish"
			Thread.sleep(30000);

			// Wait for flush to "finish"
			admin.flush(TABLE_NAME);
		}

	}

	@After
	public void cleanup() throws IOException {

		table.close();
		admin.disableTable(TABLE_NAME);
		admin.deleteTable(TABLE_NAME);

	}

	@Test
	public void runConditions() throws IOException {
		showAllColumns();
		scanFirst5Columns();
		scanNextFileColumns();
		getFirstFiveColumns();
	}

	public void showAllColumns() throws IOException {
		logger.info("Current contents of row:" + Bytes.toString(rowKey));
		Get get = new Get(rowKey);
		get.addFamily(COLUMN_FAMILY);
		Result result = table.get(get);
		for (KeyValue kv : result.list()) {
			// logger.info("kv:" + kv);
			logger.info("key:" + Bytes.toString(kv.getQualifier()) + " value:"
					+ Bytes.toInt(kv.getValue()));
		}

		// Confirm the # of columns is as expected
		Assert.assertTrue(result.list().size() == columnCount);
	}

	public void scanFirst5Columns() throws IOException {

		// Try Scanning for 5 columns
		logger.info("Scan the first 5 columns of row:" + Bytes.toString(rowKey));
		Scan scan = new Scan(rowKey, rowKey);
		Filter filter = new ColumnPaginationFilter(5, 0);
		scan.setFilter(filter);

		ResultScanner rs = table.getScanner(scan);
		try {
			for (Result r = rs.next(); r != null; r = rs.next()) {
				for (KeyValue kv : r.list()) {
					// logger.info("kv:" + kv);
					logger.info("key:" + Bytes.toString(kv.getQualifier())
							+ " value:" + Bytes.toInt(kv.getValue()));
				}
				// Confirm the # of columns is as expected
				Assert.assertTrue(r.list().size() == 5);
			}
		} finally {
			rs.close();
		}
	}

	public void scanNextFileColumns() throws IOException {

		// Try Scanning for next 5 columns
		logger.info("Scan the next 5 columns of row:" + Bytes.toString(rowKey));
		Scan scan = new Scan(rowKey, rowKey);
		Filter filter = new ColumnPaginationFilter(5, 5);
		scan.setFilter(filter);

		ResultScanner rs = table.getScanner(scan);
		try {
			for (Result r = rs.next(); r != null; r = rs.next()) {
				for (KeyValue kv : r.list()) {
					// logger.info("kv:" + kv);
					logger.info("key:" + Bytes.toString(kv.getQualifier())
							+ " value:" + Bytes.toInt(kv.getValue()));
				}
				// Confirm the # of columns is as expected
				Assert.assertTrue(r.list().size() == 5);
			}
		} finally {
			rs.close();
		}

	}

	public void getFirstFiveColumns() throws IOException {

		// Try Getting 5 columns
		logger.info("Get the first 5 columns of row:" + Bytes.toString(rowKey));
		Get get = new Get(rowKey);
		get.addFamily(COLUMN_FAMILY);

		Filter filter = new ColumnPaginationFilter(5, 0);
		get.setFilter(filter);
		Result firstFiveResult = table.get(get);
		for (KeyValue kv : firstFiveResult.list()) {
			// logger.info("kv:" + kv);
			logger.info("key:" + Bytes.toString(kv.getQualifier()) + " value:"
					+ Bytes.toInt(kv.getValue()));

		}
		// Confirm the # of columns is as expected
		Assert.assertTrue(firstFiveResult.list().size() == 5);

	}
}



)
    
> ColumnPaginationFilter doesn't return the requested number of columns
> ---------------------------------------------------------------------
>
>                 Key: HBASE-5247
>                 URL: https://issues.apache.org/jira/browse/HBASE-5247
>             Project: HBase
>          Issue Type: Bug
>          Components: client
>    Affects Versions: 0.90.4
>         Environment: Hbase 0.90.4-cdh3u2 & standard apache download 0.90.4
> Hadoop 0.20.2-cdh3u2 & standard apache download 0.20.205
>            Reporter: Jamal
>            Priority: Minor
>         Attachments: pagination.zip
>
>
> The ColumnPaginationFilter doesn't return the correct number of columns when the limit
is less than the total number of columns.  I don't know much about the internals of Hbase,
but it appears that the problem is more re-producible when a table is split accross at least
2 regions.  I have created a unit test which can re-produce the issue which is now attached.

--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators: https://issues.apache.org/jira/secure/ContactAdministrators!default.jspa
For more information on JIRA, see: http://www.atlassian.com/software/jira

        

Mime
View raw message