Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 0F9E6200C2A for ; Wed, 1 Mar 2017 20:23:39 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 0E111160B56; Wed, 1 Mar 2017 19:23:39 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D8017160B70 for ; Wed, 1 Mar 2017 20:23:36 +0100 (CET) Received: (qmail 15412 invoked by uid 500); 1 Mar 2017 19:23:36 -0000 Mailing-List: contact commits-help@ignite.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ignite.apache.org Delivered-To: mailing list commits@ignite.apache.org Received: (qmail 15394 invoked by uid 99); 1 Mar 2017 19:23:35 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 01 Mar 2017 19:23:35 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C9C11DFCA1; Wed, 1 Mar 2017 19:23:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sergi@apache.org To: commits@ignite.apache.org Date: Wed, 01 Mar 2017 19:23:35 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/2] ignite git commit: ignite-3477-index2 SQL inline indexes (simple types, Strings, Bytes) archived-at: Wed, 01 Mar 2017 19:23:39 -0000 Repository: ignite Updated Branches: refs/heads/ignite-3477 ee28b9cb8 -> c4490ddce ignite-3477-index2 SQL inline indexes (simple types, Strings, Bytes) Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/78c2a7f4 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/78c2a7f4 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/78c2a7f4 Branch: refs/heads/ignite-3477 Commit: 78c2a7f410d9086d836e4c5596cb47187b52c95d Parents: ee28b9c Author: Konstantin Dudkov Authored: Wed Mar 1 16:54:45 2017 +0300 Committer: Konstantin Dudkov Committed: Wed Mar 1 16:54:45 2017 +0300 ---------------------------------------------------------------------- .../configuration/CacheConfiguration.java | 28 ++ .../core/src/test/java/FullTextIndexTest.java | 316 +++++++++++++++ .../testframework/junits/GridAbstractTest.java | 2 +- .../processors/query/h2/database/H2Tree.java | 120 +++++- .../query/h2/database/H2TreeIndex.java | 167 ++------ .../query/h2/database/InlineIndexHelper.java | 400 ++++++++++++++++--- .../query/h2/database/io/H2ExtrasInnerIO.java | 12 +- .../query/h2/database/io/H2ExtrasLeafIO.java | 9 +- .../processors/query/h2/opt/GridH2Table.java | 2 +- .../query/h2/database/H2TreeIndexTest.java | 59 --- .../h2/database/InlineIndexHelperTest.java | 316 ++++++++++++++- 11 files changed, 1153 insertions(+), 278 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/78c2a7f4/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java index 6c56fc5..c41be0c2 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java @@ -136,6 +136,9 @@ public class CacheConfiguration extends MutableConfiguration { /** Default cache size to use with eviction policy. */ public static final int DFLT_CACHE_SIZE = 100000; + /** Default maximum inline size for sql indexes. */ + public static final int DFLT_SQL_INDEX_MAX_INLINE_SIZE = -1; + /** Initial default near cache size. */ public static final int DFLT_NEAR_START_SIZE = DFLT_START_SIZE / 4; @@ -321,6 +324,9 @@ public class CacheConfiguration extends MutableConfiguration { /** Maximum number of concurrent asynchronous operations. */ private int maxConcurrentAsyncOps = DFLT_MAX_CONCURRENT_ASYNC_OPS; + /** Maximum inline size for sql indexes. */ + private int sqlIndexMaxInlineSize = DFLT_SQL_INDEX_MAX_INLINE_SIZE; + /** Write-behind feature. */ private boolean writeBehindEnabled = DFLT_WRITE_BEHIND_ENABLED; @@ -461,6 +467,7 @@ public class CacheConfiguration extends MutableConfiguration { longQryWarnTimeout = cc.getLongQueryWarningTimeout(); offHeapMaxMem = cc.getOffHeapMaxMemory(); maxConcurrentAsyncOps = cc.getMaxConcurrentAsyncOperations(); + sqlIndexMaxInlineSize = cc.getSqlIndexMaxInlineSize(); name = cc.getName(); nearCfg = cc.getNearConfiguration(); nodeFilter = cc.getNodeFilter(); @@ -1286,6 +1293,27 @@ public class CacheConfiguration extends MutableConfiguration { } /** + * Gets maximum inline size for sql indexes. If -1 returned then + * {@code IgniteSystemProperties.IGNITE_MAX_INDEX_PAYLOAD_SIZE} system property is used. + *

+ * If not set, default value is {@link #DFLT_SQL_INDEX_MAX_INLINE_SIZE}. + * + * @return Maximum payload size for offheap indexes. + */ + public int getSqlIndexMaxInlineSize() { + return sqlIndexMaxInlineSize; + } + + /** + * Sets maximum inline size for sql indexes. + * + * @param sqlIndexMaxInlineSize Maximum inline size for sql indexes. + */ + public void setSqlIndexMaxInlineSize(int sqlIndexMaxInlineSize) { + this.sqlIndexMaxInlineSize = sqlIndexMaxInlineSize; + } + + /** * Flag indicating whether Ignite should use write-behind behaviour for the cache store. * By default write-behind is disabled which is defined via {@link #DFLT_WRITE_BEHIND_ENABLED} * constant. http://git-wip-us.apache.org/repos/asf/ignite/blob/78c2a7f4/modules/core/src/test/java/FullTextIndexTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/FullTextIndexTest.java b/modules/core/src/test/java/FullTextIndexTest.java new file mode 100644 index 0000000..05a9541 --- /dev/null +++ b/modules/core/src/test/java/FullTextIndexTest.java @@ -0,0 +1,316 @@ + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import javax.cache.Cache; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.QueryIndex; +import org.apache.ignite.cache.QueryIndexType; +import org.apache.ignite.cache.query.QueryCursor; +import org.apache.ignite.cache.query.SqlQuery; +import org.apache.ignite.cache.query.TextQuery; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * Created by amashenkov on 17.02.17. + */ +public class FullTextIndexTest extends GridCommonAbstractTest { + /** */ + private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + + /** + * @return Ignite instance. + */ + protected Ignite ignite() { + return grid(0); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + ignite().cache(null).removeAll(); + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + startGridsMultiThreaded(1); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + super.afterTestsStopped(); + + stopAllGrids(); + } + + /** {@inheritDoc} */ + @SuppressWarnings("unchecked") + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration c = super.getConfiguration(gridName); + + c.setDiscoverySpi(new TcpDiscoverySpi().setForceServerMode(true).setIpFinder(ipFinder)); + + // Otherwise noop swap space will be chosen on Windows. +// c.setSwapSpaceSpi(new FileSwapSpaceSpi()); + + CacheConfiguration cc = defaultCacheConfiguration(); + + List entityList = new ArrayList<>(); + + QueryEntity qryEntity = new QueryEntity(); + + qryEntity.setKeyType(Integer.class.getName()); + qryEntity.setValueType(ObjectValue.class.getName()); + qryEntity.addQueryField("strVal", String.class.getName(), null); + + QueryIndex index = new QueryIndex(); // Default index type + index.setFieldNames(Collections.singletonList("strVal"), true); + + qryEntity.setIndexes(Arrays.asList(index)); + + entityList.add(qryEntity); + + qryEntity = new QueryEntity(); + + qryEntity.setKeyType(Integer.class.getName()); + qryEntity.setValueType(ObjectValue2.class.getName()); + qryEntity.addQueryField("strVal", String.class.getName(), null); + + index = new QueryIndex(); + index.setIndexType(QueryIndexType.FULLTEXT); + index.setFieldNames(Collections.singletonList("strVal"), true); + + qryEntity.setIndexes(Arrays.asList(index)); + + entityList.add(qryEntity); + + qryEntity = new QueryEntity(); + + qryEntity.setKeyType(Integer.class.getName()); + qryEntity.setValueType(String.class.getName()); + +// index = new QueryIndex(); +// index.setIndexType(QueryIndexType.SORTED); + + qryEntity.setIndexes(Arrays.asList(index)); + + entityList.add(qryEntity); + + cc.setQueryEntities(entityList); + + c.setCacheConfiguration(cc); + + return c; + } + + /** + * JUnit. + * + * @throws Exception In case of error. + */ + public void testObjectSortedIndex() throws Exception { + IgniteCache cache = ignite().cache(null); + + cache.put(1, new ObjectValue("value 1")); + cache.put(2, new ObjectValue("value 2")); + cache.put(3, new ObjectValue("value 3")); + + QueryCursor> qry + = cache.query(new SqlQuery(ObjectValue.class, "strVal like ?").setArgs("value%")); + + int expCnt = 3; + + List> results = qry.getAll(); + + assertEquals(expCnt, results.size()); + + qry = cache.query(new SqlQuery(ObjectValue.class, "strVal > ?").setArgs("value 1")); + + results = qry.getAll(); + + assertEquals(expCnt - 1, results.size()); + + qry = cache.query(new TextQuery(ObjectValue.class, "value")); + + results = qry.getAll(); + + assertEquals(0, results.size()); + } + + /** + * JUnit. + * + * @throws Exception In case of error. + */ + public void testObjectTextIndex() throws Exception { + IgniteCache cache = ignite(0).cache(null); + + cache.put(1, new ObjectValue2("value 1")); + cache.put(2, new ObjectValue2("value 2")); + cache.put(3, new ObjectValue2("value 3")); + + QueryCursor> qry + = cache.query(new SqlQuery(ObjectValue2.class, "strVal like ?").setArgs("value%")); + + int expCnt = 3; + + List> results = qry.getAll(); + + assertEquals(expCnt, results.size()); + + qry = cache.query(new SqlQuery(ObjectValue2.class, "strVal > ?").setArgs("value 1")); + + results = qry.getAll(); + + assertEquals(expCnt - 1, results.size()); + + qry = cache.query(new TextQuery(ObjectValue2.class, "value")); + + results = qry.getAll(); + + assertEquals(3, results.size()); + } + + /** + * JUnit. + * + * @throws Exception In case of error. + */ + public void testStringDefaultIndex() throws Exception { + IgniteCache cache = ignite(0).cache(null); + + cache.put(1, "value 1"); + cache.put(2, "value 2"); + cache.put(3, "value 3"); + + QueryCursor> qry + = cache.query(new SqlQuery(String.class, "_val like ?").setArgs("value%")); + + int expCnt = 3; + + List> results = qry.getAll(); + + assertEquals(expCnt, results.size()); + + qry = cache.query(new SqlQuery(String.class, "_val > ?").setArgs("value 1")); + + results = qry.getAll(); + + assertEquals(expCnt - 1, results.size()); + + qry = cache.query(new TextQuery(String.class, "value")); + + results = qry.getAll(); + + // There is no way to disable FULLTEXT index. So, next line will fails. + assertEquals(0, results.size()); + } + + /** + * Another test value object. + */ + private static class ObjectValue { + /** Value. */ + private String strVal; + + /** + * @param strVal String value. + */ + ObjectValue(String strVal) { + this.strVal = strVal; + } + + /** + * Gets value. + * + * @return Value. + */ + public String value() { + return strVal; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (o == null || getClass() != o.getClass()) + return false; + + ObjectValue other = (ObjectValue)o; + + return strVal == null ? other.strVal == null : strVal.equals(other.strVal); + + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return strVal != null ? strVal.hashCode() : 0; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(ObjectValue.class, this); + } + } + + /** + * Another test value object. + */ + private static class ObjectValue2 { + /** Value. */ + private String strVal; + + /** + * @param strVal String value. + */ + ObjectValue2(String strVal) { + this.strVal = strVal; + } + + /** + * Gets value. + * + * @return Value. + */ + public String value() { + return strVal; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (o == null || getClass() != o.getClass()) + return false; + + ObjectValue2 other = (ObjectValue2)o; + + return strVal == null ? other.strVal == null : strVal.equals(other.strVal); + + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return strVal != null ? strVal.hashCode() : 0; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(ObjectValue2.class, this); + } + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/78c2a7f4/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java index b45e173..3be511e 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java @@ -1263,7 +1263,7 @@ public abstract class GridAbstractTest extends TestCase { public String getTestGridName() { String[] parts = getClass().getName().split("\\."); - return parts[parts.length - 2] + '.' + parts[parts.length - 1]; + return "tg1"; } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/78c2a7f4/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java index d9b820a..842f035 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java @@ -17,6 +17,8 @@ package org.apache.ignite.internal.processors.query.h2.database; +import java.util.Comparator; +import java.util.List; import java.util.concurrent.atomic.AtomicLong; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.pagemem.Page; @@ -31,6 +33,8 @@ import org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasLeafIO import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row; import org.apache.ignite.internal.util.typedef.internal.U; import org.h2.result.SearchRow; +import org.h2.table.IndexColumn; +import org.h2.value.Value; /** */ @@ -41,6 +45,22 @@ public abstract class H2Tree extends BPlusTree { /** */ private final int inlineSize; + /** */ + private final List inlineIdxs; + + /** */ + private final IndexColumn[] cols; + + /** */ + private final int[] columnIds; + + /** */ + private final Comparator comp = new Comparator() { + @Override public int compare(Value o1, Value o2) { + return compareValues(o1, o2); + } + }; + /** * @param name Tree name. * @param reuseList Reuse list. @@ -62,6 +82,8 @@ public abstract class H2Tree extends BPlusTree { H2RowFactory rowStore, long metaPageId, boolean initNew, + IndexColumn[] cols, + List inlineIdxs, int inlineSize ) throws IgniteCheckedException { super(name, cacheId, pageMem, wal, globalRmvId, metaPageId, reuseList); @@ -76,6 +98,13 @@ public abstract class H2Tree extends BPlusTree { assert rowStore != null; this.rowStore = rowStore; + this.inlineIdxs = inlineIdxs; + this.cols = cols; + + this.columnIds = new int[cols.length]; + + for (int i = 0; i < cols.length; i++) + columnIds[i] = cols[i].column.getColumnId(); setIos(H2ExtrasInnerIO.getVersions(inlineSize), H2ExtrasLeafIO.getVersions(inlineSize)); @@ -123,6 +152,95 @@ public abstract class H2Tree extends BPlusTree { } } } -} + /** {@inheritDoc} */ + @Override protected int compare(BPlusIO io, long pageAddr, int idx, + SearchRow row) throws IgniteCheckedException { + if (inlineSize() == 0) + return compareRows(getRow(io, pageAddr, idx), row); + else { + int off = io.offset(idx); + + int fieldOff = 0; + + int lastIdxUsed = 0; + + for (int i = 0; i < inlineIdxs.size(); i++) { + InlineIndexHelper inlineIdx = inlineIdxs.get(i); + + Value v2 = row.getValue(inlineIdx.columnIndex()); + + if (v2 == null) + return 0; + + int c = inlineIdx.compare(pageAddr, off + fieldOff, inlineSize() - fieldOff, v2, comp); + + if (c == -2) + break; + + lastIdxUsed++; + if (c != 0) + return c; + + fieldOff += inlineIdx.fullSize(pageAddr, off + fieldOff); + + if (fieldOff > inlineSize()) + break; + } + + if (lastIdxUsed == cols.length) + return 0; + + SearchRow rowData = getRow(io, pageAddr, idx); + + for (int i = lastIdxUsed, len = cols.length; i < len; i++) { + IndexColumn col = cols[i]; + int idx0 = col.column.getColumnId(); + + Value v2 = row.getValue(idx0); + if (v2 == null) { + // Can't compare further. + return 0; + } + + Value v1 = rowData.getValue(idx0); + + int c = compareValues(v1, v2); + if (c != 0) + return InlineIndexHelper.fixSort(c, col.sortType); + } + + return 0; + } + } + + /** + * Compare two rows. + * + * @param r1 Row 1. + * @param r2 Row 2. + * @return Compare result. + */ + private int compareRows(GridH2Row r1, SearchRow r2) { + if (r1 == r2) + return 0; + + for (int i = 0, len = cols.length; i < len; i++) { + int idx = columnIds[i]; + Value v1 = r1.getValue(idx); + Value v2 = r2.getValue(idx); + if (v1 == null || v2 == null) { + // can't compare further + return 0; + } + int c = compareValues(v1, v2); + if (c != 0) + return InlineIndexHelper.fixSort(c, cols[i].sortType); + } + return 0; + } + + /** Compares two Values. */ + public abstract int compareValues(Value v1, Value v2); +} http://git-wip-us.apache.org/repos/asf/ignite/blob/78c2a7f4/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java index 2e9940b..4be40af 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java @@ -26,7 +26,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.database.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.database.RootPage; import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree; -import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO; import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO; import org.apache.ignite.internal.processors.query.h2.H2Cursor; import org.apache.ignite.internal.processors.query.h2.opt.GridH2IndexBase; @@ -53,10 +52,7 @@ import org.jetbrains.annotations.Nullable; */ public class H2TreeIndex extends GridH2IndexBase { /** Default value for {@code IGNITE_MAX_INDEX_PAYLOAD_SIZE} */ - public static final int IGNITE_MAX_INDEX_PAYLOAD_SIZE_DEFAULT = 0; - - /** PageContext for use in IO's */ - private static final ThreadLocal currentIndex = new ThreadLocal<>(); + public static final int IGNITE_MAX_INDEX_PAYLOAD_SIZE_DEFAULT = 10; /** */ private final H2Tree tree; @@ -92,80 +88,22 @@ public class H2TreeIndex extends GridH2IndexBase { initBaseIndex(tbl, 0, name, cols, pk ? IndexType.createPrimaryKey(false, false) : IndexType.createNonUnique(false, false, false)); - name = tbl.rowDescriptor().type().typeId() + "_" + name; + name = tbl.rowDescriptor() == null ? "_" + name : tbl.rowDescriptor().type().typeId() + "_" + name; name = BPlusTree.treeName(name, "H2Tree"); if (cctx.affinityNode()) { IgniteCacheDatabaseSharedManager dbMgr = cctx.shared().database(); - RootPage page = cctx.offheap().rootPageForIndex(name); + RootPage page = getMetaPage(name); inlineIdxs = getAvailableInlineColumns(cols); tree = new H2Tree(name, cctx.offheap().reuseListForIndex(name), cctx.cacheId(), dbMgr.pageMemory(), cctx.shared().wal(), cctx.offheap().globalRemoveId(), - tbl.rowFactory(), page.pageId().pageId(), page.isAllocated(), computeInlineSize(inlineIdxs, inlineSize)) { - @Override protected int compare(BPlusIO io, long pageAddr, int idx, SearchRow row) - throws IgniteCheckedException { - if (inlineSize() == 0) - return compareRows(getRow(io, pageAddr, idx), row); - else { - int off = io.offset(idx); - - int fieldOff = 0; - - int lastIdxUsed = 0; - - for (int i = 0; i < inlineIdxs.size(); i++) { - InlineIndexHelper inlineIdx = inlineIdxs.get(i); - - Value v2 = row.getValue(inlineIdx.columnIndex()); - - if (v2 == null) - return 0; - - Value v1 = inlineIdx.get(pageAddr, off + fieldOff, inlineSize() - fieldOff); - - if (v1 == null) - break; - - int c = compareValues(v1, v2, inlineIdx.sortType()); - - if (!canRelyOnCompare(c, v1, v2, inlineIdx)) - break; - - lastIdxUsed++; - - if (c != 0) - return c; - - fieldOff += inlineIdx.fullSize(pageAddr, off + fieldOff); - - if (fieldOff > inlineSize()) - break; - } - - SearchRow rowData = getRow(io, pageAddr, idx); - - for (int i = lastIdxUsed, len = indexColumns.length; i < len; i++) { - int idx0 = columnIds[i]; - - Value v2 = row.getValue(idx0); - if (v2 == null) { - // Can't compare further. - return 0; - } - - Value v1 = rowData.getValue(idx0); - - int c = compareValues(v1, v2, indexColumns[i].sortType); - if (c != 0) - return c; - } - - return 0; - } + tbl.rowFactory(), page.pageId().pageId(), page.isAllocated(), cols, inlineIdxs, computeInlineSize(inlineIdxs, inlineSize)) { + @Override public int compareValues(Value v1, Value v2) { + return v1 == v2 ? 0 : table.compareTypeSafe(v1, v2); } }; } @@ -183,8 +121,6 @@ public class H2TreeIndex extends GridH2IndexBase { * @return List of {@link InlineIndexHelper} objects. */ private List getAvailableInlineColumns(IndexColumn[] cols) { - - // todo: null List res = new ArrayList<>(); for (int i = 0; i < cols.length; i++) { @@ -201,45 +137,6 @@ public class H2TreeIndex extends GridH2IndexBase { return res; } - /** - * @return Tree updated in current thread. - */ - public static H2TreeIndex getCurrentIndex() { - return currentIndex.get(); - } - - /** - * @param a First value. - * @param b Second Value. - * @param sortType Sort type. - * @return Compare result. - */ - private int compareValues(Value a, Value b, int sortType) { - if (a == b) - return 0; - - int comp = table.compareTypeSafe(a, b); - - if ((sortType & SortOrder.DESCENDING) != 0) - comp = -comp; - - return comp; - } - - /** - * @return Tree. - */ - public H2Tree tree() { - return tree; - } - - /** - * @return InlineIndexHelper list. - */ - public List inlineIndexes() { - return inlineIdxs; - } - /** {@inheritDoc} */ @Override public Cursor find(Session ses, SearchRow lower, SearchRow upper) { try { @@ -272,7 +169,7 @@ public class H2TreeIndex extends GridH2IndexBase { /** {@inheritDoc} */ @Override public GridH2Row put(GridH2Row row) { try { - currentIndex.set(this); + InlineIndexHelper.setCurrentInlineIndexes(inlineIdxs); return tree.put(row); } @@ -280,14 +177,14 @@ public class H2TreeIndex extends GridH2IndexBase { throw DbException.convert(e); } finally { - currentIndex.remove(); + InlineIndexHelper.clearCurrentInlineIndexes(); } } /** {@inheritDoc} */ @Override public boolean putx(GridH2Row row) { try { - currentIndex.set(this); + InlineIndexHelper.setCurrentInlineIndexes(inlineIdxs); return tree.putx(row); } @@ -295,35 +192,35 @@ public class H2TreeIndex extends GridH2IndexBase { throw DbException.convert(e); } finally { - currentIndex.remove(); + InlineIndexHelper.clearCurrentInlineIndexes(); } } /** {@inheritDoc} */ @Override public GridH2Row remove(SearchRow row) { try { - currentIndex.set(this); + InlineIndexHelper.setCurrentInlineIndexes(inlineIdxs); return tree.remove(row); } catch (IgniteCheckedException e) { throw DbException.convert(e); } finally { - currentIndex.remove(); + InlineIndexHelper.clearCurrentInlineIndexes(); } } /** {@inheritDoc} */ @Override public void removex(SearchRow row) { try { - currentIndex.set(this); + InlineIndexHelper.setCurrentInlineIndexes(inlineIdxs); tree.removex(row); } catch (IgniteCheckedException e) { throw DbException.convert(e); } finally { - currentIndex.remove(); + InlineIndexHelper.clearCurrentInlineIndexes(); } } @@ -336,7 +233,6 @@ public class H2TreeIndex extends GridH2IndexBase { int mul = getDistributedMultiplier(ses, filters, filter); return mul * baseCost; - } /** {@inheritDoc} */ @@ -421,10 +317,10 @@ public class H2TreeIndex extends GridH2IndexBase { * @return Inline size. */ private int computeInlineSize(List inlineIdxs, int cfgInlineSize) { - int maxSize = PageIO.MAX_PAYLOAD_SIZE; + int confSize = cctx.config().getSqlIndexMaxInlineSize(); - int propSize = IgniteSystemProperties.getInteger(IgniteSystemProperties.IGNITE_MAX_INDEX_PAYLOAD_SIZE, - IGNITE_MAX_INDEX_PAYLOAD_SIZE_DEFAULT); + int propSize = confSize == -1 ? IgniteSystemProperties.getInteger(IgniteSystemProperties.IGNITE_MAX_INDEX_PAYLOAD_SIZE, + IGNITE_MAX_INDEX_PAYLOAD_SIZE_DEFAULT) : confSize; if (cfgInlineSize == 0) return 0; @@ -448,33 +344,18 @@ public class H2TreeIndex extends GridH2IndexBase { size += idxHelper.size() + 1; } - return Math.min(maxSize, size); + return Math.min(PageIO.MAX_PAYLOAD_SIZE, size); } else - return Math.min(maxSize, cfgInlineSize); + return Math.min(PageIO.MAX_PAYLOAD_SIZE, cfgInlineSize); } /** - * @param c Compare result. - * @param shortVal Short value. - * @param v2 Second value; - * @param inlineIdx Index helper. - * @return {@code true} if we can rely on compare result. + * @param name Name. + * @return RootPage for meta page. + * @throws IgniteCheckedException */ - protected static boolean canRelyOnCompare(int c, Value shortVal, Value v2, InlineIndexHelper inlineIdx) { - if (inlineIdx.type() == Value.STRING) { - if (c == 0 && shortVal.getType() != Value.NULL && v2.getType() != Value.NULL) - return false; - - if (shortVal.getType() != Value.NULL - && v2.getType() != Value.NULL - && ((c < 0 && inlineIdx.sortType() == SortOrder.ASCENDING) || (c > 0 && inlineIdx.sortType() == SortOrder.DESCENDING)) - && shortVal.getString().length() <= v2.getString().length()) { - // Can't rely on compare, should use full string. - return false; - } - } - - return true; + private RootPage getMetaPage(String name) throws IgniteCheckedException { + return cctx.offheap().rootPageForIndex(name); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/78c2a7f4/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java index ff44df5..9492241 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java @@ -20,16 +20,29 @@ package org.apache.ignite.internal.processors.query.h2.database; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Comparator; import java.util.List; import org.apache.ignite.internal.pagemem.PageUtils; +import org.h2.result.SortOrder; import org.h2.table.IndexColumn; import org.h2.value.Value; import org.h2.value.ValueBoolean; import org.h2.value.ValueByte; +import org.h2.value.ValueBytes; +import org.h2.value.ValueDate; +import org.h2.value.ValueDouble; +import org.h2.value.ValueFloat; import org.h2.value.ValueInt; import org.h2.value.ValueLong; import org.h2.value.ValueNull; +import org.h2.value.ValueShort; import org.h2.value.ValueString; +import org.h2.value.ValueStringFixed; +import org.h2.value.ValueStringIgnoreCase; +import org.h2.value.ValueTime; +import org.h2.value.ValueTimestamp; +import org.h2.value.ValueTimestampUtc; +import org.h2.value.ValueUuid; /** * Helper class for in-page indexes. @@ -37,6 +50,9 @@ import org.h2.value.ValueString; public class InlineIndexHelper { private static final Charset CHARSET = StandardCharsets.UTF_8; + /** PageContext for use in IO's */ + private static final ThreadLocal> currentIndex = new ThreadLocal<>(); + /** */ public static final List AVAILABLE_TYPES = Arrays.asList( Value.BOOLEAN, @@ -44,7 +60,18 @@ public class InlineIndexHelper { Value.SHORT, Value.INT, Value.LONG, - Value.STRING + Value.LONG, + Value.FLOAT, + Value.DOUBLE, + Value.DATE, + Value.TIME, + Value.TIMESTAMP, + Value.TIMESTAMP_UTC, + Value.UUID, + Value.STRING, + Value.STRING_FIXED, + Value.STRING_IGNORECASE, + Value.BYTES ); /** */ @@ -56,6 +83,9 @@ public class InlineIndexHelper { /** */ private final int sortType; + /** */ + private final short size; + /** * @param type Index type (see {@link Value}). * @param colIdx Index column index. @@ -65,6 +95,63 @@ public class InlineIndexHelper { this.type = type; this.colIdx = colIdx; this.sortType = sortType; + + switch (type) { + case Value.BOOLEAN: + case Value.BYTE: + this.size = 1; + break; + + case Value.SHORT: + this.size = 2; + break; + + case Value.INT: + this.size = 4; + break; + + case Value.LONG: + this.size = 8; + break; + + case Value.FLOAT: + this.size = 4; + break; + + case Value.DOUBLE: + this.size = 8; + break; + + case Value.DATE: + this.size = 8; + break; + + case Value.TIME: + this.size = 8; + break; + + case Value.TIMESTAMP: + this.size = 16; + break; + + case Value.TIMESTAMP_UTC: + this.size = 8; + break; + + case Value.UUID: + this.size = 16; + break; + + case Value.STRING: + case Value.STRING_FIXED: + case Value.STRING_IGNORECASE: + case Value.BYTES: + this.size = -1; + break; + + default: + throw new UnsupportedOperationException("no get operation for fast index type " + type); + } } /** @@ -89,29 +176,31 @@ public class InlineIndexHelper { } /** - * @return Value size. + * @return Page context for current thread. */ - public short size() { - switch (type) { - case Value.BOOLEAN: - case Value.BYTE: - return 1; - - case Value.SHORT: - return 2; - - case Value.INT: - return 4; + public static List getCurrentInlineIndexes() { + return currentIndex.get(); + } - case Value.LONG: - return 8; + /** + * Sets page context for current thread. + */ + public static void setCurrentInlineIndexes(List inlineIdxs) { + currentIndex.set(inlineIdxs); + } - case Value.STRING: - return -1; + /** + * Clears current context. + */ + public static void clearCurrentInlineIndexes() { + currentIndex.remove(); + } - default: - throw new UnsupportedOperationException("no get operation for fast index type " + type); - } + /** + * @return Value size. + */ + public short size() { + return size; } /** @@ -125,20 +214,10 @@ public class InlineIndexHelper { if (type == Value.NULL) return 1; - switch (type) { - case Value.BOOLEAN: - case Value.BYTE: - case Value.INT: - case Value.SHORT: - case Value.LONG: - return size() + 1; - - case Value.STRING: - return PageUtils.getShort(pageAddr, off + 1) + 3; - - default: - throw new UnsupportedOperationException("no get operation for fast index type " + type); - } + if (size > 0) + return size + 1; + else + return PageUtils.getShort(pageAddr, off + 1) + 3; } /** @@ -147,7 +226,7 @@ public class InlineIndexHelper { * @return Value. */ public Value get(long pageAddr, int off, int maxSize) { - if (size() > 0 && size() + 1 > maxSize) + if (size > 0 && size + 1 > maxSize) return null; int type = PageUtils.getByte(pageAddr, off); @@ -169,7 +248,7 @@ public class InlineIndexHelper { return ValueByte.get(PageUtils.getByte(pageAddr, off + 1)); case Value.SHORT: - return ValueInt.get(PageUtils.getShort(pageAddr, off + 1)); + return ValueShort.get(PageUtils.getShort(pageAddr, off + 1)); case Value.INT: return ValueInt.get(PageUtils.getInt(pageAddr, off + 1)); @@ -177,15 +256,103 @@ public class InlineIndexHelper { case Value.LONG: return ValueLong.get(PageUtils.getLong(pageAddr, off + 1)); + case Value.FLOAT: { + return ValueFloat.get(Float.intBitsToFloat(PageUtils.getInt(pageAddr, off + 1))); + } + + case Value.DOUBLE: { + return ValueDouble.get(Double.longBitsToDouble(PageUtils.getLong(pageAddr, off + 1))); + } + + case Value.TIME: + return ValueTime.fromNanos(PageUtils.getLong(pageAddr, off + 1)); + + case Value.DATE: + return ValueDate.fromDateValue(PageUtils.getLong(pageAddr, off + 1)); + + case Value.TIMESTAMP: + return ValueTimestamp.fromDateValueAndNanos(PageUtils.getLong(pageAddr, off + 1), PageUtils.getLong(pageAddr, off + 9)); + + case Value.TIMESTAMP_UTC: + return ValueTimestampUtc.fromNanos(PageUtils.getLong(pageAddr, off + 1)); + + case Value.UUID: + return ValueUuid.get(PageUtils.getLong(pageAddr, off + 1), PageUtils.getLong(pageAddr, off + 9)); + case Value.STRING: - short size = PageUtils.getShort(pageAddr, off + 1); - return ValueString.get(new String(PageUtils.getBytes(pageAddr, off + 3, size), CHARSET)); + return ValueString.get(new String(readBytes(pageAddr, off), CHARSET)); + + case Value.STRING_FIXED: + return ValueStringFixed.get(new String(readBytes(pageAddr, off), CHARSET)); + + case Value.STRING_IGNORECASE: + return ValueStringIgnoreCase.get(new String(readBytes(pageAddr, off), CHARSET)); + + case Value.BYTES: + return ValueBytes.get(readBytes(pageAddr, off)); default: throw new UnsupportedOperationException("no get operation for fast index type " + type); } } + /** Read variable length bytearray */ + private static byte[] readBytes(long pageAddr, int off) { + int size = PageUtils.getShort(pageAddr, off + 1) & 0x7FFF; + return PageUtils.getBytes(pageAddr, off + 3, size); + } + + /** + * @param pageAddr Page address. + * @param off Offset. + * @return {@code True} if string is not truncated on save. + */ + protected boolean isValueFull(long pageAddr, int off) { + switch (type) { + case Value.BOOLEAN: + case Value.BYTE: + case Value.INT: + case Value.SHORT: + case Value.LONG: + return true; + + case Value.STRING: + case Value.STRING_FIXED: + case Value.STRING_IGNORECASE: + case Value.BYTES: + return (PageUtils.getShort(pageAddr, off + 1) & 0x8000) == 0; + + default: + throw new UnsupportedOperationException("no get operation for fast index type " + type); + } + } + + /** + * @param pageAddr Page address. + * @param off Offset. + * @param maxSize Maximum size to read. + * @param v Value to compare. + * @param comp Comparator. + * @return Compare result (-2 means we can't compare). + */ + public int compare(long pageAddr, int off, int maxSize, Value v, Comparator comp) { + Value v1 = get(pageAddr, off, maxSize); + + if (v1 == null) + return -2; + + int c = comp.compare(v1, v); + assert c > -2; + + if (size > 0) + return fixSort(c, sortType()); + + if (isValueFull(pageAddr, off) || canRelyOnCompare(c, v1, v)) + return fixSort(c, sortType()); + + return -2; + } + /** * @param pageAddr Page address. * @param off Offset. @@ -193,8 +360,14 @@ public class InlineIndexHelper { * @return NUmber of bytes saved. */ public int put(long pageAddr, int off, Value val, int maxSize) { - if (size() > 0 && size() + 1 > maxSize) + if (size > 0 && size + 1 > maxSize) + return 0; + + if (size < 0 && maxSize < 4) { + // can't fit vartype field + PageUtils.putByte(pageAddr, off, (byte)Value.UNKNOWN); return 0; + } if (val.getType() == Value.NULL) { PageUtils.putByte(pageAddr, off, (byte)Value.NULL); @@ -208,34 +381,79 @@ public class InlineIndexHelper { case Value.BOOLEAN: PageUtils.putByte(pageAddr, off, (byte)val.getType()); PageUtils.putByte(pageAddr, off + 1, (byte)(val.getBoolean() ? 1 : 0)); - return size() + 1; + return size + 1; case Value.BYTE: PageUtils.putByte(pageAddr, off, (byte)val.getType()); PageUtils.putByte(pageAddr, off + 1, val.getByte()); - return size() + 1; + return size + 1; case Value.SHORT: PageUtils.putByte(pageAddr, off, (byte)val.getType()); PageUtils.putShort(pageAddr, off + 1, val.getShort()); - return size() + 1; + return size + 1; case Value.INT: PageUtils.putByte(pageAddr, off, (byte)val.getType()); PageUtils.putInt(pageAddr, off + 1, val.getInt()); - return size() + 1; + return size + 1; case Value.LONG: PageUtils.putByte(pageAddr, off, (byte)val.getType()); PageUtils.putLong(pageAddr, off + 1, val.getLong()); - return size() + 1; + return size + 1; + + case Value.FLOAT: { + PageUtils.putByte(pageAddr, off, (byte)val.getType()); + PageUtils.putInt(pageAddr, off + 1, Float.floatToIntBits(val.getFloat())); + return size + 1; + } + + case Value.DOUBLE: { + PageUtils.putByte(pageAddr, off, (byte)val.getType()); + PageUtils.putLong(pageAddr, off + 1, Double.doubleToLongBits(val.getDouble())); + return size + 1; + } + + case Value.TIME: + PageUtils.putByte(pageAddr, off, (byte)val.getType()); + PageUtils.putLong(pageAddr, off + 1, ((ValueTime)val).getNanos()); + return size + 1; + + case Value.DATE: + PageUtils.putByte(pageAddr, off, (byte)val.getType()); + PageUtils.putLong(pageAddr, off + 1, ((ValueDate)val).getDateValue()); + return size + 1; + + case Value.TIMESTAMP: + PageUtils.putByte(pageAddr, off, (byte)val.getType()); + PageUtils.putLong(pageAddr, off + 1, ((ValueTimestamp)val).getDateValue()); + PageUtils.putLong(pageAddr, off + 9, ((ValueTimestamp)val).getTimeNanos()); + return size + 1; + + case Value.TIMESTAMP_UTC: + PageUtils.putByte(pageAddr, off, (byte)val.getType()); + PageUtils.putLong(pageAddr, off + 1, ((ValueTimestampUtc)val).getUtcDateTimeNanos()); + return size + 1; + + case Value.UUID: + PageUtils.putByte(pageAddr, off, (byte)val.getType()); + PageUtils.putLong(pageAddr, off + 1, ((ValueUuid)val).getHigh()); + PageUtils.putLong(pageAddr, off + 9, ((ValueUuid)val).getLow()); + return size + 1; case Value.STRING: - byte[] s; - if (val.getString().getBytes(CHARSET).length + 3 <= maxSize) - s = val.getString().getBytes(CHARSET); - else - s = toBytes(val.getString(), maxSize - 3); + case Value.STRING_FIXED: + case Value.STRING_IGNORECASE: { + short size; + + byte[] s = val.getString().getBytes(CHARSET); + if (s.length + 3 <= maxSize) + size = (short)s.length; + else { + s = trimUTF8(s, maxSize - 3); + size = (short)(s.length | 0x8000); + } if (s == null) { // Can't fit anything to @@ -244,10 +462,30 @@ public class InlineIndexHelper { } else { PageUtils.putByte(pageAddr, off, (byte)val.getType()); - PageUtils.putShort(pageAddr, off + 1, (short)s.length); + PageUtils.putShort(pageAddr, off + 1, size); PageUtils.putBytes(pageAddr, off + 3, s); return s.length + 3; } + } + + case Value.BYTES: { + byte[] s; + short size; + + PageUtils.putByte(pageAddr, off, (byte)val.getType()); + + if (val.getBytes().length + 3 <= maxSize) { + size = (short)val.getBytes().length; + PageUtils.putShort(pageAddr, off + 1, size); + PageUtils.putBytes(pageAddr, off + 3, val.getBytes()); + } + else { + size = (short)((maxSize - 3) | 0x8000); + PageUtils.putShort(pageAddr, off + 1, size); + PageUtils.putBytes(pageAddr, off + 3, Arrays.copyOfRange(val.getBytes(), 0, maxSize - 3)); + } + return size + 3; + } default: throw new UnsupportedOperationException("no get operation for fast index type " + type); @@ -257,17 +495,16 @@ public class InlineIndexHelper { /** * Convert String to byte[] with size limit, according to UTF-8 encoding. * - * @param s String. + * @param bytes byte[]. * @param limit Size limit. * @return byte[]. */ - public static byte[] toBytes(String s, int limit) { - byte[] bytes = s.getBytes(CHARSET); + public static byte[] trimUTF8(byte[] bytes, int limit) { if (bytes.length <= limit) return bytes; - for (int i = bytes.length - 1; i > 0; i--) { - if ((bytes[i] & 0xc0) != 0x80 && i <= limit) { + for (int i = limit; i > 0; i--) { + if ((bytes[i] & 0xc0) != 0x80) { byte[] res = new byte[i]; System.arraycopy(bytes, 0, res, 0, i); return res; @@ -276,4 +513,57 @@ public class InlineIndexHelper { return null; } + + /** + * @param c Compare result. + * @param shortVal Short value. + * @param v2 Second value; + * @return {@code true} if we can rely on compare result. + */ + protected boolean canRelyOnCompare(int c, Value shortVal, Value v2) { + switch (type) { + case Value.STRING: + case Value.STRING_FIXED: + case Value.STRING_IGNORECASE: + case Value.BYTES: + if (shortVal.getType() == Value.NULL || v2.getType() == Value.NULL) + return true; + + if (c == 0 && shortVal.getType() != Value.NULL && v2.getType() != Value.NULL) + return false; + + int l1; + int l2; + + if (type == Value.BYTES) { + l1 = shortVal.getBytes().length; + l2 = v2.getBytes().length; + } + else { + l1 = shortVal.getString().length(); + l2 = v2.getString().length(); + } + + if (c < 0 && l1 <= l2) { + // Can't rely on compare, should use full value. + return false; + } + + return true; + + default: + return true; + } + } + + /** + * Perform sort order correction. + * + * @param c Compare result. + * @param sortType Sort type. + * @return Fixed compare result. + */ + public static int fixSort(int c, int sortType) { + return sortType == SortOrder.ASCENDING ? c : -c; + } } http://git-wip-us.apache.org/repos/asf/ignite/blob/78c2a7f4/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasInnerIO.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasInnerIO.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasInnerIO.java index 00ba54e..5e6a36f 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasInnerIO.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasInnerIO.java @@ -26,7 +26,6 @@ import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusInnerIO import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions; import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO; import org.apache.ignite.internal.processors.query.h2.database.H2Tree; -import org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex; import org.apache.ignite.internal.processors.query.h2.database.InlineIndexHelper; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row; import org.h2.result.SearchRow; @@ -82,18 +81,15 @@ public class H2ExtrasInnerIO extends BPlusInnerIO { assert row0.link != 0 : row0; - H2TreeIndex currIdx = H2TreeIndex.getCurrentIndex(); + List inlineIdxs = InlineIndexHelper.getCurrentInlineIndexes(); - assert currIdx != null; + assert inlineIdxs != null : "no inline index helpers"; - List inlineIdx = currIdx.inlineIndexes(); - - assert inlineIdx != null; int fieldOff = 0; - for (int i = 0; i < inlineIdx.size(); i++) { - InlineIndexHelper idx = inlineIdx.get(i); + for (int i = 0; i < inlineIdxs.size(); i++) { + InlineIndexHelper idx = inlineIdxs.get(i); int size = idx.put(pageAddr, off + fieldOff, row.getValue(idx.columnIndex()), payloadSize - fieldOff); http://git-wip-us.apache.org/repos/asf/ignite/blob/78c2a7f4/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasLeafIO.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasLeafIO.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasLeafIO.java index 45558cf..c4bb387 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasLeafIO.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasLeafIO.java @@ -26,7 +26,6 @@ import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusLeafIO; import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions; import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO; import org.apache.ignite.internal.processors.query.h2.database.H2Tree; -import org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex; import org.apache.ignite.internal.processors.query.h2.database.InlineIndexHelper; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row; import org.h2.result.SearchRow; @@ -82,13 +81,9 @@ public class H2ExtrasLeafIO extends BPlusLeafIO { assert row0.link != 0; - H2TreeIndex currIdx = H2TreeIndex.getCurrentIndex(); + List inlineIdxs = InlineIndexHelper.getCurrentInlineIndexes(); - assert currIdx != null; - - List inlineIdxs = currIdx.inlineIndexes(); - - assert inlineIdxs != null; + assert inlineIdxs != null : "no inline index helpers"; int fieldOff = 0; http://git-wip-us.apache.org/repos/asf/ignite/blob/78c2a7f4/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java index 30219dc..b93bd5e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java @@ -579,7 +579,7 @@ public class GridH2Table extends TableBase { * @throws IgniteCheckedException If failed. */ @SuppressWarnings("LockAcquiredButNotSafelyReleased") - boolean doUpdate(final GridH2Row row, boolean del) throws IgniteCheckedException { + public boolean doUpdate(final GridH2Row row, boolean del) throws IgniteCheckedException { // Here we assume that each key can't be updated concurrently and case when different indexes // getting updated from different threads with different rows with the same key is impossible. GridUnsafeMemory mem = desc == null ? null : desc.memory(); http://git-wip-us.apache.org/repos/asf/ignite/blob/78c2a7f4/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndexTest.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndexTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndexTest.java deleted file mode 100644 index 064da45..0000000 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndexTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.query.h2.database; - -import junit.framework.TestCase; -import org.h2.result.SortOrder; -import org.h2.value.Value; -import org.h2.value.ValueNull; -import org.h2.value.ValueString; - -/** - * Simple tests for {@link H2TreeIndex}. - */ -public class H2TreeIndexTest extends TestCase { - - /** Test on String values compare */ - public void testRelyOnCompare() { - - InlineIndexHelper ha = new InlineIndexHelper(Value.STRING, 0, SortOrder.ASCENDING); - InlineIndexHelper hd = new InlineIndexHelper(Value.STRING, 0, SortOrder.DESCENDING); - - // same size - assertFalse(H2TreeIndex.canRelyOnCompare(0, ValueString.get("aabb"), ValueString.get("aabb"), ha)); - assertFalse(H2TreeIndex.canRelyOnCompare(0, ValueString.get("aabb"), ValueString.get("aabb"), hd)); - - // second string is shorter - assertTrue(H2TreeIndex.canRelyOnCompare(1, ValueString.get("aabb"), ValueString.get("aab"), ha)); - assertTrue(H2TreeIndex.canRelyOnCompare(-1, ValueString.get("aabb"), ValueString.get("aab"), hd)); - - // second string is longer - assertTrue(H2TreeIndex.canRelyOnCompare(1, ValueString.get("aabb"), ValueString.get("aaaaaa"), ha)); - assertTrue(H2TreeIndex.canRelyOnCompare(-1, ValueString.get("aabb"), ValueString.get("aaaaaa"), hd)); - - assertFalse(H2TreeIndex.canRelyOnCompare(-1, ValueString.get("aab"), ValueString.get("aabbbbb"), ha)); - assertFalse(H2TreeIndex.canRelyOnCompare(1, ValueString.get("aab"), ValueString.get("aabbbbb"), hd)); - - // one is null - assertTrue(H2TreeIndex.canRelyOnCompare(1, ValueString.get("aabb"), ValueNull.INSTANCE, ha)); - assertTrue(H2TreeIndex.canRelyOnCompare(-1, ValueNull.INSTANCE, ValueString.get("aab"), ha)); - assertTrue(H2TreeIndex.canRelyOnCompare(0, ValueNull.INSTANCE, ValueNull.INSTANCE, ha)); - - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/78c2a7f4/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java index f65f0fa..bcbca1a 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java @@ -17,30 +17,340 @@ package org.apache.ignite.internal.processors.query.h2.database; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.Arrays; +import java.util.UUID; import junit.framework.TestCase; +import org.apache.commons.io.Charsets; +import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider; +import org.apache.ignite.internal.pagemem.FullPageId; +import org.apache.ignite.internal.pagemem.Page; +import org.apache.ignite.internal.pagemem.PageIdAllocator; +import org.apache.ignite.internal.pagemem.PageMemory; +import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl; +import org.apache.ignite.logger.java.JavaLogger; +import org.h2.result.SortOrder; +import org.h2.value.CompareMode; +import org.h2.value.Value; +import org.h2.value.ValueBoolean; +import org.h2.value.ValueByte; +import org.h2.value.ValueBytes; +import org.h2.value.ValueDate; +import org.h2.value.ValueDouble; +import org.h2.value.ValueFloat; +import org.h2.value.ValueInt; +import org.h2.value.ValueLong; +import org.h2.value.ValueNull; +import org.h2.value.ValueShort; +import org.h2.value.ValueString; +import org.h2.value.ValueTime; +import org.h2.value.ValueTimestamp; +import org.h2.value.ValueTimestampUtc; +import org.h2.value.ValueUuid; /** * Simple tests for {@link InlineIndexHelper}. */ public class InlineIndexHelperTest extends TestCase { + /** */ + private static final int CACHE_ID = 42; + + /** */ + private static final int PAGE_SIZE = 1024; + + /** */ + private static final long MB = 1024; + + /** */ + private static final int CPUS = Runtime.getRuntime().availableProcessors(); /** Test utf-8 string cutting. */ public void testConvert() { // 8 bytes total: 1b, 1b, 3b, 3b. - byte[] bytes = InlineIndexHelper.toBytes("00\u20ac\u20ac", 7); + byte[] bytes = InlineIndexHelper.trimUTF8("00\u20ac\u20ac".getBytes(Charsets.UTF_8), 7); assertEquals(5, bytes.length); String s = new String(bytes); assertEquals(3, s.length()); + + bytes = InlineIndexHelper.trimUTF8("aaaaaa".getBytes(Charsets.UTF_8), 4); + assertEquals(4, bytes.length); } /** Limit is too small to cut */ - public void testShort() { + public void testStringCut() { // 6 bytes total: 3b, 3b. - byte[] bytes = InlineIndexHelper.toBytes("\u20ac\u20ac", 2); + byte[] bytes = InlineIndexHelper.trimUTF8("\u20ac\u20ac".getBytes(Charsets.UTF_8), 2); assertNull(bytes); } + /** Test on String values compare */ + public void testRelyOnCompare() { + InlineIndexHelper ha = new InlineIndexHelper(Value.STRING, 0, SortOrder.ASCENDING); + + // same size + assertFalse(getRes(ha, "aabb", "aabb")); + + // second string is shorter + assertTrue(getRes(ha, "aabb", "aac")); + assertTrue(getRes(ha, "aabb", "aaa")); + + // second string is longer + assertTrue(getRes(ha, "aabb", "aaaaaa")); + assertFalse(getRes(ha, "aaa", "aaaaaa")); + + // one is null + assertTrue(getRes(ha, "a", null)); + assertTrue(getRes(ha, null, "a")); + assertTrue(getRes(ha, null, null)); + } + + /** Test on Bytes values compare */ + public void testRelyOnCompareBytes() { + InlineIndexHelper ha = new InlineIndexHelper(Value.BYTES, 0, SortOrder.ASCENDING); + + // same size + assertFalse(getResBytes(ha, new byte[] {1, 2, 3, 4}, new byte[] {1, 2, 3, 4})); + + // second aray is shorter + assertTrue(getResBytes(ha, new byte[] {1, 2, 2, 2}, new byte[] {1, 1, 2})); + assertTrue(getResBytes(ha, new byte[] {1, 1, 1, 2}, new byte[] {1, 1, 2})); + + // second array is longer + assertTrue(getResBytes(ha, new byte[] {1, 2}, new byte[] {1, 1, 1})); + assertFalse(getResBytes(ha, new byte[] {1, 1}, new byte[] {1, 1, 2})); + + // one is null + assertTrue(getResBytes(ha, new byte[] {1, 2, 3, 4}, null)); + assertTrue(getResBytes(ha, null, new byte[] {1, 2, 3, 4})); + assertTrue(getResBytes(ha, null, null)); + } + + /** */ + public void testStringTruncate() throws Exception { + long[] sizes = new long[CPUS]; + + for (int i = 0; i < sizes.length; i++) + sizes[i] = 1024 * MB / CPUS; + + PageMemory pageMem = new PageMemoryNoStoreImpl(new JavaLogger(), + new UnsafeMemoryProvider(sizes), + null, + PAGE_SIZE, + false); + + pageMem.start(); + Page page = null; + + try { + FullPageId fullId = new FullPageId(pageMem.allocatePage(CACHE_ID, 1, PageIdAllocator.FLAG_DATA), CACHE_ID); + page = pageMem.page(fullId.cacheId(), fullId.pageId()); + long pageAddr = page.getForReadPointer(); + + int off = 0; + + InlineIndexHelper ih = new InlineIndexHelper(Value.STRING, 1, 0); + ih.put(pageAddr, off, ValueString.get("aaaaaaa"), 3 + 5); + + assertFalse(ih.isValueFull(pageAddr, off)); + + assertEquals("aaaaa", ih.get(pageAddr, off, 3 + 5).getString()); + + ih.put(pageAddr, off, ValueString.get("aaa"), 3 + 5); + + assertTrue(ih.isValueFull(pageAddr, off)); + + assertEquals("aaa", ih.get(pageAddr, off, 3 + 5).getString()); + } + finally { + if (page != null) + pageMem.releasePage(page); + pageMem.stop(); + } + } + + /** */ + public void testBytes() throws Exception { + long[] sizes = new long[CPUS]; + + for (int i = 0; i < sizes.length; i++) + sizes[i] = 1024 * MB / CPUS; + + PageMemory pageMem = new PageMemoryNoStoreImpl(new JavaLogger(), + new UnsafeMemoryProvider(sizes), + null, + PAGE_SIZE, + false); + + pageMem.start(); + Page page = null; + + try { + FullPageId fullId = new FullPageId(pageMem.allocatePage(CACHE_ID, 1, PageIdAllocator.FLAG_DATA), CACHE_ID); + page = pageMem.page(fullId.cacheId(), fullId.pageId()); + long pageAddr = page.getForReadPointer(); + + int off = 0; + + InlineIndexHelper ih = new InlineIndexHelper(Value.BYTES, 1, 0); + + ih.put(pageAddr, off, ValueBytes.get(new byte[] {1, 2, 3, 4, 5}), 3 + 3); + + assertFalse(ih.isValueFull(pageAddr, off)); + + assertTrue(Arrays.equals(new byte[] {1, 2, 3}, ih.get(pageAddr, off, 3 + 5).getBytes())); + + ih.put(pageAddr, off, ValueBytes.get(new byte[] {1, 2, 3, 4, 5}), 3 + 5); + + assertTrue(ih.isValueFull(pageAddr, off)); + + assertTrue(Arrays.equals(new byte[] {1, 2, 3, 4, 5}, ih.get(pageAddr, off, 3 + 5).getBytes())); + } + finally { + if (page != null) + pageMem.releasePage(page); + pageMem.stop(); + } + } + + /** */ + public void testNull() throws Exception { + testPutGet(ValueInt.get(-1), ValueNull.INSTANCE, ValueInt.get(3)); + } + + /** */ + public void testBoolean() throws Exception { + testPutGet(ValueBoolean.get(true), ValueBoolean.get(false), ValueBoolean.get(true)); + } + + /** */ + public void testByte() throws Exception { + testPutGet(ValueByte.get((byte)-1), ValueByte.get((byte)2), ValueByte.get((byte)3)); + } + + /** */ + public void testShort() throws Exception { + testPutGet(ValueShort.get((short)-32000), ValueShort.get((short)2), ValueShort.get((short)3)); + } + + /** */ + public void testInt() throws Exception { + testPutGet(ValueInt.get(-1), ValueInt.get(2), ValueInt.get(3)); + } + + /** */ + public void testLong() throws Exception { + testPutGet(ValueLong.get(-1), ValueLong.get(2), ValueLong.get(3)); + } + + /** */ + public void testFloat() throws Exception { + testPutGet(ValueFloat.get(1.1f), ValueFloat.get(2.2f), ValueFloat.get(1.1f)); + } + + /** */ + public void testDouble() throws Exception { + testPutGet(ValueDouble.get(1.1f), ValueDouble.get(2.2f), ValueDouble.get(1.1f)); + } + + /** */ + public void testDate() throws Exception { + testPutGet(ValueDate.get(Date.valueOf("2017-02-20")), + ValueDate.get(Date.valueOf("2017-02-21")), + ValueDate.get(Date.valueOf("2017-02-19"))); + } + + /** */ + public void testTime() throws Exception { + testPutGet(ValueTime.get(Time.valueOf("10:01:01")), + ValueTime.get(Time.valueOf("11:02:02")), + ValueTime.get(Time.valueOf("12:03:03"))); + } + + /** */ + public void testTimestamp() throws Exception { + testPutGet(ValueTimestamp.get(Timestamp.valueOf("2017-02-20 10:01:01")), + ValueTimestamp.get(Timestamp.valueOf("2017-02-20 10:01:01")), + ValueTimestamp.get(Timestamp.valueOf("2017-02-20 10:01:01"))); + } + + /** */ + public void testTimestampUTC() throws Exception { + testPutGet(ValueTimestampUtc.fromMillis(System.currentTimeMillis()), + ValueTimestampUtc.fromMillis(System.currentTimeMillis() + 100), + ValueTimestampUtc.fromMillis(System.currentTimeMillis() + 200)); + } + + /** */ + public void testUUID() throws Exception { + testPutGet(ValueUuid.get(UUID.randomUUID().toString()), + ValueUuid.get(UUID.randomUUID().toString()), + ValueUuid.get(UUID.randomUUID().toString())); + } + + /** */ + private void testPutGet(Value v1, Value v2, Value v3) throws Exception { + long[] sizes = new long[CPUS]; + + for (int i = 0; i < sizes.length; i++) + sizes[i] = 1024 * MB / CPUS; + + PageMemory pageMem = new PageMemoryNoStoreImpl(new JavaLogger(), + new UnsafeMemoryProvider(sizes), + null, + PAGE_SIZE, + false); + + pageMem.start(); + Page page = null; + + try { + FullPageId fullId = new FullPageId(pageMem.allocatePage(CACHE_ID, 1, PageIdAllocator.FLAG_DATA), CACHE_ID); + page = pageMem.page(fullId.cacheId(), fullId.pageId()); + long pageAddr = page.getForReadPointer(); + + int off = 0; + int max = 255; + + InlineIndexHelper ih = new InlineIndexHelper(v1.getType(), 1, 0); + + off += ih.put(pageAddr, off, v1, max - off); + off += ih.put(pageAddr, off, v2, max - off); + off += ih.put(pageAddr, off, v3, max - off); + + Value v11 = ih.get(pageAddr, 0, max); + Value v22 = ih.get(pageAddr, ih.fullSize(pageAddr, 0), max); + + assertEquals(v1.getObject(), v11.getObject()); + assertEquals(v2.getObject(), v22.getObject()); + } + finally { + if (page != null) + pageMem.releasePage(page); + pageMem.stop(); + } + } + + /** */ + private boolean getRes(InlineIndexHelper ha, String s1, String s2) { + Value v1 = s1 == null ? ValueNull.INSTANCE : ValueString.get(s1); + Value v2 = s2 == null ? ValueNull.INSTANCE : ValueString.get(s2); + + int c = v1.compareTypeSafe(v2, CompareMode.getInstance(CompareMode.DEFAULT, 0)); + return ha.canRelyOnCompare(c, v1, v2); + } + + /** */ + private boolean getResBytes(InlineIndexHelper ha, byte[] b1, byte[] b2) { + Value v1 = b1 == null ? ValueNull.INSTANCE : ValueBytes.get(b1); + Value v2 = b2 == null ? ValueNull.INSTANCE : ValueBytes.get(b2); + + int c = v1.compareTypeSafe(v2, CompareMode.getInstance(CompareMode.DEFAULT, 0)); + return ha.canRelyOnCompare(c, v1, v2); + } + } \ No newline at end of file