asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From buyin...@apache.org
Subject [41/51] [partial] incubator-asterixdb git commit: SQL++ parser: 1. refactored asterix-aql to become asterix-lang-common and asterix-lang-aql, where the former is the common part for different languages; 2. added asterix-lang-sqlpp on top of asterix-lang-
Date Fri, 30 Oct 2015 23:16:49 GMT
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/olist-fuzzyeq-jaccard.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/olist-fuzzyeq-jaccard.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/olist-fuzzyeq-jaccard.sqlpp
new file mode 100644
index 0000000..40df5f9
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/olist-fuzzyeq-jaccard.sqlpp
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether a keyword is applied to optimize a selection query with ~= using Jaccard on lists.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+  number : int32,
+  street : string,
+  city : string
+}
+
+create type test.CustomerType as
+ closed {
+  cid : int32,
+  name : string,
+  age : int32?,
+  address : AddressType?,
+  interests : [string],
+  children : [{
+          name : string,
+          age : int32?
+      }
+]
+}
+
+create  table Customers(CustomerType) primary key cid;
+
+create  index interests_index  on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-basic_olist-fuzzyeq-jaccard.adm"
+set "simfunction" "jaccard";
+
+set "simthreshold" "0.8f";
+
+select element c
+from  Customers as c
+where (c.interests ~= ['databases','computers','wine'])
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/olist-jaccard-check.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/olist-jaccard-check.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/olist-jaccard-check.sqlpp
new file mode 100644
index 0000000..b30a924
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/olist-jaccard-check.sqlpp
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether a keyword index is applied to optimize a selection query using the similarity-jaccard-check function on lists.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+  number : int32,
+  street : string,
+  city : string
+}
+
+create type test.CustomerType as
+ closed {
+  cid : int32,
+  name : string,
+  age : int32?,
+  address : AddressType?,
+  interests : [string],
+  children : [{
+          name : string,
+          age : int32?
+      }
+]
+}
+
+create  table Customers(CustomerType) primary key cid;
+
+create  index interests_index  on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-basic_olist-jaccard-check.adm"
+select element c
+from  Customers as c
+where test."similarity-jaccard-check"(c.interests,['databases','computers','wine'],0.700000f)[0]
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/olist-jaccard.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/olist-jaccard.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/olist-jaccard.sqlpp
new file mode 100644
index 0000000..635face
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/olist-jaccard.sqlpp
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether a keyword index is applied to optimize a selection query using the similarity-jaccard function on lists.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+  number : int32,
+  street : string,
+  city : string
+}
+
+create type test.CustomerType as
+ closed {
+  cid : int32,
+  name : string,
+  age : int32?,
+  address : AddressType?,
+  interests : [string],
+  children : [{
+          name : string,
+          age : int32?
+      }
+]
+}
+
+create  table Customers(CustomerType) primary key cid;
+
+create  index interests_index  on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-basic_olist-jaccard.adm"
+select element c
+from  Customers as c
+where (test."similarity-jaccard"(c.interests,['databases','computers','wine']) >= 0.700000f)
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/ulist-fuzzyeq-jaccard.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/ulist-fuzzyeq-jaccard.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/ulist-fuzzyeq-jaccard.sqlpp
new file mode 100644
index 0000000..f56d978
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/ulist-fuzzyeq-jaccard.sqlpp
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether a keyword is applied to optimize a selection query with ~= using Jaccard on sets.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+  number : int32,
+  street : string,
+  city : string
+}
+
+create type test.CustomerType as
+ closed {
+  cid : int32,
+  name : string,
+  age : int32?,
+  address : AddressType?,
+  interests : {{string}},
+  children : [{
+          name : string,
+          age : int32?
+      }
+]
+}
+
+create  table Customers(CustomerType) primary key cid;
+
+create  index interests_index  on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-basic_ulist-fuzzyeq-jaccard.adm"
+set "simfunction" "jaccard";
+
+set "simthreshold" "0.8f";
+
+select element c
+from  Customers as c
+where (c.interests ~= {{'computers','wine','databases'}})
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/ulist-jaccard-check.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/ulist-jaccard-check.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/ulist-jaccard-check.sqlpp
new file mode 100644
index 0000000..266b18a
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/ulist-jaccard-check.sqlpp
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether a keyword index is applied to optimize a selection query using the similarity-jaccard-check function on sets.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+  number : int32,
+  street : string,
+  city : string
+}
+
+create type test.CustomerType as
+ closed {
+  cid : int32,
+  name : string,
+  age : int32?,
+  address : AddressType?,
+  interests : {{string}},
+  children : [{
+          name : string,
+          age : int32?
+      }
+]
+}
+
+create  table Customers(CustomerType) primary key cid;
+
+create  index interests_index  on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-basic_ulist-jaccard.adm"
+select element c
+from  Customers as c
+where test."similarity-jaccard-check"(c.interests,{{'computers','wine','databases'}},0.700000f)[0]
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/ulist-jaccard.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/ulist-jaccard.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/ulist-jaccard.sqlpp
new file mode 100644
index 0000000..1e3c845
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/ulist-jaccard.sqlpp
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether a keyword index is applied to optimize a selection query using the similarity-jaccard function on sets.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+  number : int32,
+  street : string,
+  city : string
+}
+
+create type test.CustomerType as
+ closed {
+  cid : int32,
+  name : string,
+  age : int32?,
+  address : AddressType?,
+  interests : {{string}},
+  children : [{
+          name : string,
+          age : int32?
+      }
+]
+}
+
+create  table Customers(CustomerType) primary key cid;
+
+create  index interests_index  on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-basic_ulist-jaccard.adm"
+select element c
+from  Customers as c
+where (test."similarity-jaccard"(c.interests,{{'computers','databases','wine'}}) >= 0.700000f)
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/word-contains.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/word-contains.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/word-contains.sqlpp
new file mode 100644
index 0000000..41f8234
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/word-contains.sqlpp
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether a keyword index is applied to optimize a selection query using the contains function.
+ *                  The index should *not* be applied (see below).
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  index keyword_index  on DBLP (title) type keyword;
+
+write output to nc1:"rttest/inverted-index-basic_word-contains.adm"
+select element o
+from  DBLP as o
+where test.contains(o.title,'Multimedia')
+order by o.id
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/word-fuzzyeq-jaccard.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/word-fuzzyeq-jaccard.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/word-fuzzyeq-jaccard.sqlpp
new file mode 100644
index 0000000..5ff613d
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/word-fuzzyeq-jaccard.sqlpp
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether a keyword is applied to optimize a selection query with ~= using Jaccard on word tokens.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  index keyword_index  on DBLP (title) type keyword;
+
+write output to nc1:"rttest/inverted-index-basic_word-fuzzyeq-jaccard.adm"
+set "simfunction" "jaccard";
+
+set "simthreshold" "0.5f";
+
+select element o
+from  DBLP as o
+where (test."word-tokens"(o.title) ~= test."word-tokens"('Transactions for Cooperative Environments'))
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/word-jaccard-check.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/word-jaccard-check.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/word-jaccard-check.sqlpp
new file mode 100644
index 0000000..7317c59
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/word-jaccard-check.sqlpp
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether a keyword index is applied to optimize a selection query using the similarity-jaccard-check function on word tokens.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  index keyword_index  on DBLP (title) type keyword;
+
+write output to nc1:"rttest/inverted-index-basic_word-jaccard-check.adm"
+select element o
+from  DBLP as o
+where test."similarity-jaccard-check"(test."word-tokens"(o.title),test."word-tokens"('Transactions for Cooperative Environments'),0.500000f)[0]
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/word-jaccard.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/word-jaccard.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/word-jaccard.sqlpp
new file mode 100644
index 0000000..faee18b
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-basic/word-jaccard.sqlpp
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether a keyword index is applied to optimize a selection query using the similarity-jaccard function on word tokens.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  index keyword_index  on DBLP (title) type keyword;
+
+write output to nc1:"rttest/inverted-index-basic_word-jaccard.adm"
+select element o
+from  DBLP as o
+where (test."similarity-jaccard"(test."word-tokens"(o.title),test."word-tokens"('Transactions for Cooperative Environments')) >= 0.500000f)
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-let-panic-nopanic_01.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-let-panic-nopanic_01.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-let-panic-nopanic_01.sqlpp
new file mode 100644
index 0000000..754e614
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-let-panic-nopanic_01.sqlpp
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether an ngram_index is applied to optimize a selection query using 
+ *                  two edit-distance-check function of which only the first can be optimized with an index. 
+ *                  Tests that the optimizer rule correctly drills through the let clauses.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  index ngram_index  on DBLP (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-complex_ngram-edit-distance-check-let-panic-nopanic_01.adm"
+select element o
+from  DBLP as o
+with  eda as test."edit-distance-check"(o.authors,'Amihay Motro',3),
+      edb as test."edit-distance-check"(o.authors,'Amihay Motro',5)
+where (eda[0] and edb[0])
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-let-panic-nopanic_02.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-let-panic-nopanic_02.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-let-panic-nopanic_02.sqlpp
new file mode 100644
index 0000000..dacf18b
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-let-panic-nopanic_02.sqlpp
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether an ngram_index is applied to optimize a selection query using 
+ *                  two edit-distance-check function of which only the second can be optimized with an index.
+ *                  Tests that the optimizer rule correctly drills through the let clauses.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  index ngram_index  on DBLP (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-complex_ngram-edit-distance-check-let-panic-nopanic_01.adm"
+select element o
+from  DBLP as o
+with  edb as test."edit-distance-check"(o.authors,'Amihay Motro',5),
+      eda as test."edit-distance-check"(o.authors,'Amihay Motro',3)
+where (edb[0] and eda[0])
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-let-panic.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-let-panic.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-let-panic.sqlpp
new file mode 100644
index 0000000..31b1e23
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-let-panic.sqlpp
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether an ngram_index is applied to optimize a selection query using the edit-distance-check function on strings.
+ *                  Tests that the optimizer rule correctly drills through the let clauses.
+ *                  The index should *not* be applied (see below).
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  index ngram_index  on DBLP (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-complex_ngram-edit-distance-check-let-panic.adm"
+select element o
+from  DBLP as o
+with  ed as test."edit-distance-check"(o.authors,'Amihay Motro',5)
+where ed[0]
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-let.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-let.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-let.sqlpp
new file mode 100644
index 0000000..61925e7
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-let.sqlpp
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether an ngram_index is applied to optimize a selection query using the edit-distance-check function on strings.
+ *                  Tests that the optimizer rule correctly drills through the let clauses.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  index ngram_index  on DBLP (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-complex_ngram-edit-distance-check-let.adm"
+select element o
+from  DBLP as o
+with  ed as test."edit-distance-check"(o.authors,'Amihay Motro',1)
+where ed[0]
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-substring.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-substring.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-substring.sqlpp
new file mode 100644
index 0000000..cddd14c
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-substring.sqlpp
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether an ngram_index index is applied to optimize a selection query using the similarity-edit-distance-check function on the substring of the field.
+ *                  Tests that the optimizer rule correctly drills through the substring function.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  index ngram_index  on DBLP (title) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-complex_ngram-edit-distance-check-substring.adm"
+select element {'id':paper.id,'title':paper.title}
+from  DBLP as paper
+where test."edit-distance-check"(test.substring(paper.title,0,8),'datbase',1)[0]
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-word-tokens.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-word-tokens.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-word-tokens.sqlpp
new file mode 100644
index 0000000..72dbd3b
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-edit-distance-check-word-tokens.sqlpp
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether an ngram_index index is applied to optimize a selection query using the similarity-edit-distance-check function on individual word tokens.
+ *                  Tests that the optimizer rule correctly drills through the word-tokens function and existential query.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  index ngram_index  on DBLP (title) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-complex_ngram-edit-distance-check-word-tokens.adm"
+select element {'id':paper.id,'title':paper.title}
+from  DBLP as paper,
+      test."word-tokens"(paper.title) as word
+where test."edit-distance-check"(word,'Multmedia',1)[0]
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-jaccard-check-let.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-jaccard-check-let.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-jaccard-check-let.sqlpp
new file mode 100644
index 0000000..32ebdc6
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-jaccard-check-let.sqlpp
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether an ngram_index is applied to optimize a selection query using the similarity-jaccard-check function on 3-gram tokens.
+ *                  Tests that the optimizer rule correctly drills through the let clauses.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+set "import-private-functions" "true";
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  index ngram_index  on DBLP (title) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-complex_ngram-jaccard-check-let.adm"
+select element o
+from  DBLP as o
+with  jacc as test."similarity-jaccard-check"(test."gram-tokens"(o.title,3,false),test."gram-tokens"('Transactions for Cooperative Environments',3,false),0.500000f)
+where jacc[0]
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-jaccard-check-multi-let.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-jaccard-check-multi-let.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-jaccard-check-multi-let.sqlpp
new file mode 100644
index 0000000..f5886b3
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ngram-jaccard-check-multi-let.sqlpp
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether an ngram_index is applied to optimize a selection query using the similarity-jaccard-check function on 3-gram tokens.
+ *                  Tests that the optimizer rule correctly drills through the let clauses.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+set "import-private-functions" "true";
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  index ngram_index  on DBLP (title) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-complex_ngram-jaccard-check-multi-let.adm"
+select element {'Paper':paper_tokens,'Query':query_tokens}
+from  DBLP as paper
+with  paper_tokens as test."gram-tokens"(paper.title,3,false),
+      query_tokens as test."gram-tokens"('Transactions for Cooperative Environments',3,false),
+      jacc as test."similarity-jaccard-check"(paper_tokens,query_tokens,0.500000f)
+where jacc[0]
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/olist-edit-distance-check-let-panic.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/olist-edit-distance-check-let-panic.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/olist-edit-distance-check-let-panic.sqlpp
new file mode 100644
index 0000000..4be9f47
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/olist-edit-distance-check-let-panic.sqlpp
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether an ngram_index is applied to optimize a selection query using the edit-distance-check function on lists.
+ *                  Tests that the optimizer rule correctly drills through the let clauses.
+ *                  The index should *not* be applied (see below).
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+  number : int32,
+  street : string,
+  city : string
+}
+
+create type test.CustomerType as
+ closed {
+  cid : int32,
+  name : string,
+  age : int32?,
+  address : AddressType?,
+  interests : [string],
+  children : [{
+          name : string,
+          age : int32?
+      }
+]
+}
+
+create  table Customers(CustomerType) primary key cid;
+
+create  index interests_index  on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-complex_olist-edit-distance-check-let-panic.adm"
+select element c
+from  Customers as c
+with  ed as test."edit-distance-check"(c.interests,['computers','wine','walking'],3)
+where ed[0]
+order by c.cid
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/olist-edit-distance-check-let.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/olist-edit-distance-check-let.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/olist-edit-distance-check-let.sqlpp
new file mode 100644
index 0000000..f6c4ba1
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/olist-edit-distance-check-let.sqlpp
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether an ngram_index is applied to optimize a selection query using the edit-distance-check function on lists.
+ *                  Tests that the optimizer rule correctly drills through the let clauses.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+  number : int32,
+  street : string,
+  city : string
+}
+
+create type test.CustomerType as
+ closed {
+  cid : int32,
+  name : string,
+  age : int32?,
+  address : AddressType?,
+  interests : [string],
+  children : [{
+          name : string,
+          age : int32?
+      }
+]
+}
+
+create  table Customers(CustomerType) primary key cid;
+
+create  index interests_index  on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-complex_olist-edit-distance-check-let.adm"
+select element c
+from  Customers as c
+with  ed as test."edit-distance-check"(c.interests,['computers','wine','walking'],1)
+where ed[0]
+order by c.cid
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/olist-jaccard-check-let.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/olist-jaccard-check-let.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/olist-jaccard-check-let.sqlpp
new file mode 100644
index 0000000..5dbb332
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/olist-jaccard-check-let.sqlpp
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether a keyword index is applied to optimize a selection query using the similarity-jaccard-check function on lists.
+ *                  Tests that the optimizer rule correctly drills through the let clauses.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+  number : int32,
+  street : string,
+  city : string
+}
+
+create type test.CustomerType as
+ closed {
+  cid : int32,
+  name : string,
+  age : int32?,
+  address : AddressType?,
+  interests : [string],
+  children : [{
+          name : string,
+          age : int32?
+      }
+]
+}
+
+create  table Customers(CustomerType) primary key cid;
+
+create  index interests_index  on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-complex_olist-jaccard-check-let.adm"
+select element c
+from  Customers as c
+with  jacc as test."similarity-jaccard-check"(c.interests,['databases','computers','wine'],0.700000f)
+where jacc[0]
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ulist-jaccard-check-let.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ulist-jaccard-check-let.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ulist-jaccard-check-let.sqlpp
new file mode 100644
index 0000000..05eaf9b
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/ulist-jaccard-check-let.sqlpp
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether a keyword index is applied to optimize a selection query using the similarity-jaccard-check function on lists.
+ *                  Tests that the optimizer rule correctly drills through the let clauses.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+  number : int32,
+  street : string,
+  city : string
+}
+
+create type test.CustomerType as
+ closed {
+  cid : int32,
+  name : string,
+  age : int32?,
+  address : AddressType?,
+  interests : {{string}},
+  children : [{
+          name : string,
+          age : int32?
+      }
+]
+}
+
+create  table Customers(CustomerType) primary key cid;
+
+create  index interests_index  on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-complex_ulist-jaccard-check-let.adm"
+select element c
+from  Customers as c
+with  jacc as test."similarity-jaccard-check"(c.interests,['databases','computers','wine'],0.700000f)
+where jacc[0]
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/word-jaccard-check-let.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/word-jaccard-check-let.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/word-jaccard-check-let.sqlpp
new file mode 100644
index 0000000..9458c4c
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/word-jaccard-check-let.sqlpp
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether a keyword index is applied to optimize a selection query using the similarity-jaccard-check function on word tokens.
+ *                  Tests that the optimizer rule correctly drills through the let clauses.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  index keyword_index  on DBLP (title) type keyword;
+
+write output to nc1:"rttest/inverted-index-complex_word-jaccard-check-let.adm"
+select element o
+from  DBLP as o
+with  jacc as test."similarity-jaccard-check"(test."word-tokens"(o.title),test."word-tokens"('Transactions for Cooperative Environments'),0.500000f)
+where jacc[0]
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/word-jaccard-check-multi-let.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/word-jaccard-check-multi-let.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/word-jaccard-check-multi-let.sqlpp
new file mode 100644
index 0000000..a1e3a82
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-complex/word-jaccard-check-multi-let.sqlpp
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Tests whether a keyword index is applied to optimize a selection query using the similarity-jaccard-check function on word tokens.
+ *                  Tests that the optimizer rule correctly drills through the let clauses.
+ *                  The index should be applied.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  index keyword_index  on DBLP (title) type keyword;
+
+write output to nc1:"rttest/inverted-index-complex_word-jaccard-check-multi-let.adm"
+select element {'Paper':paper_tokens,'Query':query_tokens}
+from  DBLP as paper
+with  paper_tokens as test."word-tokens"(paper.title),
+      query_tokens as test."word-tokens"('Transactions for Cooperative Environments'),
+      jacc as test."similarity-jaccard-check"(paper_tokens,query_tokens,0.800000f)
+where jacc[0]
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-edit-distance-inline.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-edit-distance-inline.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-edit-distance-inline.sqlpp
new file mode 100644
index 0000000..7f1951e
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-edit-distance-inline.sqlpp
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Fuzzy self joins a dataset, DBLP, based on the edit-distance function of its authors.
+ *                  DBLP has a 3-gram index on authors, and we expect the join to be transformed into an indexed nested-loop join.
+ *                  We test the inlining of variables that enable the select to be pushed into the join for subsequent optimization with an index.
+ *                  We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  index ngram_index  on DBLP (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_ngram-edit-distance-inline.adm"
+select element {'aauthors':a.authors,'bauthors':b.authors,'ed':ed}
+from  DBLP as a,
+      DBLP as b
+with  ed as test."edit-distance"(a.authors,b.authors)
+where ((ed < 3) and (a.id < b.id))
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-edit-distance.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-edit-distance.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-edit-distance.sqlpp
new file mode 100644
index 0000000..82492f0
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-edit-distance.sqlpp
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Fuzzy joins two datasets, DBLP and CSX, based on the edit-distance function of their authors.
+ *                  DBLP has a 3-gram index on authors, and we expect the join to be transformed into an indexed nested-loop join.
+ *                  We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary. 
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create type test.CSXType as
+ closed {
+  id : int32,
+  csxid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  table CSX(CSXType) primary key id;
+
+create  index ngram_index  on DBLP (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_ngram-edit-distance.adm"
+select element {'aauthors':a.authors,'bauthors':b.authors}
+from  DBLP as a,
+      CSX as b
+where ((test."edit-distance"(a.authors,b.authors) < 3) and (a.id < b.id))
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-fuzzyeq-edit-distance.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-fuzzyeq-edit-distance.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-fuzzyeq-edit-distance.sqlpp
new file mode 100644
index 0000000..e6d57c1
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-fuzzyeq-edit-distance.sqlpp
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Fuzzy joins two datasets, DBLP and CSX, based on ~= using edit distance of their authors.
+ *                  DBLP has a 3-gram index on authors, and we expect the join to be transformed into an indexed nested-loop join.
+ *                  We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary. 
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create type test.CSXType as
+ closed {
+  id : int32,
+  csxid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  table CSX(CSXType) primary key id;
+
+create  index ngram_index  on CSX (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_ngram-fuzzyeq-edit-distance.adm"
+set "simfunction" "edit-distance";
+
+set "simthreshold" "3";
+
+select element {'aauthors':a.authors,'bauthors':b.authors}
+from  DBLP as a,
+      CSX as b
+where ((a.authors ~= b.authors) and (a.id < b.id))
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-fuzzyeq-jaccard.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-fuzzyeq-jaccard.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-fuzzyeq-jaccard.sqlpp
new file mode 100644
index 0000000..6a4e7b4
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-fuzzyeq-jaccard.sqlpp
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard of their titles' 3-gram tokens.
+ *                  DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
+ *                  We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+set "import-private-functions" "true";
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create type test.CSXType as
+ closed {
+  id : int32,
+  csxid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  table CSX(CSXType) primary key id;
+
+create  index ngram_index  on DBLP (title) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_ngram-fuzzyeq-jaccard.adm"
+set "simfunction" "jaccard";
+
+set "simthreshold" "0.5f";
+
+select element {'atitle':a.title,'btitle':b.title}
+from  DBLP as a,
+      CSX as b
+where ((test."gram-tokens"(a.title,3,false) ~= test."gram-tokens"(b.title,3,false)) and (a.id < b.id))
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-jaccard-inline.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-jaccard-inline.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-jaccard-inline.sqlpp
new file mode 100644
index 0000000..4bdcb05
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-jaccard-inline.sqlpp
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Fuzzy self joins a dataset, DBLP, based on the similarity-jaccard function of its titles' 3-gram tokens.
+ *                  DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
+ *                  We test the inlining of variables that enable the select to be pushed into the join for subsequent optimization with an index.
+ *                  We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+set "import-private-functions" "true";
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  index ngram_index  on DBLP (title) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_ngram-jaccard-inline.adm"
+select element {'atitle':a.title,'btitle':b.title,'jacc':jacc}
+from  DBLP as a,
+      DBLP as b
+with  jacc as test."similarity-jaccard"(test."gram-tokens"(a.title,3,false),test."gram-tokens"(b.title,3,false))
+where ((jacc >= 0.500000f) and (a.id < b.id))
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-jaccard.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-jaccard.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-jaccard.sqlpp
new file mode 100644
index 0000000..0981ab2
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ngram-jaccard.sqlpp
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard function of their titles' 3-gram tokens.
+ *                  DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
+ *                  We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+set "import-private-functions" "true";
+
+create type test.DBLPType as
+ closed {
+  id : int32,
+  dblpid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create type test.CSXType as
+ closed {
+  id : int32,
+  csxid : string,
+  title : string,
+  authors : string,
+  misc : string
+}
+
+create  table DBLP(DBLPType) primary key id;
+
+create  table CSX(CSXType) primary key id;
+
+create  index ngram_index  on DBLP (title) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_ngram-jaccard.adm"
+select element {'atitle':a.title,'btitle':b.title}
+from  DBLP as a,
+      CSX as b
+where ((test."similarity-jaccard"(test."gram-tokens"(a.title,3,false),test."gram-tokens"(b.title,3,false)) >= 0.500000f) and (a.id < b.id))
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-edit-distance-inline.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-edit-distance-inline.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-edit-distance-inline.sqlpp
new file mode 100644
index 0000000..456eba1
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-edit-distance-inline.sqlpp
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Fuzzy self joins a dataset, Customers, based on the edit-distance function of its interest lists.
+ *                  Customers has a keyword index on interests, and we expect the join to be transformed into an indexed nested-loop join.
+ *                  We test the inlining of variables that enable the select to be pushed into the join for subsequent optimization with an index.
+ *                  We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+  number : int32,
+  street : string,
+  city : string
+}
+
+create type test.CustomerType as
+ closed {
+  cid : int32,
+  name : string,
+  age : int32?,
+  address : AddressType?,
+  interests : [string],
+  children : [{
+          name : string,
+          age : int32?
+      }
+]
+}
+
+create  table Customers(CustomerType) primary key cid;
+
+create  index interests_index  on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_olist-edit-distance-inline.adm"
+select element {'ainterests':a.interests,'binterests':b.interests,'ed':ed}
+from  Customers as a,
+      Customers as b
+with  ed as test."edit-distance"(a.interests,b.interests)
+where ((ed <= 2) and (a.cid < b.cid))
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-edit-distance.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-edit-distance.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-edit-distance.sqlpp
new file mode 100644
index 0000000..89a9f07
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-edit-distance.sqlpp
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Fuzzy joins two datasets, Customer and Customer2, based on the edit-distance function of their interest lists.
+ *                  Customers has a keyword index on interests, and we expect the join to be transformed into an indexed nested-loop join.
+ *                  We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+  number : int32,
+  street : string,
+  city : string
+}
+
+create type test.CustomerType as
+ closed {
+  cid : int32,
+  name : string,
+  age : int32?,
+  address : AddressType?,
+  interests : [string],
+  children : [{
+          name : string,
+          age : int32?
+      }
+]
+}
+
+create  table Customers(CustomerType) primary key cid;
+
+create  table Customers2(CustomerType) primary key cid;
+
+create  index interests_index  on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_olist-edit-distance.adm"
+select element {'ainterests':a.interests,'binterests':b.interests}
+from  Customers as a,
+      Customers2 as b
+where ((test."edit-distance"(a.interests,b.interests) <= 2) and (a.cid < b.cid))
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-fuzzyeq-edit-distance.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-fuzzyeq-edit-distance.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-fuzzyeq-edit-distance.sqlpp
new file mode 100644
index 0000000..16ddaf7
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-fuzzyeq-edit-distance.sqlpp
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Fuzzy joins two datasets, Customer and Customer2, based on ~= using edit distance of their interest lists.
+ *                  Customers has a keyword index on interests, and we expect the join to be transformed into an indexed nested-loop join.
+ *                  We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+  number : int32,
+  street : string,
+  city : string
+}
+
+create type test.CustomerType as
+ closed {
+  cid : int32,
+  name : string,
+  age : int32?,
+  address : AddressType?,
+  interests : [string],
+  children : [{
+          name : string,
+          age : int32?
+      }
+]
+}
+
+create  table Customers(CustomerType) primary key cid;
+
+create  table Customers2(CustomerType) primary key cid;
+
+create  index interests_index  on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_olist-fuzzyeq-edit-distance.adm"
+set "simfunction" "edit-distance";
+
+set "simthreshold" "3";
+
+select element {'ainterests':a.interests,'binterests':b.interests}
+from  Customers as a,
+      Customers2 as b
+where ((a.interests ~= b.interests) and (a.cid < b.cid))
+;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-fuzzyeq-jaccard.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-fuzzyeq-jaccard.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-fuzzyeq-jaccard.sqlpp
new file mode 100644
index 0000000..d5f9ad8
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-fuzzyeq-jaccard.sqlpp
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Fuzzy joins two datasets, Customer and Customer2, based on ~= using Jaccard of their interest lists.
+ *                  Customers has a keyword index on interests, and we expect the join to be transformed into an indexed nested-loop join.
+ *                  We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
+ * Success        : Yes
+ */
+
+drop  database test if exists;
+create  database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+  number : int32,
+  street : string,
+  city : string
+}
+
+create type test.CustomerType as
+ closed {
+  cid : int32,
+  name : string,
+  age : int32?,
+  address : AddressType?,
+  interests : [string],
+  children : [{
+          name : string,
+          age : int32?
+      }
+]
+}
+
+create  table Customers(CustomerType) primary key cid;
+
+create  table Customers2(CustomerType) primary key cid;
+
+create  index interests_index  on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_olist-fuzzyeq-jaccard.adm"
+set "simfunction" "jaccard";
+
+set "simthreshold" "0.7f";
+
+select element {'ainterests':a.interests,'binterests':b.interests}
+from  Customers as a,
+      Customers2 as b
+where ((a.interests /*+ indexnl */  ~= b.interests) and (a.cid < b.cid))
+;


Mime
View raw message