Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id A1136200D23 for ; Thu, 19 Oct 2017 10:30:56 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 9F844160BEE; Thu, 19 Oct 2017 08:30:56 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id EAA76160BEF for ; Thu, 19 Oct 2017 10:30:54 +0200 (CEST) Received: (qmail 77694 invoked by uid 500); 19 Oct 2017 08:30:52 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 74782 invoked by uid 99); 19 Oct 2017 08:30:50 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 19 Oct 2017 08:30:50 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1DFF8DFF77; Thu, 19 Oct 2017 08:30:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ab@apache.org To: commits@lucene.apache.org Date: Thu, 19 Oct 2017 08:31:07 -0000 Message-Id: <1d7875a8c6224e79a16c031a01cbb2e0@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [19/50] [abbrv] lucene-solr:jira/solr-11072: SOLR-11145, SOLR-11146: Added comprehensive unit tests for Analytics Component 2.0 as well as analytics bug fixes. archived-at: Thu, 19 Oct 2017 08:30:56 -0000 http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86d84bff/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/expression/LegacyExpressionTest.java ---------------------------------------------------------------------- diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/expression/LegacyExpressionTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/expression/LegacyExpressionTest.java new file mode 100644 index 0000000..89fe2c7 --- /dev/null +++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/expression/LegacyExpressionTest.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.analytics.legacy.expression; + +import java.time.Instant; +import java.util.Date; + +import org.apache.solr.analytics.legacy.LegacyAbstractAnalyticsTest; +import org.apache.solr.util.DateMathParser; +import org.junit.BeforeClass; +import org.junit.Test; + +public class LegacyExpressionTest extends LegacyAbstractAnalyticsTest { + private static final String fileName = "expressions.txt"; + + private static final int INT = 71; + private static final int LONG = 36; + private static final int FLOAT = 93; + private static final int DOUBLE = 49; + private static final int DATE = 12; + private static final int STRING = 28; + private static final int NUM_LOOPS = 100; + + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-analytics.xml", "schema-analytics.xml"); + h.update("*:*"); + + for (int j = 0; j < NUM_LOOPS; ++j) { + int i = j % INT; + long l = j % LONG; + float f = j % FLOAT; + double d = j % DOUBLE; + String dt = (1800 + j % DATE) + "-12-31T23:59:59Z"; + String s = "str" + (j % STRING); + assertU(adoc("id", "1000" + j, "int_id", "" + i, "long_ld", "" + l, "float_fd", "" + f, + "double_dd", "" + d, "date_dtd", dt, "string_sd", s)); + + if (usually()) { + assertU(commit()); // to have several segments + } + } + + assertU(commit()); + + setResponse(h.query(request(fileToStringArr(LegacyExpressionTest.class, fileName)))); + } + + @Test + public void addTest() throws Exception { + double sumResult = (Double) getStatResult("ar", "sum", VAL_TYPE.DOUBLE); + double uniqueResult = ((Long) getStatResult("ar", "unique", VAL_TYPE.LONG)).doubleValue(); + double result = (Double) getStatResult("ar", "su", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), sumResult + uniqueResult, result, 0.0); + + double meanResult = (Double) getStatResult("ar", "mean", VAL_TYPE.DOUBLE); + double medianResult = (Double) getStatResult("ar", "median", VAL_TYPE.DOUBLE); + double countResult = ((Long) getStatResult("ar", "count", VAL_TYPE.LONG)).doubleValue(); + result = (Double) getStatResult("ar", "mcm", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), meanResult + countResult + medianResult, result, 0.0); + } + + @Test + public void multiplyTest() throws Exception { + double sumResult = (Double) getStatResult("mr", "sum", VAL_TYPE.DOUBLE); + double uniqueResult = ((Long) getStatResult("mr", "unique", VAL_TYPE.LONG)).doubleValue(); + double result = (Double) getStatResult("mr", "su", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), sumResult * uniqueResult, result, 0.0); + + double meanResult = (Double) getStatResult("mr", "mean", VAL_TYPE.DOUBLE); + double medianResult = (Double) getStatResult("mr", "median", VAL_TYPE.DOUBLE); + double countResult = ((Long) getStatResult("mr", "count", VAL_TYPE.LONG)).doubleValue(); + result = (Double) getStatResult("mr", "mcm", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), meanResult * countResult * medianResult, result, 0.0); + } + + @Test + public void divideTest() throws Exception { + double sumResult = (Double) getStatResult("dr", "sum", VAL_TYPE.DOUBLE); + double uniqueResult = ((Long) getStatResult("dr", "unique", VAL_TYPE.LONG)).doubleValue(); + double result = (Double) getStatResult("dr", "su", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), sumResult / uniqueResult, result, 0.0); + + double meanResult = (Double) getStatResult("dr", "mean", VAL_TYPE.DOUBLE); + double countResult = ((Long) getStatResult("dr", "count", VAL_TYPE.LONG)).doubleValue(); + result = (Double) getStatResult("dr", "mc", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), meanResult / countResult, result, 0.0); + } + + @Test + public void powerTest() throws Exception { + double sumResult = (Double) getStatResult("pr", "sum", VAL_TYPE.DOUBLE); + double uniqueResult = ((Long) getStatResult("pr", "unique", VAL_TYPE.LONG)).doubleValue(); + double result = (Double) getStatResult("pr", "su", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), Math.pow(sumResult, uniqueResult), result, 0.0); + + double meanResult = (Double) getStatResult("pr", "mean", VAL_TYPE.DOUBLE); + double countResult = ((Long) getStatResult("pr", "count", VAL_TYPE.LONG)).doubleValue(); + result = (Double) getStatResult("pr", "mc", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), Math.pow(meanResult, countResult), result, 0.0); + } + + @Test + public void negateTest() throws Exception { + double sumResult = (Double) getStatResult("nr", "sum", VAL_TYPE.DOUBLE); + double result = (Double) getStatResult("nr", "s", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), -1 * sumResult, result, 0.0); + + long countResult = ((Long) getStatResult("nr", "count", VAL_TYPE.LONG)); + long lresult = (Long) getStatResult("nr", "c", VAL_TYPE.LONG); + assertEquals(getRawResponse(), -1 * countResult, lresult, 0.0); + } + + @Test + public void absoluteValueTest() throws Exception { + double sumResult = (Double) getStatResult("avr", "sum", VAL_TYPE.DOUBLE); + double result = (Double) getStatResult("avr", "s", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), sumResult, result, 0.0); + + long countResult = ((Long) getStatResult("avr", "count", VAL_TYPE.LONG)); + long lresult = (Long) getStatResult("avr", "c", VAL_TYPE.LONG); + assertEquals(getRawResponse(), countResult, lresult, 0.0); + } + + @Test + public void constantNumberTest() throws Exception { + int result = (Integer) getStatResult("cnr", "c8", VAL_TYPE.INTEGER); + assertEquals(getRawResponse(), 8, result, 0.0); + + double dresult = (Double) getStatResult("cnr", "c10", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), 10.0, dresult, 0.0); + } + + @Test + public void dateMathTest() throws Exception { + String math = (String) getStatResult("dmr", "cme", VAL_TYPE.STRING); + DateMathParser dateMathParser = new DateMathParser(); + dateMathParser.setNow(new Date(Instant.parse((String) getStatResult("dmr", "median", VAL_TYPE.DATE)).toEpochMilli())); + String dateMath = (String) getStatResult("dmr", "dmme", VAL_TYPE.DATE); + assertEquals(getRawResponse(), new Date(Instant.parse(dateMath).toEpochMilli()), dateMathParser.parseMath(math)); + + math = (String) getStatResult("dmr", "cma", VAL_TYPE.STRING); + dateMathParser = new DateMathParser(); + dateMathParser.setNow(new Date(Instant.parse((String) getStatResult("dmr", "max", VAL_TYPE.DATE)).toEpochMilli())); + dateMath = (String) getStatResult("dmr", "dmma", VAL_TYPE.DATE); + assertEquals(getRawResponse(), new Date(Instant.parse(dateMath).toEpochMilli()), dateMathParser.parseMath(math)); + } + + @Test + public void constantDateTest() throws Exception { + String date = (String) getStatResult("cdr", "cd1", VAL_TYPE.DATE); + String str = (String) getStatResult("cdr", "cs1", VAL_TYPE.STRING); + assertEquals(getRawResponse(), date, str); + + date = (String) getStatResult("cdr", "cd2", VAL_TYPE.DATE); + str = (String) getStatResult("cdr", "cs2", VAL_TYPE.STRING); + assertEquals(getRawResponse(), date, str); + } + + @Test + public void constantStringTest() throws Exception { + String str = (String) getStatResult("csr", "cs1", VAL_TYPE.STRING); + assertEquals(getRawResponse(), str, "this is the first"); + + str = (String) getStatResult("csr", "cs2", VAL_TYPE.STRING); + assertEquals(getRawResponse(), str, "this is the second"); + + str = (String) getStatResult("csr", "cs3", VAL_TYPE.STRING); + assertEquals(getRawResponse(), str, "this is the third"); + } + + @Test + public void concatenateTest() throws Exception { + StringBuilder builder = new StringBuilder(); + builder.append((String) getStatResult("cr", "csmin", VAL_TYPE.STRING)); + builder.append((String) getStatResult("cr", "min", VAL_TYPE.STRING)); + String concat = (String) getStatResult("cr", "ccmin", VAL_TYPE.STRING); + assertEquals(getRawResponse(), concat, builder.toString()); + + builder.setLength(0); + builder.append((String) getStatResult("cr", "csmax", VAL_TYPE.STRING)); + builder.append((String) getStatResult("cr", "max", VAL_TYPE.STRING)); + concat = (String) getStatResult("cr", "ccmax", VAL_TYPE.STRING); + assertEquals(getRawResponse(), concat, builder.toString()); + } +} http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86d84bff/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/expression/LegacyFunctionTest.java ---------------------------------------------------------------------- diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/expression/LegacyFunctionTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/expression/LegacyFunctionTest.java new file mode 100644 index 0000000..85c1333 --- /dev/null +++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/expression/LegacyFunctionTest.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.analytics.legacy.expression; + + +import org.apache.solr.analytics.legacy.LegacyAbstractAnalyticsTest; +import org.apache.solr.analytics.legacy.facet.LegacyAbstractAnalyticsFacetTest; +import org.junit.BeforeClass; +import org.junit.Test; + +public class LegacyFunctionTest extends LegacyAbstractAnalyticsTest { + static String fileName = "functions.txt"; + + static public final int INT = 71; + static public final int LONG = 36; + static public final int FLOAT = 93; + static public final int DOUBLE = 49; + static public final int DATE = 12; + static public final int STRING = 28; + static public final int NUM_LOOPS = 100; + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-analytics.xml","schema-analytics.xml"); + h.update("*:*"); + + for (int j = 0; j < NUM_LOOPS; ++j) { + int i = j%INT+1; + long l = j%LONG+1; + float f = j%FLOAT+1; + double d = j%DOUBLE+1; + double d0 = j%DOUBLE; + String dt = (1800+j%DATE) + "-06-30T23:59:59Z"; + String s = "str" + (j%STRING); + + double add_if = (double)i+f; + double add_ldf = (double)l+d+f; + double mult_if = (double)i*f; + double mult_ldf = (double)l*d*f; + double div_if = (double)i/f; + double div_ld = (double)l/d; + double pow_if = Math.pow(i,f); + double pow_ld = Math.pow(l,d); + int neg_i = i*-1; + long neg_l = l*-1; + String dm_2y = (1802+j%DATE) + "-06-30T23:59:59Z"; + String dm_2m = (1800+j%DATE) + "-08-30T23:59:59Z"; + String concat_first = "this is the first"+s; + String concat_second = "this is the second"+s; + + assertU(adoc(LegacyAbstractAnalyticsFacetTest.filter("id", "1000" + j, "int_id", "" + i, "long_ld", "" + l, "float_fd", "" + f, + "double_dd", "" + d, "date_dtd", dt, "string_sd", s, + "add_if_dd", ""+add_if, "add_ldf_dd", ""+add_ldf, "mult_if_dd", ""+mult_if, "mult_ldf_dd", ""+mult_ldf, + "div_if_dd", ""+div_if, "div_ld_dd", ""+div_ld, "pow_if_dd", ""+pow_if, "pow_ld_dd", ""+pow_ld, + "neg_id", ""+neg_i, "neg_ld", ""+neg_l, "const_8_dd", "8", "const_10_dd", "10", "dm_2y_dtd", dm_2y, "dm_2m_dtd", dm_2m, + "const_00_dtd", "1800-06-30T23:59:59Z", "const_04_dtd", "1804-06-30T23:59:59Z", "const_first_sd", "this is the first", "const_second_sd", "this is the second", + "concat_first_sd", concat_first, "concat_second_sd", concat_second, "miss_dd", ""+d0 ))); + + + if (usually()) { + assertU(commit()); // to have several segments + } + } + + assertU(commit()); + + setResponse(h.query(request(fileToStringArr(LegacyFunctionTest.class, fileName)))); + } + + @Test + public void addTest() throws Exception { + double result = (Double)getStatResult("ar", "sum", VAL_TYPE.DOUBLE); + double calculated = (Double)getStatResult("ar", "sumc", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), result, calculated, 0.0); + // TODO checfk why asserted 2times + assertEquals(getRawResponse(), result, calculated, 0.0); + + result = (Double)getStatResult("ar", "mean", VAL_TYPE.DOUBLE); + calculated = (Double)getStatResult("ar", "meanc", VAL_TYPE.DOUBLE); + assertTrue(result==calculated); + assertEquals(getRawResponse(), result, calculated, 0.0); + } + + @Test + public void multiplyTest() throws Exception { + double result = (Double)getStatResult("mr", "sum", VAL_TYPE.DOUBLE); + double calculated = (Double)getStatResult("mr", "sumc", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), result, calculated, 0.0); + + result = (Double)getStatResult("mr", "mean", VAL_TYPE.DOUBLE); + calculated = (Double)getStatResult("mr", "meanc", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), result, calculated, 0.0); + } + + @Test + public void divideTest() throws Exception { + Double result = (Double)getStatResult("dr", "sum", VAL_TYPE.DOUBLE); + Double calculated = (Double)getStatResult("dr", "sumc", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), result, calculated, 0.0); + + result = (Double)getStatResult("dr", "mean", VAL_TYPE.DOUBLE); + calculated = (Double)getStatResult("dr", "meanc", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), result, calculated, 0.0); + } + + @Test + public void powerTest() throws Exception { + double result = (Double)getStatResult("pr", "sum", VAL_TYPE.DOUBLE); + double calculated = (Double)getStatResult("pr", "sumc", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), result, calculated, 0.0); + assertEquals(getRawResponse(), result, calculated, 0.0); + + result = (Double)getStatResult("pr", "mean", VAL_TYPE.DOUBLE); + calculated = (Double)getStatResult("pr", "meanc", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), result, calculated, 0.0); + assertEquals(getRawResponse(), result, calculated, 0.0); + } + + @Test + public void negateTest() throws Exception { + double result = (Double)getStatResult("nr", "sum", VAL_TYPE.DOUBLE); + double calculated = (Double)getStatResult("nr", "sumc", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), result, calculated, 0.0); + + result = (Double)getStatResult("nr", "mean", VAL_TYPE.DOUBLE); + calculated = (Double)getStatResult("nr", "meanc", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), result, calculated, 0.0); + } + + @Test + public void absoluteValueTest() throws Exception { + double result = (Double)getStatResult("avr", "sum", VAL_TYPE.DOUBLE); + double calculated = (Double)getStatResult("avr", "sumc", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), result, calculated, 0.0); + + result = (Double)getStatResult("avr", "mean", VAL_TYPE.DOUBLE); + calculated = (Double)getStatResult("avr", "meanc", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), result, calculated, 0.0); + } + + @Test + public void constantNumberTest() throws Exception { + double result = (Double)getStatResult("cnr", "sum", VAL_TYPE.DOUBLE); + double calculated = (Double)getStatResult("cnr", "sumc", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), result, calculated, 0.0); + assertEquals(getRawResponse(), result, calculated, 0.0); + + result = (Double)getStatResult("cnr", "mean", VAL_TYPE.DOUBLE); + calculated = (Double)getStatResult("cnr", "meanc", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), result, calculated, 0.0); + assertEquals(getRawResponse(), result, calculated, 0.0); + } + + @Test + public void dateMathTest() throws Exception { + String result = (String)getStatResult("dmr", "median", VAL_TYPE.DATE); + String calculated = (String)getStatResult("dmr", "medianc", VAL_TYPE.DATE); + assertEquals(getRawResponse(), result, calculated); + + result = (String)getStatResult("dmr", "max", VAL_TYPE.DATE); + calculated = (String)getStatResult("dmr", "maxc", VAL_TYPE.DATE); + assertEquals(getRawResponse(), result, calculated); + } + + @Test + public void constantDateTest() throws Exception { + String result = (String)getStatResult("cdr", "median", VAL_TYPE.DATE); + String calculated = (String)getStatResult("cdr", "medianc", VAL_TYPE.DATE); + assertEquals(getRawResponse(), result, calculated); + assertEquals(getRawResponse(), result, calculated); + + result = (String)getStatResult("cdr", "max", VAL_TYPE.DATE); + calculated = (String)getStatResult("cdr", "maxc", VAL_TYPE.DATE); + assertEquals(getRawResponse(), result, calculated); + } + + @Test + public void constantStringTest() throws Exception { + String result = (String)getStatResult("csr", "min", VAL_TYPE.STRING); + String calculated = (String)getStatResult("csr", "minc", VAL_TYPE.STRING); + assertEquals(getRawResponse(), result, calculated); + + result = (String)getStatResult("csr", "max", VAL_TYPE.STRING); + calculated = (String)getStatResult("csr", "maxc", VAL_TYPE.STRING); + assertEquals(getRawResponse(), result, calculated); + } + + @Test + public void concatenateTest() throws Exception { + String result = (String)getStatResult("cr", "min", VAL_TYPE.STRING); + String calculated = (String)getStatResult("cr", "minc", VAL_TYPE.STRING); + assertEquals(getRawResponse(), result, calculated); + + result = (String)getStatResult("cr", "max", VAL_TYPE.STRING); + calculated = (String)getStatResult("cr", "maxc", VAL_TYPE.STRING); + assertEquals(getRawResponse(), result, calculated); + } + + @Test + public void missingTest() throws Exception { + double min = (Double)getStatResult("ms", "min", VAL_TYPE.DOUBLE); + double max = (Double)getStatResult("ms", "max", VAL_TYPE.DOUBLE); + assertEquals(getRawResponse(), 48.0d, max, 0.0); + assertEquals(getRawResponse(), 1.0d, min, 0.0); + } + +} http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86d84bff/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyAbstractAnalyticsFacetCloudTest.java ---------------------------------------------------------------------- diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyAbstractAnalyticsFacetCloudTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyAbstractAnalyticsFacetCloudTest.java new file mode 100644 index 0000000..783177d --- /dev/null +++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyAbstractAnalyticsFacetCloudTest.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.analytics.legacy.facet; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +import org.apache.solr.analytics.legacy.LegacyAbstractAnalyticsCloudTest; +import org.apache.solr.analytics.util.AnalyticsResponseHeadings; +import org.apache.solr.analytics.util.MedianCalculator; +import org.apache.solr.analytics.util.OrdinalCalculator; +import org.apache.solr.common.util.NamedList; +import org.junit.AfterClass; + +public class LegacyAbstractAnalyticsFacetCloudTest extends LegacyAbstractAnalyticsCloudTest { + protected static final HashMap defaults = new HashMap<>(); + + protected String latestType = ""; + + @AfterClass + public static void afterClassAbstractAnalysis() { + defaults.clear(); + } + + @SuppressWarnings("unchecked") + protected ArrayList getValueList(NamedList response, String infoName, String facetType, String facetName, String exprName, boolean includeMissing) { + NamedList> facetList = + (NamedList>)response.findRecursive(AnalyticsResponseHeadings.COMPLETED_OLD_HEADER, + infoName, + facetType, + facetName); + + ArrayList results = new ArrayList<>(); + facetList.forEach( (name, expressions) -> { + if (!includeMissing && !name.equals("(MISSING)")) { + T result = (T)expressions.get(exprName); + if (result != null) + results.add(result); + } + }); + return results; + } + + protected boolean responseContainsFacetValue(NamedList response, String infoName, String facetType, String facetName, String facetValue) { + return null != response.findRecursive(AnalyticsResponseHeadings.COMPLETED_OLD_HEADER, + infoName, + facetType, + facetName, + facetValue); + } + + + public static void increment(List list, int idx){ + Long i = list.remove(idx); + list.add(idx, i+1); + } + + protected void setLatestType(String latestType) { + this.latestType = latestType; + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + public > ArrayList calculateFacetedNumberStat(ArrayList> lists, String stat) { + ArrayList result; + if (stat.equals("median")) { + result = new ArrayList(); + for (List list : lists) { + result.add(MedianCalculator.getMedian(list)); + } + } else if (stat.equals("mean")) { + result = new ArrayList(); + for (List list : lists) { + double d = 0; + for (T element : list) { + d += element.doubleValue(); + } + result.add(d/list.size()); + } + } else if (stat.equals("sum")) { + result = new ArrayList(); + for (Collection list : lists) { + double d = 0; + for (T element : list) { + d += element.doubleValue(); + } + result.add(d); + } + } else if (stat.equals("sumOfSquares")) { + result = new ArrayList(); + for (List list : lists) { + double d = 0; + for (T element : list) { + d += element.doubleValue()*element.doubleValue(); + } + result.add(d); + } + } else if (stat.equals("stddev")) { + result = new ArrayList(); + for (List list : lists) { + double sum = 0; + double sumSquares = 0; + for (T element : list) { + sum += element.doubleValue(); + sumSquares += element.doubleValue()*element.doubleValue(); + } + String res = Double.toString(Math.sqrt(sumSquares/list.size()-sum*sum/(list.size()*list.size()))); + result.add(Double.parseDouble(res)); + } + } else { + throw new IllegalArgumentException(); + } + return result; + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + public > ArrayList calculateFacetedStat(ArrayList> lists, String stat) { + ArrayList result; + if (stat.contains("perc_")) { + result = new ArrayList(); + for (List list : lists) { + if( list.size() == 0) continue; + int ord = (int) Math.ceil(Double.parseDouble(stat.substring(5))/100 * list.size()) - 1; + ArrayList percs = new ArrayList<>(1); + percs.add(ord); + OrdinalCalculator.putOrdinalsInPosition(list, percs); + result.add(list.get(ord)); + } + } else if (stat.equals("count")) { + result = new ArrayList(); + for (List list : lists) { + result.add((long)list.size()); + } + } else if (stat.equals("missing")) { + result = new ArrayList(); + for (ArrayList list : lists) { + result.add(calculateMissing(list,latestType)); + } + } else if (stat.equals("unique")) { + result = new ArrayList(); + for (List list : lists) { + HashSet set = new HashSet<>(); + set.addAll(list); + result.add((long)set.size()); + } + } else if (stat.equals("max")) { + result = new ArrayList(); + for (List list : lists) { + if( list.size() == 0) continue; + Collections.sort(list); + result.add(list.get(list.size()-1)); + } + } else if (stat.equals("min")) { + result = new ArrayList(); + for (List list : lists) { + if( list.size() == 0) continue; + Collections.sort((List)list); + result.add(list.get(0)); + } + } else { + result = null; + } + return result; + } + + @SuppressWarnings("unchecked") + public > Long calculateMissing(ArrayList list, String type) { + T def = (T)defaults.get(type); + long miss = 0; + for (T element : list) { + if (element.compareTo(def)==0) { + miss++; + } + } + return Long.valueOf(miss); + } +} http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86d84bff/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyAbstractAnalyticsFacetTest.java ---------------------------------------------------------------------- diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyAbstractAnalyticsFacetTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyAbstractAnalyticsFacetTest.java new file mode 100644 index 0000000..68c9826 --- /dev/null +++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyAbstractAnalyticsFacetTest.java @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.analytics.legacy.facet; + +import java.io.ByteArrayInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Scanner; + +import org.apache.lucene.util.IOUtils; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.analytics.util.AnalyticsResponseHeadings; +import org.apache.solr.analytics.util.MedianCalculator; +import org.apache.solr.analytics.util.OrdinalCalculator; +import org.apache.solr.request.SolrQueryRequest; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import com.google.common.collect.ObjectArrays; + +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; + +public class LegacyAbstractAnalyticsFacetTest extends SolrTestCaseJ4 { + protected static final HashMap defaults = new HashMap<>(); + + protected String latestType = ""; + + private static Document doc; + private static XPathFactory xPathFact; + private static String rawResponse; + + @BeforeClass + public static void beforeClassAbstractAnalysis() { + xPathFact = XPathFactory.newInstance(); + } + + @AfterClass + public static void afterClassAbstractAnalysis() { + xPathFact = null; + doc = null; + rawResponse = null; + defaults.clear(); + } + + protected static void setResponse(String response) throws ParserConfigurationException, IOException, SAXException { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(true); // never forget this! + DocumentBuilder builder = factory.newDocumentBuilder(); + doc = builder.parse(new InputSource(new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8)))); + rawResponse = response; + } + + protected String getRawResponse() { + return rawResponse; + } + + protected Node getNode(String xPath) throws XPathExpressionException { + return (Node)xPathFact.newXPath().compile(xPath).evaluate(doc, XPathConstants.NODE); + } + private NodeList getNodes(String n1, String n2, String n3, String element, String n4) throws XPathExpressionException { + // Construct the XPath expression. The form better not change or all these will fail. + StringBuilder sb = new StringBuilder("/response/lst[@name='"+AnalyticsResponseHeadings.COMPLETED_OLD_HEADER+"']/lst[@name='").append(n1).append("']"); + sb.append("/lst[@name='").append(n2).append("']"); + sb.append("/lst[@name='").append(n3).append("']"); + sb.append("/lst[@name!='(MISSING)']"); + sb.append("//").append(element).append("[@name='").append(n4).append("']"); + return (NodeList)xPathFact.newXPath().compile(sb.toString()).evaluate(doc, XPathConstants.NODESET); + + } + protected ArrayList getStringList(String n1, String n2, String n3, String element, String n4) + throws XPathExpressionException { + ArrayList ret = new ArrayList<>(); + NodeList nodes = getNodes(n1, n2, n3, element, n4); + for (int idx = 0; idx < nodes.getLength(); ++idx) { + ret.add(nodes.item(idx).getTextContent()); + } + return ret; + } + + protected ArrayList getIntegerList(String n1, String n2, String n3, String element, String n4) + throws XPathExpressionException { + ArrayList ret = new ArrayList<>(); + NodeList nodes = getNodes(n1, n2, n3, element, n4); + for (int idx = 0; idx < nodes.getLength(); ++idx) { + ret.add(Integer.parseInt(nodes.item(idx).getTextContent())); + } + return ret; + } + protected ArrayList getLongList(String n1, String n2, String n3, String element, String n4) + throws XPathExpressionException { + ArrayList ret = new ArrayList<>(); + NodeList nodes = getNodes(n1, n2, n3, element, n4); + for (int idx = 0; idx < nodes.getLength(); ++idx) { + ret.add(Long.parseLong(nodes.item(idx).getTextContent())); + } + return ret; + } + protected ArrayList getFloatList(String n1, String n2, String n3, String element, String n4) + throws XPathExpressionException { + ArrayList ret = new ArrayList<>(); + NodeList nodes = getNodes(n1, n2, n3, element, n4); + for (int idx = 0; idx < nodes.getLength(); ++idx) { + ret.add(Float.parseFloat(nodes.item(idx).getTextContent())); + } + return ret; + } + + protected ArrayList getDoubleList(String n1, String n2, String n3, String element, String n4) + throws XPathExpressionException { + ArrayList ret = new ArrayList<>(); + NodeList nodes = getNodes(n1, n2, n3, element, n4); + for (int idx = 0; idx < nodes.getLength(); ++idx) { + ret.add(Double.parseDouble(nodes.item(idx).getTextContent())); + } + return ret; + } + + + public static void increment(List list, int idx){ + Long i = list.remove(idx); + list.add(idx, i+1); + } + + public static String[] filter(String...args){ + List l = new ArrayList<>(); + for( int i=0; i > ArrayList calculateNumberStat(ArrayList> lists, String stat) { + ArrayList result; + if (stat.equals("median")) { + result = new ArrayList(); + for (List list : lists) { + result.add(MedianCalculator.getMedian(list)); + } + } else if (stat.equals("mean")) { + result = new ArrayList(); + for (List list : lists) { + double d = 0; + for (T element : list) { + d += element.doubleValue(); + } + result.add(d/list.size()); + } + } else if (stat.equals("sum")) { + result = new ArrayList(); + for (Collection list : lists) { + double d = 0; + for (T element : list) { + d += element.doubleValue(); + } + result.add(d); + } + } else if (stat.equals("sumOfSquares")) { + result = new ArrayList(); + for (List list : lists) { + double d = 0; + for (T element : list) { + d += element.doubleValue()*element.doubleValue(); + } + result.add(d); + } + } else if (stat.equals("stddev")) { + result = new ArrayList(); + for (List list : lists) { + double sum = 0; + double sumSquares = 0; + for (T element : list) { + sum += element.doubleValue(); + sumSquares += element.doubleValue()*element.doubleValue(); + } + String res = Double.toString(Math.sqrt(sumSquares/list.size()-sum*sum/(list.size()*list.size()))); + result.add(Double.parseDouble(res)); + } + } else { + throw new IllegalArgumentException(); + } + return result; + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + public > ArrayList calculateStat(ArrayList> lists, String stat) { + ArrayList result; + if (stat.contains("perc_")) { + result = new ArrayList(); + for (List list : lists) { + if( list.size() == 0) continue; + int ord = (int) Math.ceil(Double.parseDouble(stat.substring(5))/100 * list.size()) - 1; + ArrayList percs = new ArrayList<>(1); + percs.add(ord); + OrdinalCalculator.putOrdinalsInPosition(list, percs); + result.add(list.get(ord)); + } + } else if (stat.equals("count")) { + result = new ArrayList(); + for (List list : lists) { + //if( list.size() == 0) continue; + result.add((long)list.size()); + } + } else if (stat.equals("missing")) { + result = new ArrayList(); + for (ArrayList list : lists) { + if( list.size() == 0) continue; + result.add(calculateMissing(list,latestType)); + } + } else if (stat.equals("unique")) { + result = new ArrayList(); + for (List list : lists) { + HashSet set = new HashSet<>(); + set.addAll(list); + result.add((long)set.size()); + } + } else if (stat.equals("max")) { + result = new ArrayList(); + for (List list : lists) { + if( list.size() == 0) continue; + Collections.sort(list); + result.add(list.get(list.size()-1)); + } + } else if (stat.equals("min")) { + result = new ArrayList(); + for (List list : lists) { + if( list.size() == 0) continue; + Collections.sort((List)list); + result.add(list.get(0)); + } + } else { + result = null; + } + return result; + } + + @SuppressWarnings("unchecked") + public > Long calculateMissing(ArrayList list, String type) { + T def = (T)defaults.get(type); + long miss = 0; + for (T element : list) { + if (element.compareTo(def)==0) { + miss++; + } + } + return Long.valueOf(miss); + } + + public static SolrQueryRequest request(String...args){ + return SolrTestCaseJ4.req( ObjectArrays.concat(BASEPARMS, args,String.class) ); + } + + public static final String[] BASEPARMS = new String[]{ "q", "*:*", "indent", "true", "olap", "true", "rows", "0" }; + + + public static String[] fileToStringArr(Class clazz, String fileName) throws FileNotFoundException { + InputStream in = clazz.getResourceAsStream("/solr/analytics/legacy/" + fileName); + if (in == null) throw new FileNotFoundException("Resource not found: " + fileName); + Scanner file = new Scanner(in, "UTF-8"); + try { + ArrayList strList = new ArrayList<>(); + while (file.hasNextLine()) { + String line = file.nextLine(); + if (line.length()<2) { + continue; + } + int commentStart = line.indexOf("//"); + if (commentStart >= 0) { + line = line.substring(0,commentStart); + } + String[] param = line.split("="); + if (param.length != 2) { + continue; + } + strList.add(param[0]); + strList.add(param[1]); + } + return strList.toArray(new String[0]); + } finally { + IOUtils.closeWhileHandlingException(file, in); + } + } + + protected void removeNodes(String xPath, List string) throws XPathExpressionException { + NodeList missingNodes = getNodes(xPath); + List result = new ArrayList(); + for (int idx = 0; idx < missingNodes.getLength(); ++idx) { + result.add(Double.parseDouble(missingNodes.item(idx).getTextContent())); + } + string.removeAll(result); + } + + protected NodeList getNodes(String xPath) throws XPathExpressionException { + StringBuilder sb = new StringBuilder(xPath); + return (NodeList) xPathFact.newXPath().compile(sb.toString()).evaluate(doc, XPathConstants.NODESET); + } + +} http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86d84bff/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyFacetSortingTest.java ---------------------------------------------------------------------- diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyFacetSortingTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyFacetSortingTest.java new file mode 100644 index 0000000..d89c713 --- /dev/null +++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyFacetSortingTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.analytics.legacy.facet; + +import org.apache.solr.analytics.legacy.LegacyAbstractAnalyticsTest; +import org.apache.solr.analytics.legacy.expression.LegacyExpressionTest; +import org.junit.BeforeClass; +import org.junit.Test; + +public class LegacyFacetSortingTest extends LegacyAbstractAnalyticsTest { + private static String fileName = "facetSorting.txt"; + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-analytics.xml", "schema-analytics.xml"); + h.update("*:*"); + + // The data set below is so generated that in bucket corresponding fieldFacet B, double_dd column has null values + // and in bucket C corresponding to fieldFacet C has null values for column long_ld. + // FieldFaceting occurs on string_sd field + assertU(adoc("id", "1001", "string_sd", "A", "double_dd", "" + 3, "long_ld", "" + 1)); + assertU(adoc("id", "1002", "string_sd", "A", "double_dd", "" + 25, "long_ld", "" + 2)); + assertU(adoc("id", "1003", "string_sd", "B", "long_ld", "" + 3)); + assertU(adoc("id", "1004", "string_sd", "B", "long_ld", "" + 4)); + assertU(adoc("id", "1005", "string_sd", "C", "double_dd", "" + 17)); + + assertU(commit()); + String response = h.query(request(fileToStringArr(LegacyExpressionTest.class, fileName))); + setResponse(response); + } + + @Test + public void addTest() throws Exception { + Double minResult = (Double) getStatResult("ar", "min", VAL_TYPE.DOUBLE); + Long maxResult = (Long) getStatResult("ar", "max", VAL_TYPE.LONG); + assertEquals(Double.valueOf(minResult), Double.valueOf(3.0)); + assertEquals(Long.valueOf(maxResult),Long.valueOf(4)); + } +}