lucene-solr-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Erick Erickson <erickerick...@gmail.com>
Subject Re: NullPointerException on openStreams
Date Fri, 14 Jul 2017 16:54:29 GMT
Joel:

Would it make sense to throw a more informative error when the stream
context wasn't set? Maybe an explicit check in open() or some such?

Erick

On Fri, Jul 14, 2017 at 8:25 AM, Joe Obernberger
<joseph.obernberger@gmail.com> wrote:
> Still stuck on this one.  I suspect there is something I'm not setting in
> the StreamContext.  I'm not sure what to put for these two?
> context.put("core", this.coreName);
> context.put("solr-core", req.getCore());
>
> Also not sure what the class is for ClassifyStream?  Error that I'm getting
> is:
>
> java.io.IOException: java.lang.IndexOutOfBoundsException: Index: 0, Size: 0
>         at
> org.apache.solr.client.solrj.io.stream.CloudSolrStream.constructStreams(CloudSolrStream.java:408)
>         at
> org.apache.solr.client.solrj.io.stream.CloudSolrStream.open(CloudSolrStream.java:299)
>         at
> com.ngc.bigdata.ie_machinelearningprofile.MachineLearningProfileProcessor.profile(MachineLearningProfileProcessor.java:344)
>         at
> com.ngc.bigdata.ie_machinelearningprofile.ProfileThread.run(ProfileThread.java:41)
>         at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>         at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>         at java.lang.Thread.run(Thread.java:748)
> Caused by: java.lang.IndexOutOfBoundsException: Index: 0, Size: 0
>         at java.util.ArrayList.rangeCheck(ArrayList.java:653)
>         at java.util.ArrayList.get(ArrayList.java:429)
>         at
> org.apache.solr.client.solrj.io.stream.TupleStream.getShards(TupleStream.java:133)
>         at
> org.apache.solr.client.solrj.io.stream.CloudSolrStream.constructStreams(CloudSolrStream.java:393)
>
> Thanks for any ideas!
>
> -Joe
>
>
>
> On 7/13/2017 4:33 PM, Joe Obernberger wrote:
>>
>> Thanks for this.  I'm now trying to use stream for classify, but am
>> getting an ArrayIndexOutOfBounds error on the stream.open().  I'm setting
>> the streamFactory up, and including .withFunctionName("classify",
>> ClassifyStream.class) - but is that class in orga.apache.solr.handler?
>>
>> -----
>> StringBuilder expression = new StringBuilder();
>>             solrCollection = getCollectionFromProfileBean(pBean);
>>
>> expression.append("classify(model(models,id=\"").append(pBean.getModelID()).append("\",cacheMillis=5000),");
>>
>> expression.append("search(").append(solrCollection).append(",q=\"DocumentId:").append(docID).append("\",");
>>             expression.append("fl=\"ClusterText,id\",sort=\"id
>> asc\"),field=\"ClusterText\")");
>>             logger.info("Have classify expression:\n" +
>> expression.toString() + "\n");
>>             params.set("expr", expression.toString());
>>             params.set("qt", "/stream");
>>             params.set("explain", "true");
>>             params.set("q", "*:*");
>>             params.set("fl", "id");
>>             params.set("sort", "id asc");
>>
>>             context = new StreamContext();
>>
>> context.setSolrClientCache(StaticInfo.getSingleton(props).getClientCache());
>>             context.workerID = 0;
>>             context.numWorkers = 1;
>> context.setModelCache(StaticInfo.getSingleton(props).getModelCache());
>>
>>             streamFactory.withCollectionZkHost(solrCollection,
>> props.getProperty("hbase.zookeeper.solr.quorum"))
>>                     .withFunctionName("search", CloudSolrStream.class)
>>                     .withFunctionName("facet", FacetStream.class)
>>                     .withFunctionName("update", UpdateStream.class)
>>                     .withFunctionName("jdbc", JDBCStream.class)
>>                     .withFunctionName("topic", TopicStream.class)
>>                     .withFunctionName("commit", CommitStream.class)
>>                     // decorator streams
>>                     .withFunctionName("merge", MergeStream.class)
>>                     .withFunctionName("unique", UniqueStream.class)
>>                     .withFunctionName("top", RankStream.class)
>>                     .withFunctionName("reduce", ReducerStream.class)
>>                     .withFunctionName("parallel", ParallelStream.class)
>>                     .withFunctionName("rollup", RollupStream.class)
>>                     .withFunctionName("stats", StatsStream.class)
>>                     .withFunctionName("innerJoin", InnerJoinStream.class)
>>                     .withFunctionName("leftOuterJoin",
>> LeftOuterJoinStream.class)
>>                     .withFunctionName("hashJoin", HashJoinStream.class)
>>                     .withFunctionName("outerHashJoin",
>> OuterHashJoinStream.class)
>>                     .withFunctionName("intersect", IntersectStream.class)
>>                     .withFunctionName("complement",
>> ComplementStream.class)
>>                     .withFunctionName(SORT, SortStream.class)
>>                     .withFunctionName("train", TextLogitStream.class)
>>                     .withFunctionName("features",
>> FeaturesSelectionStream.class)
>>                     .withFunctionName("daemon", DaemonStream.class)
>>                     .withFunctionName("shortestPath",
>> ShortestPathStream.class)
>>                     .withFunctionName("gatherNodes",
>> GatherNodesStream.class)
>>                     .withFunctionName("nodes", GatherNodesStream.class)
>>                     .withFunctionName("select", SelectStream.class)
>>                     .withFunctionName("shortestPath",
>> ShortestPathStream.class)
>>                     .withFunctionName("gatherNodes",
>> GatherNodesStream.class)
>>                     .withFunctionName("nodes", GatherNodesStream.class)
>>                     .withFunctionName("scoreNodes",
>> ScoreNodesStream.class)
>>                     .withFunctionName("model", ModelStream.class)
>>                     .withFunctionName("classify", ClassifyStream.class)
>>                     .withFunctionName("fetch", FetchStream.class)
>>                     .withFunctionName("executor", ExecutorStream.class)
>>                     .withFunctionName("null", NullStream.class)
>>                     .withFunctionName("priority", PriorityStream.class)
>>                     .withFunctionName("significantTerms",
>> SignificantTermsStream.class)
>>                     .withFunctionName("cartesianProduct",
>> CartesianProductStream.class)
>>                     .withFunctionName("shuffle", ShuffleStream.class)
>>                     .withFunctionName("calc", CalculatorStream.class)
>>                     .withFunctionName("eval", EvalStream.class)
>>                     .withFunctionName("echo", EchoStream.class)
>>                     .withFunctionName("cell", CellStream.class)
>>                     .withFunctionName("list", ListStream.class)
>>                     .withFunctionName("let", LetStream.class)
>>                     .withFunctionName("get", GetStream.class)
>>                     .withFunctionName("timeseries",
>> TimeSeriesStream.class)
>>                     .withFunctionName("tuple", TupStream.class)
>>                     // metrics
>>                     .withFunctionName("min", MinMetric.class)
>>                     .withFunctionName("max", MaxMetric.class)
>>                     .withFunctionName("avg", MeanMetric.class)
>>                     .withFunctionName("sum", SumMetric.class)
>>                     .withFunctionName("count", CountMetric.class)
>>                     // tuple manipulation operations
>>                     .withFunctionName("replace", ReplaceOperation.class)
>>                     .withFunctionName("concat", ConcatOperation.class)
>>                     // stream reduction operations
>>                     .withFunctionName("group", GroupOperation.class)
>>                     .withFunctionName("distinct", DistinctOperation.class)
>>                     .withFunctionName("having", HavingStream.class)
>>                     // Stream Evaluators
>>                     .withFunctionName("val", RawValueEvaluator.class)
>>                     // Boolean Stream Evaluators
>>                     .withFunctionName("and", AndEvaluator.class)
>>                     .withFunctionName("eor", ExclusiveOrEvaluator.class)
>>                     .withFunctionName("eq", EqualsEvaluator.class)
>>                     .withFunctionName("gt", GreaterThanEvaluator.class)
>>                     .withFunctionName("gteq",
>> GreaterThanEqualToEvaluator.class)
>>                     .withFunctionName("lt", LessThanEvaluator.class)
>>                     .withFunctionName("lteq",
>> LessThanEqualToEvaluator.class)
>>                     .withFunctionName("not", NotEvaluator.class)
>>                     .withFunctionName("or", OrEvaluator.class)
>>                     // Date Time Evaluators
>> .withFunctionName(TemporalEvaluatorYear.FUNCTION_NAME,
>> TemporalEvaluatorYear.class)
>> .withFunctionName(TemporalEvaluatorMonth.FUNCTION_NAME,
>> TemporalEvaluatorMonth.class)
>> .withFunctionName(TemporalEvaluatorDay.FUNCTION_NAME,
>> TemporalEvaluatorDay.class)
>> .withFunctionName(TemporalEvaluatorDayOfYear.FUNCTION_NAME,
>> TemporalEvaluatorDayOfYear.class)
>> .withFunctionName(TemporalEvaluatorHour.FUNCTION_NAME,
>> TemporalEvaluatorHour.class)
>> .withFunctionName(TemporalEvaluatorMinute.FUNCTION_NAME,
>> TemporalEvaluatorMinute.class)
>> .withFunctionName(TemporalEvaluatorSecond.FUNCTION_NAME,
>> TemporalEvaluatorSecond.class)
>> .withFunctionName(TemporalEvaluatorEpoch.FUNCTION_NAME,
>> TemporalEvaluatorEpoch.class)
>> .withFunctionName(TemporalEvaluatorWeek.FUNCTION_NAME,
>> TemporalEvaluatorWeek.class)
>> .withFunctionName(TemporalEvaluatorQuarter.FUNCTION_NAME,
>> TemporalEvaluatorQuarter.class)
>> .withFunctionName(TemporalEvaluatorDayOfQuarter.FUNCTION_NAME,
>> TemporalEvaluatorDayOfQuarter.class)
>>                     // Number Stream Evaluators
>>                     .withFunctionName("abs", AbsoluteValueEvaluator.class)
>>                     .withFunctionName("add", AddEvaluator.class)
>>                     .withFunctionName("div", DivideEvaluator.class)
>>                     .withFunctionName("mult", MultiplyEvaluator.class)
>>                     .withFunctionName("sub", SubtractEvaluator.class)
>>                     .withFunctionName("log", NaturalLogEvaluator.class)
>>                     .withFunctionName("pow", PowerEvaluator.class)
>>                     .withFunctionName("mod", ModuloEvaluator.class)
>>                     .withFunctionName("ceil", CeilingEvaluator.class)
>>                     .withFunctionName("floor", FloorEvaluator.class)
>>                     .withFunctionName("sin", SineEvaluator.class)
>>                     .withFunctionName("asin", ArcSineEvaluator.class)
>>                     .withFunctionName("sinh",
>> HyperbolicSineEvaluator.class)
>>                     .withFunctionName("cos", CosineEvaluator.class)
>>                     .withFunctionName("acos", ArcCosineEvaluator.class)
>>                     .withFunctionName("cosh",
>> HyperbolicCosineEvaluator.class)
>>                     .withFunctionName("tan", TangentEvaluator.class)
>>                     .withFunctionName("atan", ArcTangentEvaluator.class)
>>                     .withFunctionName("tanh",
>> HyperbolicTangentEvaluator.class)
>>                     .withFunctionName("round", RoundEvaluator.class)
>>                     .withFunctionName("sqrt", SquareRootEvaluator.class)
>>                     .withFunctionName("cbrt", CubedRootEvaluator.class)
>>                     .withFunctionName("coalesce", CoalesceEvaluator.class)
>>                     .withFunctionName("uuid", UuidEvaluator.class)
>>                     .withFunctionName("corr", CorrelationEvaluator.class)
>>                     // Conditional Stream Evaluators
>>                     .withFunctionName("if", IfThenElseEvaluator.class)
>>                     .withFunctionName("convert",
>> ConversionEvaluator.class);
>>             context.setStreamFactory(streamFactory);
>>             try {
>>                 stream = new
>> CloudSolrStream(props.getProperty("hbase.zookeeper.solr.quorum"),
>> solrCollection, params);
>>                 stream.setStreamContext(context);
>>
>>                 stream.open();
>>
>> On 7/13/2017 2:43 PM, Susheel Kumar wrote:
>>>
>>> This the working code snippet I have, if that helps
>>>
>>> public static void main(String []args) throws IOException
>>> {
>>> String clause;
>>>     TupleStream stream;
>>>     List<Tuple> tuples;
>>>     StreamContext streamContext = new StreamContext();
>>>     SolrClientCache solrClientCache = new SolrClientCache();
>>>     streamContext.setSolrClientCache(solrClientCache);
>>>
>>>     StreamFactory factory = new StreamFactory()
>>>       .withCollectionZkHost("gettingstarted",
>>> "server1:2182, server2:2182,server3:2182/solr66")
>>>    //  .withCollectionZkHost("gettingstarted", "localhost:2181")
>>>      .withFunctionName("search", CloudSolrStream.class)
>>>       .withFunctionName("select", SelectStream.class)
>>>       .withFunctionName("add", AddEvaluator.class)
>>>       .withFunctionName("if", IfThenElseEvaluator.class)
>>>       .withFunctionName("gt", GreaterThanEvaluator.class)
>>>       .withFunctionName("let", LetStream.class)
>>>       .withFunctionName("get", GetStream.class)
>>>       .withFunctionName("echo", EchoStream.class)
>>>       .withFunctionName("merge", MergeStream.class)
>>>       .withFunctionName("sort", SortStream.class)
>>>       .withFunctionName("tuple", TupStream.class)
>>>       .withFunctionName("rollup",RollupStream.class)
>>>       .withFunctionName("hashJoin", HashJoinStream.class)
>>>       .withFunctionName("complement", ComplementStream.class)
>>>       .withFunctionName("fetch", FetchStream.class)
>>>       .withFunctionName("having",HavingStream.class)
>>>       .withFunctionName("eq", EqualsEvaluator.class)
>>>       .withFunctionName("count", CountMetric.class)
>>>       .withFunctionName("facet", FacetStream.class)
>>>       .withFunctionName("sum", SumMetric.class)
>>>       .withFunctionName("unique", UniqueStream.class)
>>>       .withFunctionName("uniq", UniqueMetric.class)
>>>       .withFunctionName("innerJoin", InnerJoinStream.class)
>>>       .withFunctionName("intersect", IntersectStream.class)
>>>
>>>       ;
>>>     try {
>>>      clause = getClause2();
>>>   //   clause = getFacet();
>>>       stream = factory.constructStream(clause);
>>>       stream.setStreamContext(streamContext);
>>>       tuples = getTuples(stream);
>>>
>>>       for(Tuple tuple : tuples )
>>>       {
>>>       System.out.println(tuple.getString("id"));
>>> System.out.println(tuple.getString("sr_sv_business_email_s"));
>>>       System.out.println(tuple.getString("sum(price_i)"));
>>>       System.out.println(tuple.getString("count(price_i)"));
>>>       System.out.println(tuple.getString("unique(price_i)"));
>>>       System.out.println(tuple.getString("email"));
>>>       }
>>>
>>>       System.out.println("Total tuples retunred "+tuples.size());
>>>     } finally {
>>>         solrClientCache.close();
>>>       }
>>>
>>> On Thu, Jul 13, 2017 at 2:18 PM, Joel Bernstein <joelsolr@gmail.com>
>>> wrote:
>>>
>>>> It's most likely that you're not setting the StreamContext. New versions
>>>> of
>>>> Solr expect the StreamContext to be set before the stream is opened. The
>>>> SolrClientCache also needs to present in the StreamContext. You can take
>>>> a
>>>> look at how the StreamHandler does this for an example:
>>>> https://github.com/apache/lucene-solr/blob/master/solr/
>>>> core/src/java/org/apache/solr/handler/StreamHandler.java#L339
>>>>
>>>> Joel Bernstein
>>>> http://joelsolr.blogspot.com/
>>>>
>>>> On Thu, Jul 13, 2017 at 2:06 PM, Joe Obernberger <
>>>> joseph.obernberger@gmail.com> wrote:
>>>>
>>>>> Hi All - trying to call ClouderSolrStream.open(), but I'm getting this
>>>>> error:
>>>>>
>>>>> java.io.IOException: java.lang.NullPointerException
>>>>>       at org.apache.solr.client.solrj.io.stream.CloudSolrStream.const
>>>>> ructStreams(CloudSolrStream.java:408)
>>>>>       at org.apache.solr.client.solrj.io.stream.CloudSolrStream.open(
>>>>> CloudSolrStream.java:299)
>>>>>
>>>>> I'm passing in a valid zkHost, collection name, and parameters. In
>>>>> fact,
>>>>> if I take the stream expression and past it into the GUI, it works OK.
>>>>
>>>> I'm
>>>>>
>>>>> stumped by what could be null here.
>>>>>
>>>>> My code looks like the following, and I'm getting the error on
>>>>> stream.open().
>>>>>
>>>>>          StringBuilder expression = new StringBuilder();
>>>>>          expression.append("update(models, batchSize=\"50\",");
>>>>> expression.append("train(").append(solrCollectionName).append(",");
>>>>> expression.append("features(").append(solrCollectionName).append(",");
>>>>> expression.append("q=\"*:*\",featureSet=\"FSet_").append(sol
>>>>> rCollectionName).append("\",");
>>>>> expression.append("field=\"Text\",outcome=\"out_i\",positive
>>>>> Label=1,numTerms=").append(numTerms).append("),");
>>>>> expression.append("q=\"*:*\",name=\"").append(docID).append(
>>>>> "\",field=\"Text\",outcome=\"out_i\",maxIterations=\"").
>>>>> append(maxIterations).append("\"))");
>>>>>          logger.info("Have update expression:\n"+expression.toSt
>>>>> ring()+"\n");
>>>>>          params.set("expr", expression.toString());
>>>>>          params.set("qt", "/stream");
>>>>>          params.set("explain", "true");
>>>>>          params.set("q", "*:*");
>>>>>          params.set("fl", "id");
>>>>>          params.set("sort", "id asc");
>>>>>          try {
>>>>>              System.out.println("Open: "+props.getProperty("hbase.zoo
>>>>> keeper.solr.quorum")+"\nCollection: "+solrCollectionName+" \nWith
>>>>
>>>> params:
>>>>>
>>>>> "+params);
>>>>>              stream = new CloudSolrStream(props.getPrope
>>>>> rty("hbase.zookeeper.solr.quorum"), solrCollectionName, params);
>>>>>              stream.open();
>>>>>              while (true) {
>>>>>                  Tuple tuple = stream.read();
>>>>>                  logger.info("Tuple Read: "+tuple.fields.toString());
>>>>>                  if (tuple.EOF) {
>>>>>                      break;
>>>>>                  }
>>>>>              }
>>>>>          } catch (IOException ex) {
>>>>>              logger.error("Solr stream error: " + ex);
>>>>>              ex.printStackTrace();
>>>>>          } finally {
>>>>>              if (stream != null) {
>>>>>                  try {
>>>>>                      stream.close();
>>>>>                  } catch (IOException ex) {
>>>>>                      logger.error("Could not close stream: "+ex);
>>>>>                  }
>>>>>              }
>>>>>          }
>>>>>
>>>>> I'm stuck!  Thanks!
>>>>>
>>>>> -Joe
>>>>>
>>>>>
>>>
>>> ---
>>> This email has been checked for viruses by AVG.
>>> http://www.avg.com
>>>
>>
>

Mime
View raw message