lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tflo...@apache.org
Subject [49/58] [abbrv] lucene-solr:jira/solr-10233: SOLR-10617: JDBCStream to support additional types, minor refactoring to separate out CalciteJDBCStream, upgrade hsqldb for JDBCStream & DIH tests.
Date Fri, 12 May 2017 23:39:16 GMT
SOLR-10617: JDBCStream to support additional types, minor refactoring to separate out CalciteJDBCStream,
upgrade hsqldb for JDBCStream & DIH tests.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e61b5b34
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e61b5b34
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e61b5b34

Branch: refs/heads/jira/solr-10233
Commit: e61b5b34bf14b9addd98eeafdad43b92e6208d5f
Parents: 904df0e
Author: jdyer1 <jdyer@apache.org>
Authored: Fri May 12 08:44:16 2017 -0500
Committer: jdyer1 <jdyer@apache.org>
Committed: Fri May 12 08:44:16 2017 -0500

----------------------------------------------------------------------
 dev-tools/idea/.idea/libraries/HSQLDB.xml       |   2 +-
 lucene/ivy-versions.properties                  |   3 +-
 solr/CHANGES.txt                                |   7 +-
 solr/contrib/dataimporthandler/ivy.xml          |   2 +-
 .../TestVariableResolverEndToEnd.java           |   8 +-
 .../apache/solr/handler/CalciteJDBCStream.java  |  75 ++++
 .../org/apache/solr/handler/SQLHandler.java     |   3 +-
 solr/example/example-DIH/ivy.xml                |   2 +-
 solr/licenses/hsqldb-1.8.0.10.jar.sha1          |   1 -
 solr/licenses/hsqldb-2.4.0.jar.sha1             |   1 +
 solr/licenses/hsqldb-LICENSE-BSD_LIKE.txt       |   9 +-
 solr/licenses/hsqldb-NOTICE.txt                 | 135 ++++----
 .../solr/client/solrj/io/stream/JDBCStream.java | 342 ++++++++++++++-----
 .../client/solrj/io/stream/JDBCStreamTest.java  |  34 +-
 14 files changed, 450 insertions(+), 174 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/dev-tools/idea/.idea/libraries/HSQLDB.xml
----------------------------------------------------------------------
diff --git a/dev-tools/idea/.idea/libraries/HSQLDB.xml b/dev-tools/idea/.idea/libraries/HSQLDB.xml
index 4eb35d5..39efcbf 100644
--- a/dev-tools/idea/.idea/libraries/HSQLDB.xml
+++ b/dev-tools/idea/.idea/libraries/HSQLDB.xml
@@ -1,7 +1,7 @@
 <component name="libraryTable">
   <library name="HSQLDB">
     <CLASSES>
-      <root url="jar://$PROJECT_DIR$/solr/example/example-DIH/solr/db/lib/hsqldb-1.8.0.10.jar!/"
/>
+      <root url="jar://$PROJECT_DIR$/solr/example/example-DIH/solr/db/lib/hsqldb-2.4.0.jar!/"
/>
     </CLASSES>
     <JAVADOC />
     <SOURCES />

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index 3a1efbb..7e0e7c7 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -53,7 +53,6 @@ com.sun.jersey.version = 1.9
 /commons-logging/commons-logging = 1.1.3
 /de.l3s.boilerpipe/boilerpipe = 1.1.0
 /dom4j/dom4j = 1.6.1
-/hsqldb/hsqldb = 1.8.0.10
 /info.ganglia.gmetric4j/gmetric4j = 1.0.7
 
 io.dropwizard.metrics.version = 3.2.2
@@ -242,6 +241,8 @@ org.gagravarr.vorbis.java.version = 0.8
 /org.gagravarr/vorbis-java-core = ${org.gagravarr.vorbis.java.version}
 /org.gagravarr/vorbis-java-tika = ${org.gagravarr.vorbis.java.version}
 
+/org.hsqldb/hsqldb = 2.4.0
+
 /org.jsoup/jsoup = 1.8.2
 
 /org.locationtech.spatial4j/spatial4j = 0.6

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 5f6f482..10edc52 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -158,9 +158,14 @@ Apache UIMA 2.3.1
 Apache ZooKeeper 3.4.10
 Jetty 9.3.14.v20161028
 
+Detailed Change List
+----------------------
 
-(No Changes)
+Other Changes
+----------------------
 
+* SOLR-10617: JDBCStream accepts columns of type TIME, DATE & TIMESTAMP as well as CLOBs
and decimal
+  numeric types (James Dyer)
 
 ==================  6.6.0 ==================
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/contrib/dataimporthandler/ivy.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/ivy.xml b/solr/contrib/dataimporthandler/ivy.xml
index ea138dd..67af77b 100644
--- a/solr/contrib/dataimporthandler/ivy.xml
+++ b/solr/contrib/dataimporthandler/ivy.xml
@@ -23,7 +23,7 @@
     <conf name="test" transitive="false"/>
   </configurations>
   <dependencies>
-    <dependency org="hsqldb" name="hsqldb" rev="${/hsqldb/hsqldb}" conf="test"/>
+    <dependency org="org.hsqldb" name="hsqldb" rev="${/org.hsqldb/hsqldb}" conf="test"/>
     <dependency org="org.apache.derby" name="derby" rev="${/org.apache.derby/derby}" conf="test"/>
 
     <dependency org="org.mockito" name="mockito-core" rev="${/org.mockito/mockito-core}"
conf="test"/>

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
index 0f53b93..8ee6878 100644
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
+++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
@@ -73,14 +73,14 @@ public class TestVariableResolverEndToEnd  extends AbstractDIHJdbcTestCase
{
         "select " +
         " 1 as id, " +
         " 'SELECT' as SELECT_KEYWORD, " +
-        " CURRENT_TIMESTAMP as FIRST_TS " +
+        " {ts '2017-02-18 12:34:56'} as FIRST_TS " +
         "from DUAL \" >\n");
     sb.append("  <field column=\"SELECT_KEYWORD\" name=\"select_keyword_s\" /> \n");
     sb.append("  <entity name=\"SECOND\" processor=\"SqlEntityProcessor\" dataSource=\"hsqldb\"
transformer=\"TemplateTransformer\" ");
     sb.append("   query=\"" +
         "${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
         " 1 as SORT, " +
-        " CURRENT_TIMESTAMP as SECOND_TS, " +
+        " {ts '2017-02-18 12:34:56'} as SECOND_TS, " +
         " '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam
+ ")}' as SECOND1_S,  " +
         " 'PORK' AS MEAT, " +
         " 'GRILL' AS METHOD, " +
@@ -91,7 +91,7 @@ public class TestVariableResolverEndToEnd  extends AbstractDIHJdbcTestCase
{
         "UNION " +        
         "${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
         " 2 as SORT, " +
-        " CURRENT_TIMESTAMP as SECOND_TS, " +
+        " {ts '2017-02-18 12:34:56'} as SECOND_TS, " +
         " '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam
+ ")}' as SECOND1_S,  " +
         " 'FISH' AS MEAT, " +
         " 'FRY' AS METHOD, " +
@@ -112,7 +112,7 @@ public class TestVariableResolverEndToEnd  extends AbstractDIHJdbcTestCase
{
     sb.append("</document> \n");
     sb.append("</dataConfig> \n");
     String config = sb.toString();
-    log.debug(config); 
+    log.info(config); 
     return config;
   }
   @Override

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/core/src/java/org/apache/solr/handler/CalciteJDBCStream.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/CalciteJDBCStream.java b/solr/core/src/java/org/apache/solr/handler/CalciteJDBCStream.java
new file mode 100644
index 0000000..3738f62
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/CalciteJDBCStream.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler;
+
+import java.io.IOException;
+import java.sql.Array;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.util.Properties;
+
+import org.apache.solr.client.solrj.io.comp.StreamComparator;
+import org.apache.solr.client.solrj.io.stream.JDBCStream;
+
+/**
+ * Used with o.a.s.Handler.SQLHandler.
+ * 
+ * @lucene.internal
+ */
+public class CalciteJDBCStream extends JDBCStream {
+  private static final long serialVersionUID = 1L;
+
+  public CalciteJDBCStream(String connectionUrl, String sqlQuery, StreamComparator definedSort,
+      Properties connectionProperties, String driverClassName) throws IOException {
+    super(connectionUrl, sqlQuery, definedSort, connectionProperties, driverClassName);
+  }
+
+  @Override
+  protected ResultSetValueSelector determineValueSelector(int columnIdx, ResultSetMetaData
metadata)
+      throws SQLException {
+    ResultSetValueSelector valueSelector = super.determineValueSelector(columnIdx, metadata);
+    if (valueSelector == null) {
+      final int columnNumber = columnIdx + 1;
+      final String columnName = metadata.getColumnLabel(columnNumber);
+      final String className = metadata.getColumnClassName(columnNumber);
+      if (Array.class.getName().equals(className)) {
+        valueSelector = new ResultSetValueSelector() {
+          @Override
+          public Object selectValue(ResultSet resultSet) throws SQLException {
+            Object o = resultSet.getObject(columnNumber);
+            if (resultSet.wasNull()) {
+              return null;
+            }
+            if (o instanceof Array) {
+              Array array = (Array) o;
+              return array.getArray();
+            } else {
+              return o;
+            }
+          }
+
+          @Override
+          public String getColumnName() {
+            return columnName;
+          }
+        };
+      }
+    }
+    return valueSelector;
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/SQLHandler.java b/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
index c80d0d7..67ea1f6 100644
--- a/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
@@ -31,7 +31,6 @@ import org.apache.calcite.config.Lex;
 import org.apache.solr.client.solrj.io.Tuple;
 import org.apache.solr.client.solrj.io.comp.StreamComparator;
 import org.apache.solr.client.solrj.io.stream.ExceptionStream;
-import org.apache.solr.client.solrj.io.stream.JDBCStream;
 import org.apache.solr.client.solrj.io.stream.TupleStream;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.CommonParams;
@@ -141,7 +140,7 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware,
Per
   /*
    * Only necessary for SolrJ JDBC driver since metadata has to be passed back
    */
-  private static class SqlHandlerStream extends JDBCStream {
+  private static class SqlHandlerStream extends CalciteJDBCStream {
     private final boolean includeMetadata;
     private boolean firstTuple = true;
     List<String> metadataFields = new ArrayList<>();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/example/example-DIH/ivy.xml
----------------------------------------------------------------------
diff --git a/solr/example/example-DIH/ivy.xml b/solr/example/example-DIH/ivy.xml
index 4060e37..3f67a79 100644
--- a/solr/example/example-DIH/ivy.xml
+++ b/solr/example/example-DIH/ivy.xml
@@ -22,7 +22,7 @@
       <conf name="compile" transitive="false"/>
     </configurations>
     <dependencies>
-      <dependency org="hsqldb" name="hsqldb" rev="${/hsqldb/hsqldb}" conf="compile"/>
+      <dependency org="org.hsqldb" name="hsqldb" rev="${/org.hsqldb/hsqldb}" conf="compile"/>
       <dependency org="org.apache.derby" name="derby" rev="${/org.apache.derby/derby}"
conf="compile"/>
       <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/> 
     </dependencies>

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/licenses/hsqldb-1.8.0.10.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/hsqldb-1.8.0.10.jar.sha1 b/solr/licenses/hsqldb-1.8.0.10.jar.sha1
deleted file mode 100644
index 8661e69..0000000
--- a/solr/licenses/hsqldb-1.8.0.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7e9978fdb754bce5fcd5161133e7734ecb683036

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/licenses/hsqldb-2.4.0.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/hsqldb-2.4.0.jar.sha1 b/solr/licenses/hsqldb-2.4.0.jar.sha1
new file mode 100644
index 0000000..72520ee
--- /dev/null
+++ b/solr/licenses/hsqldb-2.4.0.jar.sha1
@@ -0,0 +1 @@
+195957160ed990dbc798207c0d577280d9919208
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/licenses/hsqldb-LICENSE-BSD_LIKE.txt
----------------------------------------------------------------------
diff --git a/solr/licenses/hsqldb-LICENSE-BSD_LIKE.txt b/solr/licenses/hsqldb-LICENSE-BSD_LIKE.txt
index 953bfa0..b8ac4f5 100644
--- a/solr/licenses/hsqldb-LICENSE-BSD_LIKE.txt
+++ b/solr/licenses/hsqldb-LICENSE-BSD_LIKE.txt
@@ -1,4 +1,4 @@
-/* Copyright (c) 2001-2005, The HSQL Development Group
+/* Copyright (c) 2001-2017, The HSQL Development Group
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -18,9 +18,9 @@
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, 
- * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
+ * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
@@ -28,4 +28,3 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/licenses/hsqldb-NOTICE.txt
----------------------------------------------------------------------
diff --git a/solr/licenses/hsqldb-NOTICE.txt b/solr/licenses/hsqldb-NOTICE.txt
index 0929e9b..829f8a2 100644
--- a/solr/licenses/hsqldb-NOTICE.txt
+++ b/solr/licenses/hsqldb-NOTICE.txt
@@ -1,68 +1,69 @@
-=========================================================================
-==     HSQLDB Notice                                                   ==
-=========================================================================
+/*
+ * For work developed by the HSQL Development Group:
+ *
+ * Copyright (c) 2001-2017, The HSQL Development Group
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the HSQL Development Group nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *
+ *
+ * For work originally developed by the Hypersonic SQL Group:
+ *
+ * Copyright (c) 1995-2000, The Hypersonic SQL Group.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the Hypersonic SQL Group nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP,
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This software consists of voluntary contributions made by many individuals
+ * on behalf of the Hypersonic SQL Group.
+ */
 
-For content, code, and products originally developed by Thomas Mueller and the Hypersonic
SQL Group:
-
-Copyright (c) 1995-2000 by the Hypersonic SQL Group.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-Redistributions of source code must retain the above copyright notice, this
-list of conditions and the following disclaimer.
-
-Redistributions in binary form must reproduce the above copyright notice,
-this list of conditions and the following disclaimer in the documentation
-and/or other materials provided with the distribution.
-
-Neither the name of the Hypersonic SQL Group nor the names of its
-contributors may be used to endorse or promote products derived from this
-software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP,
-OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-This software consists of voluntary contributions made by many individuals on behalf of the
-Hypersonic SQL Group.
-
-For work added by the HSQL Development Group (a.k.a. hsqldb_lic.txt):
-
-Copyright (c) 2001-2005, The HSQL Development Group
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-Redistributions of source code must retain the above copyright notice, this
-list of conditions and the following disclaimer.
-
-Redistributions in binary form must reproduce the above copyright notice,
-this list of conditions and the following disclaimer in the documentation
-and/or other materials provided with the distribution.
-
-Neither the name of the HSQL Development Group nor the names of its
-contributors may be used to endorse or promote products derived from this
-software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
-OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java
index 172b9ef..99b090f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java
@@ -17,13 +17,19 @@
 package org.apache.solr.client.solrj.io.stream;
 
 import java.io.IOException;
-import java.sql.Array;
+import java.math.BigDecimal;
+import java.sql.Clob;
 import java.sql.Connection;
+import java.sql.Date;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
 import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.sql.Types;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -51,16 +57,64 @@ import static org.apache.solr.common.params.CommonParams.SORT;
  * Connects to a datasource using a registered JDBC driver and execute a query. The results
of
  * that query will be returned as tuples. An EOF tuple will indicate that all have been read.
  * 
- * Supported Datatypes
- * JDBC Type     | Tuple Type
- * --------------|---------------
- * String        | String
- * Short         | Long
- * Integer       | Long
- * Long          | Long
- * Float         | Double
- * Double        | Double
- * Boolean       | Boolean
+ * Supported Datatypes for most types vary by JDBC driver based on the specific 
+ * java type as reported by {@link java.sql.ResultSetMetaData#getColumnClassName(int)}. 
+ * The exception are {@link Types#DATE}, {@link Types#TIME} or {@link Types#TIMESTAMP}
+ * which are determined by the JDBC type.
+ * 
+ * <table rules="all" frame="box" cellpadding="3" summary="Supported Java Types">
+ * <tr>
+ *   <th>Java or JDBC Type</th>
+ *   <th>Tuple Type</th>
+ *   <th>Notes</th>
+ * </tr>
+ * <tr>
+ *   <td>Boolean</td>
+ *   <td>Boolean</td>
+ *   <td></td>
+ * </tr>
+ * <tr>
+ *   <td>String</td>
+ *   <td>String</td>
+ *   <td></td>
+ * </tr>
+ * <tr>
+ *   <td>Short, Integer, Long</td>
+ *   <td>Long</td>
+ *   <td></td>
+ * </tr>
+ * <tr>
+ *   <td>Float, Double</td>
+ *   <td>Double</td>
+ *   <td></td>
+ * </tr>
+ * <tr>
+ *   <td>{@link Clob} and subclasses</td>
+ *   <td>String</td>
+ *   <td>Clobs up to length 2<sup>31</sup>-1 are supported.</td>
+ * </tr>
+ * <tr>
+ *   <td>Other subclasses of {@link Number}</td>
+ *   <td>Long, Double</td>
+ *   <td>Tuple Type based on {@link BigDecimal#scale()}.</td>
+ * </tr>
+ * <tr>
+ *   <td>JDBC {@link Types#DATE}</td>
+ *   <td>String</td>
+ *   <td>yyyy-MM-dd, calls {@link Date#toString}</td>
+ * </tr>
+ * <tr>
+ *   <td>JDBC {@link Types#TIME}</td>
+ *   <td>String</td>
+ *   <td>hh:mm:ss, calls {@link Time#toString}</td>
+ * </tr>
+ * <tr>
+ *   <td>JDBC {@link Types#TIMESTAMP}</td>
+ *   <td>String</td>
+ *   <td>See {@link DateTimeFormatter#ISO_INSTANT}</td>
+ * </tr>
+ * </table>
+ * 
  **/
 
 public class JDBCStream extends TupleStream implements Expressible {
@@ -227,91 +281,200 @@ public class JDBCStream extends TupleStream implements Expressible
{
   }
 
   private ResultSetValueSelector[] constructValueSelectors(ResultSetMetaData metadata) throws
SQLException{
-    ResultSetValueSelector[] valueSelectors = new ResultSetValueSelector[metadata.getColumnCount()];
+    ResultSetValueSelector[] valueSelectors = new ResultSetValueSelector[metadata.getColumnCount()];
   
+    for (int columnIdx = 0; columnIdx < metadata.getColumnCount(); ++columnIdx) {    
 
+      ResultSetValueSelector valueSelector = determineValueSelector(columnIdx, metadata);
+      if(valueSelector==null) {
+        int columnNumber = columnIdx + 1;
+        String columnName = metadata.getColumnLabel(columnNumber);
+        String className = metadata.getColumnClassName(columnNumber);
+        String typeName = metadata.getColumnTypeName(columnNumber);
+        throw new SQLException(String.format(Locale.ROOT,
+            "Unable to determine the valueSelector for column '%s' (col #%d) of java class
'%s' and type '%s'",
+            columnName, columnNumber, className, typeName));
+      }
+      valueSelectors[columnIdx] = valueSelector;
+    }        
+    return valueSelectors;
+  }
+  
+  protected ResultSetValueSelector determineValueSelector(int columnIdx, ResultSetMetaData
metadata) throws SQLException {
+    final int columnNumber = columnIdx + 1; // cause it starts at 1
+    // Use getColumnLabel instead of getColumnName to make sure fields renamed with AS as
picked up properly
+    final String columnName = metadata.getColumnLabel(columnNumber);
+    final int jdbcType = metadata.getColumnType(columnNumber);      
+    final String className = metadata.getColumnClassName(columnNumber);
+    ResultSetValueSelector valueSelector = null;
     
-    for(int columnIdx = 0; columnIdx < metadata.getColumnCount(); ++columnIdx){
-      final int columnNumber = columnIdx + 1; // cause it starts at 1
-      // Use getColumnLabel instead of getColumnName to make sure fields renamed with AS
as picked up properly
-      final String columnName = metadata.getColumnLabel(columnNumber);
-      String className = metadata.getColumnClassName(columnNumber);
-      String typeName = metadata.getColumnTypeName(columnNumber);
-      
-      if(directSupportedTypes.contains(className)){
-        valueSelectors[columnIdx] = new ResultSetValueSelector() {
-          public Object selectValue(ResultSet resultSet) throws SQLException {
-            Object obj = resultSet.getObject(columnNumber);
-            if(resultSet.wasNull()){ return null; }
-            if(obj instanceof String) {
-              String s = (String)obj;
-              if(s.indexOf(sep) > -1) {
-                s = s.substring(1);
-                return s.split(sep);
-              }
+    // Directly supported types can be just directly returned - no conversion really necessary
+    if(directSupportedTypes.contains(className)){
+      valueSelector = new ResultSetValueSelector() {
+        @Override
+        public Object selectValue(ResultSet resultSet) throws SQLException {
+          Object obj = resultSet.getObject(columnNumber);
+          if(resultSet.wasNull()){ return null; }
+          if(obj instanceof String) {
+            String s = (String)obj;
+            if(s.indexOf(sep) > -1) {
+              s = s.substring(1);
+              return s.split(sep);
             }
-
-            return obj;
-          }
-          public String getColumnName() {
-            return columnName;
-          }
-        };
-      } else if(Short.class.getName().equals(className)) {
-        valueSelectors[columnIdx] = new ResultSetValueSelector() {
-          public Object selectValue(ResultSet resultSet) throws SQLException {
-            Short obj = resultSet.getShort(columnNumber);
-            if(resultSet.wasNull()){ return null; }
-            return obj.longValue();
-          }
-          public String getColumnName() {
-            return columnName;
-          }
-        };
-      } else if(Integer.class.getName().equals(className)) {
-        valueSelectors[columnIdx] = new ResultSetValueSelector() {
-          public Object selectValue(ResultSet resultSet) throws SQLException {
-            Integer obj = resultSet.getInt(columnNumber);
-            if(resultSet.wasNull()){ return null; }
-            return obj.longValue();
           }
-          public String getColumnName() {
-            return columnName;
-          }
-        };
-      } else if(Float.class.getName().equals(className)) {
-        valueSelectors[columnIdx] = new ResultSetValueSelector() {
-          public Object selectValue(ResultSet resultSet) throws SQLException {
-            Float obj = resultSet.getFloat(columnNumber);
-            if(resultSet.wasNull()){ return null; }
-            return obj.doubleValue();
-          }
-          public String getColumnName() {
-            return columnName;
-          }
-        };
-      } else if(Array.class.getName().equals(className)) {
-        valueSelectors[columnIdx] = new ResultSetValueSelector() {
+
+          return obj;
+        }
+        @Override
+        public String getColumnName() {
+          return columnName;
+        }
+      };
+    } 
+    // We're checking the Java class names because there are lots of SQL types across
+    // lots of database drivers that can be mapped to standard Java types. Basically, 
+    // this makes it easier and we don't have to worry about esoteric type names in the 
+    // JDBC family of types
+    else if(Short.class.getName().equals(className)) {
+      valueSelector = new ResultSetValueSelector() {
+        @Override
+        public Object selectValue(ResultSet resultSet) throws SQLException {
+          Short obj = resultSet.getShort(columnNumber);
+          if(resultSet.wasNull()){ return null; }
+          return obj.longValue();
+        }
+        @Override
+        public String getColumnName() {
+          return columnName;
+        }
+      };
+    } else if(Integer.class.getName().equals(className)) {
+      valueSelector = new ResultSetValueSelector() {
+        @Override
+        public Object selectValue(ResultSet resultSet) throws SQLException {
+          Integer obj = resultSet.getInt(columnNumber);
+          if(resultSet.wasNull()){ return null; }
+          return obj.longValue();
+        }
+        @Override
+        public String getColumnName() {
+          return columnName;
+        }
+      };
+    } else if(Float.class.getName().equals(className)) {
+      valueSelector = new ResultSetValueSelector() {
+        @Override
+        public Object selectValue(ResultSet resultSet) throws SQLException {
+          Float obj = resultSet.getFloat(columnNumber);
+          if(resultSet.wasNull()){ return null; }
+          return obj.doubleValue();
+        }
+        @Override
+        public String getColumnName() {
+          return columnName;
+        }
+      };
+    } 
+    // Here we are switching to check against the SQL type because date/times are
+    // notorious for not being consistent. We don't know if the driver is mapping
+    // to a java.time.* type or some old-school type. 
+    else if (jdbcType == Types.DATE) {
+      valueSelector = new ResultSetValueSelector() {
+        @Override
+        public Object selectValue(ResultSet resultSet) throws SQLException {
+          Date sqlDate = resultSet.getDate(columnNumber);
+          return resultSet.wasNull() ? null : sqlDate.toString();
+        }
+        @Override
+        public String getColumnName() {
+          return columnName;
+        }
+      };
+    } else if (jdbcType == Types.TIME ) {
+      valueSelector = new ResultSetValueSelector() {
+        @Override
+        public Object selectValue(ResultSet resultSet) throws SQLException {
+          Time sqlTime = resultSet.getTime(columnNumber);
+          return resultSet.wasNull() ? null : sqlTime.toString();
+        }
+        @Override
+        public String getColumnName() {
+          return columnName;
+        }
+      };
+    } else if (jdbcType == Types.TIMESTAMP) {
+      valueSelector = new ResultSetValueSelector() {
+        @Override
+        public Object selectValue(ResultSet resultSet) throws SQLException {
+          Timestamp sqlTimestamp = resultSet.getTimestamp(columnNumber);
+          return resultSet.wasNull() ? null : sqlTimestamp.toInstant().toString();
+        }
+        @Override
+        public String getColumnName() {
+          return columnName;
+        }
+      };
+    } 
+    // Now we're going to start seeing if things are assignable from the returned type
+    // to a more general type - this allows us to cover cases where something we weren't

+    // explicitly expecting, but can handle, is being returned.
+    else {
+      Class<?> clazz;
+      try {
+        clazz = Class.forName(className, false, getClass().getClassLoader());
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+      final int scale = metadata.getScale(columnNumber);
+      if (Number.class.isAssignableFrom(clazz)) {
+        if (scale > 0) {
+          valueSelector = new ResultSetValueSelector() {
+            @Override
+            public Object selectValue(ResultSet resultSet) throws SQLException {
+              BigDecimal bd = resultSet.getBigDecimal(columnNumber);
+              return resultSet.wasNull() ? null : bd.doubleValue();                
+            }
+            @Override
+            public String getColumnName() {
+              return columnName;
+            }
+          };            
+        } else {
+          valueSelector = new ResultSetValueSelector() {
+            @Override
+            public Object selectValue(ResultSet resultSet) throws SQLException {
+              BigDecimal bd = resultSet.getBigDecimal(columnNumber);
+              return resultSet.wasNull() ? null : bd.longValue();
+            }
+            @Override
+            public String getColumnName() {
+              return columnName;
+            }
+          };            
+        }          
+      } else if (Clob.class.isAssignableFrom(clazz)) {
+        valueSelector = new ResultSetValueSelector() {
+          @Override
           public Object selectValue(ResultSet resultSet) throws SQLException {
-            Object o = resultSet.getObject(columnNumber);
-            if(resultSet.wasNull()){ return null; }
-            if(o instanceof Array) {
-              Array array = (Array)o;
-              return array.getArray();
-            } else {
-              return o;
+            Clob c = resultSet.getClob(columnNumber);
+            if (resultSet.wasNull()) {
+              return null;
             }
+            long length = c.length();
+            int lengthInt = (int) length;
+            if (length != lengthInt) {
+              throw new SQLException(String.format(Locale.ROOT,
+                  "Encountered a clob of length #%l in column '%s' (col #%d).  Max supported
length is #%i.",
+                  length, columnName, columnNumber, Integer.MAX_VALUE));
+            }
+            return c.getSubString(1, lengthInt);
           }
+          @Override
           public String getColumnName() {
             return columnName;
           }
         };
-      } else {
-        throw new SQLException(String.format(Locale.ROOT,
-            "Unable to determine the valueSelector for column '%s' (col #%d) of java class
'%s' and type '%s'",
-            columnName, columnNumber, className, typeName));
-      }
+      } 
     }
-    
-    return valueSelectors;
+    return valueSelector;
   }
   
   /**
@@ -432,9 +595,10 @@ public class JDBCStream extends TupleStream implements Expressible {
     // it's already in the sqlQuery but there's no way we can reliably determine the sort
from the query.
     return definedSort;
   }
+  
+  public interface ResultSetValueSelector {
+    String getColumnName();
+    Object selectValue(ResultSet resultSet) throws SQLException;
+  }
 }
 
-interface ResultSetValueSelector {
-  String getColumnName();
-  Object selectValue(ResultSet resultSet) throws SQLException;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e61b5b34/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java
b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java
index 9fff33a..8813f24 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java
@@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.io.stream;
 import java.io.IOException;
 import java.sql.Connection;
 import java.sql.DriverManager;
+import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.ArrayList;
@@ -96,6 +97,8 @@ public class JDBCStreamTest extends SolrCloudTestCase {
     statement.executeUpdate("create table PEOPLE(ID int not null primary key, NAME varchar(50),
COUNTRY_CODE char(2), DELETED char(1) default 'N')");
     statement.executeUpdate("create table PEOPLE_SPORTS(ID int not null primary key, PERSON_ID
int, SPORT_NAME varchar(50), DELETED char(1) default 'N')");
     statement.executeUpdate("create table UNSUPPORTED_COLUMNS(ID int not null primary key,
UNSP binary)");
+    statement.executeUpdate("create table DUAL(ID int not null primary key)");
+    statement.executeUpdate("insert into DUAL values(1)");
     
   }
 
@@ -158,8 +161,29 @@ public class JDBCStreamTest extends SolrCloudTestCase {
     assertOrderOf(tuples, "CODE", "NP", "NL", "NO", "US");
     assertOrderOf(tuples, "COUNTRY_NAME", "Nepal", "Netherlands", "Norway", "United States");
     
+    // Additional Types
+    String query = "select 1 as ID1, {ts '2017-02-18 12:34:56.789'} as TS1, {t '01:02:03'}
as T1, "
+        + "{d '1593-03-14'} as D1, cast(12.34 AS DECIMAL(4,2)) as DEC4_2, "
+        + "cast(1234 AS DECIMAL(4,0)) as DEC4_0, cast('big stuff' as CLOB(100)) as CLOB1
"
+        + "from DUAL order by ID1";
+    stream = new JDBCStream("jdbc:hsqldb:mem:.", query, new FieldComparator("ID1", ComparatorOrder.ASCENDING));
+    tuples = getTuples(stream);
+    assertEquals(1, tuples.size());
+    Tuple t;
+    try (Connection connection = DriverManager.getConnection("jdbc:hsqldb:mem:.");
+        Statement statement = connection.createStatement()) {
+      ResultSet rs = statement.executeQuery(query);
+      rs.next();
+      t = tuples.iterator().next();
+      assertString(t, "CLOB1", rs.getString("CLOB1"));
+      assertString(t, "TS1", rs.getTimestamp("TS1").toInstant().toString());
+      assertString(t, "T1", rs.getTime("T1").toString());
+      assertString(t, "D1", rs.getDate("D1").toString());
+      assertDouble(t, "DEC4_2", rs.getDouble("DEC4_2"));
+      assertLong(t, "DEC4_0", rs.getLong("DEC4_0"));      
+    }     
   }
-
+  
   @Test
   public void testJDBCJoin() throws Exception {
     
@@ -643,6 +667,14 @@ public class JDBCStreamTest extends SolrCloudTestCase {
     return true;
   }
   
+  public boolean assertDouble(Tuple tuple, String fieldName, double d) throws Exception {
+    double dv = (double)tuple.get(fieldName);
+    if(dv != d) {
+      throw new Exception("Doubles not equal:"+d+" : "+dv);
+    }
+    return true;
+  }
+  
   public boolean assertString(Tuple tuple, String fieldName, String expected) throws Exception
{
     String actual = (String)tuple.get(fieldName);
     


Mime
View raw message