zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjf...@apache.org
Subject zeppelin git commit: Zeppelin 3879: create "maxRows" and "rowsFetchSize" values in interpreter/jdbc/interpreter-setting.json
Date Fri, 14 Dec 2018 01:22:35 GMT
Repository: zeppelin
Updated Branches:
  refs/heads/branch-0.8 a6cf2dfa3 -> 9f5cf17bd


Zeppelin 3879: create "maxRows" and "rowsFetchSize" values in interpreter/jdbc/interpreter-setting.json

This PR introduces code to configure manually the "maxRows" and "rowsFetchSize" Statement
settings directly from the interpreter/jdbc/interpreter-setting.json file. The default value
for these two settings is 1000 which can slow down a Zeppelin instance quite a lot if a SQL
query returns a big chunk of data. See [1] and [2] for more info about these function calls.

[1]: https://docs.oracle.com/javase/8/docs/api/java/sql/Statement.html#setFetchSize-int-
[2]: https://docs.oracle.com/javase/8/docs/api/java/sql/Statement.html#setMaxRows-int-

Improvement.

https://issues.apache.org/jira/browse/ZEPPELIN-3879

* CI pass

* Does the licenses files need update? No.
* Is there breaking changes for older versions? No.
* Does this needs documentation? Yes. For the time being, I'd like the Zeppelin authors input
whether such a change makes sense.

Thanks!

Author: Patrice Clement <monsieurp@gentoo.org>

Closes #3239 from monsieurp/ZEPPELIN-3879 and squashes the following commits:

b27ebbba0 [Patrice Clement] ZEPPELIN-3879: create "maxRows" value in interpreter/jdbc/interpreter-setting.json

(cherry picked from commit e63ba8e897a522c6cad099286110c2eaa1496912)
Signed-off-by: jeffzhang.zjf <jeffzhang.zjf@alibaba-inc.com>


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/9f5cf17b
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/9f5cf17b
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/9f5cf17b

Branch: refs/heads/branch-0.8
Commit: 9f5cf17bdd09c400dc06679bbc61d381fa0a8c56
Parents: a6cf2df
Author: Patrice Clement <monsieurp@gentoo.org>
Authored: Fri Nov 23 23:45:44 2018 +0100
Committer: jeffzhang.zjf <jeffzhang.zjf@alibaba-inc.com>
Committed: Fri Dec 14 09:22:25 2018 +0800

----------------------------------------------------------------------
 .../org/apache/zeppelin/jdbc/JDBCInterpreter.java    | 15 +++++++++++++--
 jdbc/src/main/resources/interpreter-setting.json     |  7 +++++++
 2 files changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9f5cf17b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
index a8a4969..9b3777a 100644
--- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
+++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
@@ -134,12 +134,14 @@ public class JDBCInterpreter extends KerberosInterpreter {
   private final String CONCURRENT_EXECUTION_KEY = "zeppelin.jdbc.concurrent.use";
   private final String CONCURRENT_EXECUTION_COUNT = "zeppelin.jdbc.concurrent.max_connection";
   private final String DBCP_STRING = "jdbc:apache:commons:dbcp:";
+  private static final String MAX_ROWS_KEY = "zeppelin.jdbc.maxRows";
 
   private final HashMap<String, Properties> basePropretiesMap;
   private final HashMap<String, JDBCUserConfigurations> jdbcUserConfigurationsMap;
   private final HashMap<String, SqlCompleter> sqlCompletersMap;
 
   private int maxLineResults;
+  private int maxRows;
 
   public JDBCInterpreter(Properties property) {
     super(property);
@@ -207,6 +209,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
     logger.debug("JDBC PropretiesMap: {}", basePropretiesMap);
 
     setMaxLineResults();
+    setMaxRows();
   }
 
 
@@ -228,6 +231,14 @@ public class JDBCInterpreter extends KerberosInterpreter {
     }
   }
 
+  /**
+   * Fetch MAX_ROWS_KEYS value from property file and set it to
+   * "maxRows" value.
+   */
+  private void setMaxRows() {
+    maxRows = Integer.valueOf(getProperty(MAX_ROWS_KEY, "1000"));
+  }
+
   private SqlCompleter createOrUpdateSqlCompleter(SqlCompleter sqlCompleter,
       final Connection connection, String propertyKey, final String buf, final int cursor)
{
     String schemaFiltersKey = String.format("%s.%s", propertyKey, COMPLETER_SCHEMA_FILTERS_KEY);
@@ -706,7 +717,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
 
         // fetch n+1 rows in order to indicate there's more rows available (for large selects)
         statement.setFetchSize(getMaxResult());
-        statement.setMaxRows(getMaxResult() + 1);
+        statement.setMaxRows(maxRows);
 
         if (statement == null) {
           return new InterpreterResult(Code.ERROR, "Prefix not found.");
@@ -717,7 +728,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
 
           String statementPrecode =
               getProperty(String.format(STATEMENT_PRECODE_KEY_TEMPLATE, propertyKey));
-          
+
           if (StringUtils.isNotBlank(statementPrecode)) {
             statement.execute(statementPrecode);
           }

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9f5cf17b/jdbc/src/main/resources/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/jdbc/src/main/resources/interpreter-setting.json b/jdbc/src/main/resources/interpreter-setting.json
index 7e9dd66..34992d9 100644
--- a/jdbc/src/main/resources/interpreter-setting.json
+++ b/jdbc/src/main/resources/interpreter-setting.json
@@ -121,6 +121,13 @@
         "defaultValue": "-1",
         "description": "Maximum of connection lifetime in milliseconds. A value of zero or
less means the connection has an infinite lifetime.",
         "type": "number"
+      },
+      "zeppelin.jdbc.maxRows": {
+        "envName": null,
+        "propertyName": "zeppelin.jdbc.maxRows",
+        "defaultValue": "1000",
+        "description": "Maximum number of rows fetched from the query.",
+        "type": "number"
       }
     },
     "editor": {


Mime
View raw message