eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h..@apache.org
Subject incubator-eagle git commit: [EAGLE-273][EAGLE-269] Fix exceeding the maximum row size of 65, 535 bytes in mysql and comparisons between LONG_VARCHAR not supported
Date Fri, 27 May 2016 11:34:43 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master f90a0a49a -> 9849303eb


[EAGLE-273][EAGLE-269] Fix exceeding the maximum row size of 65,535 bytes in mysql and comparisons
between LONG_VARCHAR not supported

https://issues.apache.org/jira/browse/EAGLE-273
https://issues.apache.org/jira/browse/EAGLE-269

The problem is cause the limitation of maximum row size in mysql innodb engine, so need to
keep the column size relatively small, but it will cause it can't store large field, it's
a little tricky, so may have following possible solutions:

* Solution One: Fix mysql innodb size to resolve the limitation.
* Solution Two: Keep the field relatively small to avoid exceeding maximum row size and make
sure creating table successfully, and modify field size according to actual usage (Which is
the approach we take for this problem)

Author: Hao Chen <hao@apache.org>

Closes #158 from haoch/EAGLE-273-LATEST.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/9849303e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/9849303e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/9849303e

Branch: refs/heads/master
Commit: 9849303ebf8a63e4f54f0eb035c8060c43e50611
Parents: f90a0a4
Author: Hao Chen <hao@apache.org>
Authored: Fri May 27 19:34:32 2016 +0800
Committer: Hao Chen <hao@apache.org>
Committed: Fri May 27 19:34:32 2016 +0800

----------------------------------------------------------------------
 .../eagle/storage/jdbc/JdbcConstants.java       |  5 ++-
 .../schema/JdbcEntityDefinitionManager.java     |  3 +-
 .../jdbc/schema/JdbcEntitySchemaManager.java    |  6 +--
 .../eagle/storage/jdbc/TestJdbcStorage.java     | 19 +++++++++
 .../bin/quick-start-with-external-webservice.sh | 45 ++++++++++++++++++++
 .../resources/safemodecheck-policy-import.sh    |  2 +-
 6 files changed, 73 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9849303e/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcConstants.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcConstants.java
b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcConstants.java
index a6d0c93..ecf2845 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcConstants.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcConstants.java
@@ -28,7 +28,10 @@ public class JdbcConstants {
     public static final String ROW_KEY_COLUMN_NAME = "uuid";
 
     public static final int DEFAULT_TYPE_FOR_COMPLEX_TYPE = Types.BLOB;
-    public static final int DEFAULT_VARCHAR_SIZE =30000;
+
+    //MySQL Max Row Size: 21485
+    public static final int DEFAULT_FIELD_VARCHAR_SIZE =1000;
+    public static final int DEFAULT_TAG_VARCHAR_SIZE =254;
 
     // Eagle JDBC Storage Configuration
     public final static String EAGLE_DB_USERNAME = "eagle.service.storage-username";

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9849303e/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java
b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java
index 15810b2..afcb31c 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java
@@ -43,7 +43,6 @@ public class JdbcEntityDefinitionManager {
 
     public static JdbcEntityDefinition getJdbcEntityDefinition(EntityDefinition entityDefinition){
         checkInit();
-
         Class<? extends TaggedLogAPIEntity> entityClass = entityDefinition.getEntityClass();
         JdbcEntityDefinition jdbcEntityDefinition = sqlEntityDefinitionCache.get(entityClass);
         if(jdbcEntityDefinition == null){
@@ -150,7 +149,7 @@ public class JdbcEntityDefinitionManager {
     // Intially bind basic java types with SQL types
     //================================================
     static {
-        registerJdbcType(String.class, Types.LONGVARCHAR);
+        registerJdbcType(String.class, Types.VARCHAR);
         registerJdbcType(Integer.class, Types.INTEGER);
         registerJdbcType(Double.class, Types.DOUBLE);
         registerJdbcType(Float.class, Types.FLOAT);

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9849303e/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntitySchemaManager.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntitySchemaManager.java
b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntitySchemaManager.java
index 4cd1967..60fd810 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntitySchemaManager.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntitySchemaManager.java
@@ -155,7 +155,7 @@ public class JdbcEntitySchemaManager implements IJdbcEntityDDLManager
{
         tagColumn.setTypeCode(Types.VARCHAR);
         tagColumn.setJavaName(tagName);
 //        tagColumn.setScale(1024);
-        tagColumn.setSize(String.valueOf(JdbcConstants.DEFAULT_VARCHAR_SIZE));
+        tagColumn.setSize(String.valueOf(JdbcConstants.DEFAULT_FIELD_VARCHAR_SIZE));
         tagColumn.setDefaultValue(null);
         tagColumn.setDescription("eagle entity tag column for "+tagName);
         return tagColumn;
@@ -196,7 +196,7 @@ public class JdbcEntitySchemaManager implements IJdbcEntityDDLManager
{
 //            Index index = new UniqueIndex();
             for (String tag : entityDefinition.getInternal().getTags()) {
                 Column tagColumn = createTagColumn(tag);
-                tagColumn.setSize(String.valueOf(JdbcConstants.DEFAULT_VARCHAR_SIZE));
+                tagColumn.setSize(String.valueOf(JdbcConstants.DEFAULT_TAG_VARCHAR_SIZE));
                 table.addColumn(tagColumn);
 //                IndexColumn indexColumn = new IndexColumn();
 //                indexColumn.setName(tag);
@@ -214,7 +214,7 @@ public class JdbcEntitySchemaManager implements IJdbcEntityDDLManager
{
             fieldColumn.setJavaName(entry.getKey());
             Integer typeCode = entityDefinition.getJdbcColumnTypeCodeOrNull(entry.getKey());
             typeCode = typeCode == null? Types.VARCHAR:typeCode;
-            if(typeCode == Types.VARCHAR) fieldColumn.setSize(String.valueOf(JdbcConstants.DEFAULT_VARCHAR_SIZE));
+            if(typeCode == Types.VARCHAR) fieldColumn.setSize(String.valueOf(JdbcConstants.DEFAULT_FIELD_VARCHAR_SIZE));
             fieldColumn.setTypeCode(typeCode);
             fieldColumn.setDescription("eagle field column "+entry.getKey()+":"+entityDefinition.getColumnTypeOrNull(entry.getKey()));
             table.addColumn(fieldColumn);

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9849303e/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java
b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java
index 005a61c..f33bc64 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java
@@ -54,6 +54,25 @@ public class TestJdbcStorage extends JdbcStorageTestBase {
         Assert.assertNotNull(result);
     }
 
+    /**
+     * To make sure not exception like Comparisons between 'LONG VARCHAR (UCS_BASIC)' and
'LONG VARCHAR (UCS_BASIC)' are not supported
+     *
+     * @throws QueryCompileException
+     * @throws IOException
+     */
+    @Test
+    public void testReadByStringTypedField() throws QueryCompileException, IOException {
+        RawQuery rawQuery = new RawQuery();
+        rawQuery.setQuery("TestTimeSeriesAPIEntity[@cluster=\"c4ut\" AND @field7 =\"some_field7_value\"]{*}");
+        System.out.println(DateTimeUtil.millisecondsToHumanDateWithSeconds(baseTimestamp));
+        rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(baseTimestamp));
+        rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithMilliseconds(baseTimestamp+2000));
+        rawQuery.setPageSize(1000);
+        CompiledQuery query = new CompiledQuery(rawQuery);
+        QueryResult<TestTimeSeriesAPIEntity> result = storage.query(query, entityDefinition);
+        Assert.assertNotNull(result);
+    }
+
     @Test
     public void testReadByNotEqualCondition() throws QueryCompileException, IOException {
         RawQuery rawQuery = new RawQuery();

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9849303e/eagle-examples/eagle-cassandra-example/bin/quick-start-with-external-webservice.sh
----------------------------------------------------------------------
diff --git a/eagle-examples/eagle-cassandra-example/bin/quick-start-with-external-webservice.sh
b/eagle-examples/eagle-cassandra-example/bin/quick-start-with-external-webservice.sh
new file mode 100755
index 0000000..e0646fe
--- /dev/null
+++ b/eagle-examples/eagle-cassandra-example/bin/quick-start-with-external-webservice.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export EAGLE_CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+export EAGLE_BASE_DIR=$EAGLE_CURRENT_DIR/../../..
+export EAGLE_ASSEMBLY_DIR=${EAGLE_BASE_DIR}/eagle-assembly/target/eagle-*-bin/eagle-*/
+
+ls ${EAGLE_ASSEMBLY_DIR} 1>/dev/null 2>/dev/null
+if [ "$?" != "0" ];then
+	echo "$EAGLE_ASSEMBLY_DIR not exist, build now"
+	cd $EAGLE_BASE_DIR
+	mvn package -DskipTests
+fi
+
+cd $EAGLE_ASSEMBLY_DIR/
+
+bin/eagle-topology-init.sh
+
+echo "Starting zookeeper"
+bin/zookeeper-server-start.sh -daemon conf/zookeeper-server.properties
+sleep 1
+
+echo "Starting kafka"
+bin/kafka-server-start.sh -daemon conf/kafka-server.properties
+sleep 1
+
+echo "Creating kafka topic: cassandra_querylog_local"
+bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic cassandra_querylog_local
--partitions 3 --replication-factor 1
+
+$EAGLE_CURRENT_DIR/init.sh
+$EAGLE_CURRENT_DIR/send-sample-querylog.sh
+$EAGLE_ASSEMBLY_DIR/bin/kafka-stream-monitor.sh cassandraQueryLogStream cassandraQueryLogExecutor
$EAGLE_CURRENT_DIR/../conf/cassandra-security-local.conf
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9849303e/eagle-hadoop-metric/src/main/resources/safemodecheck-policy-import.sh
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/safemodecheck-policy-import.sh b/eagle-hadoop-metric/src/main/resources/safemodecheck-policy-import.sh
index 32a6bee..1fb3831 100644
--- a/eagle-hadoop-metric/src/main/resources/safemodecheck-policy-import.sh
+++ b/eagle-hadoop-metric/src/main/resources/safemodecheck-policy-import.sh
@@ -48,4 +48,4 @@ curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:a
 echo ""
 echo "Finished initialization for eagle topology"
 
-exit 0
+exit 0
\ No newline at end of file


Mime
View raw message