eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h..@apache.org
Subject incubator-eagle git commit: EAGLE-204 Support jdbc groupBy, timeseries, aggregation
Date Tue, 22 Mar 2016 07:08:40 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master c4bdb63f8 -> 0605b34fa


EAGLE-204 Support jdbc groupBy, timeseries, aggregation

https://issues.apache.org/jira/browse/EAGLE-204

Support timeseries metric query in front-end


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/0605b34f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/0605b34f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/0605b34f

Branch: refs/heads/master
Commit: 0605b34faaff218a847bb8b2b7ea9e2ae114b32f
Parents: c4bdb63
Author: Hao Chen <hao@apache.org>
Authored: Tue Mar 22 14:45:54 2016 +0800
Committer: Hao Chen <hao@apache.org>
Committed: Tue Mar 22 15:07:51 2016 +0800

----------------------------------------------------------------------
 .../timeseries/AbstractAggregator.java          |   8 ++
 .../eagle/storage/operation/CompiledQuery.java  |  14 ++
 .../apache/eagle/storage/jdbc/JdbcStorage.java  |   1 +
 .../criteria/impl/QueryCriteriaBuilder.java     |  48 ++++---
 .../jdbc/entity/JdbcEntitySerDeserHelper.java   |   4 +-
 .../jdbc/entity/impl/JdbcEntityReaderImpl.java  |  26 +++-
 .../jdbc/schema/JdbcEntityDefinition.java       |  42 +++++-
 .../schema/JdbcEntityDefinitionManager.java     |   7 +-
 .../schema/serializer/MetricJdbcSerDeser.java   |  56 ++++++++
 .../eagle/storage/jdbc/JdbcStorageTestBase.java |  46 +++++++
 .../storage/jdbc/TestGenericMetricStorage.java  | 129 +++++++++++++++++++
 .../eagle/storage/jdbc/TestJdbcStorage.java     |  23 +---
 .../src/main/resources/application.conf         |   7 +-
 .../app/public/feature/common/controller.js     |   2 +-
 .../src/main/webapp/app/public/js/app.config.js |   4 +-
 15 files changed, 366 insertions(+), 51 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-query-base/src/main/java/org/apache/eagle/query/aggregate/timeseries/AbstractAggregator.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-query-base/src/main/java/org/apache/eagle/query/aggregate/timeseries/AbstractAggregator.java
b/eagle-core/eagle-query/eagle-query-base/src/main/java/org/apache/eagle/query/aggregate/timeseries/AbstractAggregator.java
index 0837382..deb0838 100755
--- a/eagle-core/eagle-query/eagle-query-base/src/main/java/org/apache/eagle/query/aggregate/timeseries/AbstractAggregator.java
+++ b/eagle-core/eagle-query/eagle-query-base/src/main/java/org/apache/eagle/query/aggregate/timeseries/AbstractAggregator.java
@@ -178,6 +178,14 @@ public abstract class AbstractAggregator implements Aggregator, EntityCreationLi
 				return new Double(0.0);
 			}
 		}
+		if(obj instanceof double[]){
+			double[] value = (double[]) obj;
+			if(value.length > 0){
+				return new Double(value[0]);
+			}else{
+				return new Double(0.0);
+			}
+		}
 		
 		throw new IllegalAggregateFieldTypeException(obj.getClass().toString() + " type is not
support. The aggregated field must be numeric type, int, long or double");
 	}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java
b/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java
index eacdcca..34761cb 100644
--- a/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java
+++ b/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java
@@ -41,6 +41,7 @@ import java.util.Set;
  */
 public class CompiledQuery {
     private final static Logger LOG = LoggerFactory.getLogger(CompiledQuery.class);
+    private boolean timeSeries;
 
     public boolean isHasAgg() {
         return hasAgg;
@@ -202,6 +203,7 @@ public class CompiledQuery {
         this.sortFunctions = compiler.sortFunctions();
         this.groupByFields = compiler.groupbyFields();
         this.aggregateFields = compiler.aggregateFields();
+        this.timeSeries = this.getRawQuery().isTimeSeries();
 
         final List<String[]> partitionValues = compiler.getQueryPartitionValues();
         if (partitionValues != null) {
@@ -255,4 +257,16 @@ public class CompiledQuery {
             }
         }
     }
+
+    public boolean isTimeSeries() {
+        return timeSeries;
+    }
+
+    public void setTimeSeries(boolean timeSeries) {
+        this.timeSeries = timeSeries;
+    }
+
+    public long getIntervalMin(){
+        return this.getRawQuery().getIntervalmin();
+    }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcStorage.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcStorage.java
b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcStorage.java
index c490dcd..6dedbf4 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcStorage.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcStorage.java
@@ -18,6 +18,7 @@ package org.apache.eagle.storage.jdbc;
 
 import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
 import org.apache.eagle.log.entity.meta.EntityDefinition;
+import org.apache.eagle.query.aggregate.timeseries.TimeSeriesAggregator;
 import org.apache.eagle.storage.DataStorageBase;
 import org.apache.eagle.storage.jdbc.conn.ConnectionManagerFactory;
 import org.apache.eagle.storage.jdbc.entity.JdbcEntityDeleter;

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/criteria/impl/QueryCriteriaBuilder.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/criteria/impl/QueryCriteriaBuilder.java
b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/criteria/impl/QueryCriteriaBuilder.java
index a82d185..9687cad 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/criteria/impl/QueryCriteriaBuilder.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/criteria/impl/QueryCriteriaBuilder.java
@@ -58,17 +58,32 @@ public class QueryCriteriaBuilder implements CriteriaBuilder {
         Criteria root = new Criteria();
         SearchCondition searchCondition = query.getSearchCondition();
 
-        // SELECT
+
+
         if(query.isHasAgg()){
-            List<String> aggFields = query.getAggregateFields();
-            List<AggregateFunctionType> aggFuncs = query.getAggregateFunctionTypes();
-            for(int i=0;i<aggFuncs.size();i++){
-                AggregateFunctionType aggFunc = aggFuncs.get(i);
-                String aggField = aggFields.get(i);
-                if(aggFunc.equals(AggregateFunctionType.count)){
-                    root.addSelectColumn(new ColumnImpl(aggFunc.name()+"(*)"));
-                }else{
-                    root.addSelectColumn(new ColumnImpl(String.format("%s(%s.%s)",aggFunc.name(),this.tableName,aggField)));
+            if(this.jdbcEntityDefinition.getInternal().isTimeSeries() && query.isTimeSeries())
{
+                // SELECT
+                root.addSelectColumn(new ColumnImpl(this.tableName,JdbcConstants.TIMESTAMP_COLUMN_NAME));
+                List<String> aggFields = query.getAggregateFields();
+                for(String field:aggFields){
+                    root.addSelectColumn(new ColumnImpl(this.tableName,field));
+                }
+
+                List<String> groupByFieldsFields = query.getGroupByFields();
+                for(String field:groupByFieldsFields){
+                    root.addSelectColumn(new ColumnImpl(this.tableName,field));
+                }
+            } else {
+                List<String> aggFields = query.getAggregateFields();
+                List<AggregateFunctionType> aggFuncs = query.getAggregateFunctionTypes();
+                for (int i = 0; i < aggFuncs.size(); i++) {
+                    AggregateFunctionType aggFunc = aggFuncs.get(i);
+                    String aggField = aggFields.get(i);
+                    if (aggFunc.equals(AggregateFunctionType.count)) {
+                        root.addSelectColumn(new ColumnImpl(aggFunc.name() + "(*)"));
+                    } else {
+                        root.addSelectColumn(new ColumnImpl(String.format("%s(%s.%s)", aggFunc.name(),
this.tableName, aggField)));
+                    }
                 }
             }
         } else if(searchCondition.isOutputAll()){
@@ -88,12 +103,6 @@ public class QueryCriteriaBuilder implements CriteriaBuilder {
             }
         }
 
-        // If no columns are specified, then select * by default
-        if(root.getSelectColumns() == null || root.getSelectColumns().size() ==0){
-            // SELECT *
-            root.addSelectColumn(new ColumnImpl(this.tableName, "*"));
-        }
-
         // FROM $tableName
         root.addFrom(this.tableName);
 
@@ -119,7 +128,7 @@ public class QueryCriteriaBuilder implements CriteriaBuilder {
         }
 
         // TODO: GROUP BY
-        if(query.isHasAgg()){
+        if(query.isHasAgg() && !query.isTimeSeries()){
             for(String groupByField:query.getGroupByFields()){
                 root.addGroupByColumn(new ColumnImpl(this.tableName,groupByField));
             }
@@ -127,6 +136,11 @@ public class QueryCriteriaBuilder implements CriteriaBuilder {
 
         // TODO: ORDER BY
 
+        // If no columns are specified, then select * by default
+        if(root.getSelectColumns() == null || root.getSelectColumns().size() ==0){
+            // SELECT *
+            root.addSelectColumn(new ColumnImpl(this.tableName, "*"));
+        }
         return root;
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/JdbcEntitySerDeserHelper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/JdbcEntitySerDeserHelper.java
b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/JdbcEntitySerDeserHelper.java
index 3750f71..a4816dd 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/JdbcEntitySerDeserHelper.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/JdbcEntitySerDeserHelper.java
@@ -22,7 +22,6 @@ import org.apache.eagle.log.entity.meta.EntityDefinition;
 import org.apache.eagle.log.entity.meta.Qualifier;
 import org.apache.eagle.storage.jdbc.JdbcConstants;
 import org.apache.eagle.storage.jdbc.schema.JdbcEntityDefinition;
-import org.apache.eagle.storage.jdbc.schema.JdbcEntityDefinitionManager;
 import org.apache.eagle.storage.jdbc.schema.serializer.JdbcSerDeser;
 import org.apache.commons.beanutils.PropertyUtils;
 import org.apache.torque.ColumnImpl;
@@ -211,6 +210,7 @@ public class JdbcEntitySerDeserHelper {
                     throw new IOException(String.format("No field %s in entity %s", columnName,
entityDefinition.getInternal().getEntityClass()), e);
                 }
             }else{
+                columnName = entityDefinition.getOriginalJavaTagName(columnName);
                 // treat as tag or others
                 value = resultSet.getObject(columnName);
             }
@@ -251,7 +251,7 @@ public class JdbcEntitySerDeserHelper {
             Object fieldValue = getMethod.invoke(entity);
 
             Class<?> fieldType = qualifier.getSerDeser().type();
-            JdbcSerDeser jdbcSerDeser = JdbcEntityDefinitionManager.getJdbcSerDeser(fieldType);
+            JdbcSerDeser jdbcSerDeser = jdbcEntityDefinition.getJdbcSerDeser(displayName);
 
             JdbcTypedValue jdbcTypedValue = jdbcSerDeser.toJdbcTypedValue(fieldValue, fieldType,
qualifier);
             columnValues.put(new ColumnImpl(tableName,displayName),jdbcTypedValue);

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/impl/JdbcEntityReaderImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/impl/JdbcEntityReaderImpl.java
b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/impl/JdbcEntityReaderImpl.java
index a7b93b5..5e6ca9a 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/impl/JdbcEntityReaderImpl.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/impl/JdbcEntityReaderImpl.java
@@ -16,6 +16,9 @@
  */
 package org.apache.eagle.storage.jdbc.entity.impl;
 
+import com.google.common.collect.Lists;
+import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
+import org.apache.eagle.query.aggregate.timeseries.TimeSeriesAggregator;
 import org.apache.eagle.storage.jdbc.conn.ConnectionManagerFactory;
 import org.apache.eagle.storage.jdbc.conn.impl.TorqueStatementPeerImpl;
 import org.apache.eagle.storage.jdbc.criteria.impl.PrimaryKeyCriteriaBuilder;
@@ -32,7 +35,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 /**
  * @since 3/27/15
@@ -57,7 +62,7 @@ public class JdbcEntityReaderImpl implements JdbcEntityReader {
         if(LOG.isDebugEnabled()) LOG.debug("Querying: " + displaySql);
 
         RecordMapper<E> recordMapper;
-        if(query.isHasAgg()) {
+        if(query.isHasAgg() && !query.isTimeSeries()) {
             recordMapper = (RecordMapper<E>) new AggreagteRecordMapper(query, jdbcEntityDefinition);
         }else{
             recordMapper = new EntityRecordMapper(jdbcEntityDefinition);
@@ -69,6 +74,9 @@ public class JdbcEntityReaderImpl implements JdbcEntityReader {
             TorqueStatementPeerImpl peer = ConnectionManagerFactory.getInstance().getStatementExecutor();
             result = peer.delegate().doSelect(criteria, recordMapper);
             LOG.info(String.format("Read %s records in %s ms (sql: %s)",result.size(),stopWatch.getTime(),displaySql));
+            if(result.size() > 0 && query.isTimeSeries()){
+                    result = Lists.newArrayList((E) timeseriesAggregate(result, query));
+            }
         }catch (Exception ex){
             LOG.error("Failed to query by: "+displaySql+", due to: "+ex.getMessage(),ex);
             throw new IOException("Failed to query by: "+displaySql,ex);
@@ -78,6 +86,22 @@ public class JdbcEntityReaderImpl implements JdbcEntityReader {
         return result;
     }
 
+    private <E> Map timeseriesAggregate(List<E> result, CompiledQuery query)
throws Exception {
+        TimeSeriesAggregator aggregator = new TimeSeriesAggregator(query.getGroupByFields(),
+            query.getAggregateFunctionTypes(),
+            query.getAggregateFields(),
+            query.getStartTime(), query.getEndTime(),
+            query.getIntervalMin()
+        );
+        for(E entity: result)
+            aggregator.accumulate((TaggedLogAPIEntity) entity);
+        if(this.jdbcEntityDefinition.isGenericMetric()) {
+            return aggregator.getMetric();
+        } else {
+            return aggregator.result();
+        }
+    }
+
     @Override
     public <E> List<E> query(List<String> ids) throws Exception {
         PrimaryKeyCriteriaBuilder criteriaBuilder = new PrimaryKeyCriteriaBuilder(ids,this.jdbcEntityDefinition.getJdbcTableName());

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinition.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinition.java
b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinition.java
index eb5e874..aa95934 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinition.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinition.java
@@ -48,7 +48,7 @@ public class JdbcEntityDefinition {
      * @return jdbc table name in lowercase
      */
     public String getJdbcTableName(){
-        if(this.internal.getService().equals(GenericMetricEntity.GENERIC_METRIC_SERVICE)){
+        if(isGenericMetric()){
             return this.internal.getTable().toLowerCase();
         }else {
             return String.format("%s_%s", this.internal.getTable(),this.internal.getPrefix()).toLowerCase();
@@ -56,11 +56,11 @@ public class JdbcEntityDefinition {
     }
 
     public Class<?> getColumnType(String fieldName) throws NoSuchFieldException {
-        if (fieldName.equals(JdbcConstants.TIMESTAMP_COLUMN_NAME)){
+        if (fieldName.equalsIgnoreCase(JdbcConstants.TIMESTAMP_COLUMN_NAME)){
             return Long.class;
-        }else if(fieldName.equals(JdbcConstants.ROW_KEY_COLUMN_NAME)) {
+        }else if(fieldName.equalsIgnoreCase(JdbcConstants.ROW_KEY_COLUMN_NAME)) {
             return String.class;
-        }else if(fieldName.equals(JdbcConstants.METRIC_NAME_COLUMN_NAME)){
+        }else if(fieldName.equalsIgnoreCase(JdbcConstants.METRIC_NAME_COLUMN_NAME)){
             return String.class;
         }
         for(String realField:internal.getDisplayNameMap().keySet()){
@@ -80,6 +80,9 @@ public class JdbcEntityDefinition {
     }
 
     public Integer getJdbcColumnTypeCodeOrNull(String fieldName){
+        if(this.isGenericMetric() && GenericMetricEntity.VALUE_FIELD.equalsIgnoreCase(fieldName)){
+            return JdbcEntityDefinitionManager.getJdbcType(double.class);
+        }
         Class<?> columnType;
         try {
             columnType = getColumnType(fieldName);
@@ -91,6 +94,10 @@ public class JdbcEntityDefinition {
 
     @SuppressWarnings("unchecked")
     public JdbcSerDeser getJdbcSerDeser(String columnName) {
+        if(this.isGenericMetric() && GenericMetricEntity.VALUE_FIELD.equalsIgnoreCase(columnName)){
+            return JdbcEntityDefinitionManager.METRIC_JDBC_SERDESER;
+        }
+
         Qualifier qualifier = this.getColumnQualifier(columnName);
         if(qualifier == null){
             return JdbcEntityDefinitionManager.DEFAULT_JDBC_SERDESER;
@@ -112,6 +119,33 @@ public class JdbcEntityDefinition {
         return false;
     }
 
+    /**
+     * TODO: Optimize with hashmap
+     */
+     public String getJavaEntityFieldName(String jdbcColumnName){
+        for(String javaEntityFieldName:this.internal.getDisplayNameMap().keySet()){
+            if(javaEntityFieldName.equalsIgnoreCase(jdbcColumnName)){
+                return javaEntityFieldName;
+            }
+        }
+        throw new IllegalArgumentException("Can't map jdbc column '"+jdbcColumnName+"' with
entity: "+this.getInternal().getEntityClass());
+    }
+
+    /**
+     * TODO: Optimize with hashmap
+     *
+     * @param jdbcTagName
+     * @return
+     */
+    public String getOriginalJavaTagName(String jdbcTagName){
+        for(String tag:this.getInternal().getTags()){
+            if(tag.equalsIgnoreCase(jdbcTagName)){
+                return tag;
+            }
+        }
+        return jdbcTagName.toLowerCase();
+    }
+
     public Qualifier getColumnQualifier(String columnName) {
         for(Map.Entry<String,Qualifier> entry:this.internal.getDisplayNameMap().entrySet()){
             if(entry.getKey().equalsIgnoreCase(columnName)){

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java
b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java
index cf163fb..418f552 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java
@@ -22,6 +22,7 @@ import org.apache.eagle.log.entity.meta.EntityDefinitionManager;
 import org.apache.eagle.log.entity.meta.EntitySerDeser;
 import org.apache.eagle.storage.jdbc.schema.serializer.JdbcSerDeser;
 import org.apache.eagle.storage.jdbc.schema.serializer.DefaultJdbcSerDeser;
+import org.apache.eagle.storage.jdbc.schema.serializer.MetricJdbcSerDeser;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -85,7 +86,8 @@ public class JdbcEntityDefinitionManager {
         checkInit();
     }
 
-    public static DefaultJdbcSerDeser DEFAULT_JDBC_SERDESER = new DefaultJdbcSerDeser();
+    public static JdbcSerDeser DEFAULT_JDBC_SERDESER = new DefaultJdbcSerDeser();
+    public static JdbcSerDeser METRIC_JDBC_SERDESER = new MetricJdbcSerDeser();
     private final static Map<Class<?>,JdbcSerDeser> _columnTypeSerDeserMapping
= new HashMap<Class<?>, JdbcSerDeser>();
 
     /**
@@ -124,7 +126,8 @@ public class JdbcEntityDefinitionManager {
         if(fieldType == null){
             return Types.NULL;
         } else if(!_classJdbcType.containsKey(fieldType)){
-            LOG.debug("Unable to locate simple jdbc type for: {}, return type as JAVA_OBJECT",fieldType);
+            if(LOG.isDebugEnabled())
+                LOG.debug("Unable to locate simple jdbc type for: {}, return type as JAVA_OBJECT",fieldType);
             return Types.JAVA_OBJECT;
         }
         return _classJdbcType.get(fieldType);

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/serializer/MetricJdbcSerDeser.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/serializer/MetricJdbcSerDeser.java
b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/serializer/MetricJdbcSerDeser.java
new file mode 100644
index 0000000..8162955
--- /dev/null
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/serializer/MetricJdbcSerDeser.java
@@ -0,0 +1,56 @@
+package org.apache.eagle.storage.jdbc.schema.serializer;
+
+import org.apache.eagle.log.entity.meta.Qualifier;
+import org.apache.eagle.storage.jdbc.JdbcConstants;
+import org.apache.eagle.storage.jdbc.schema.JdbcEntityDefinitionManager;
+import org.apache.torque.util.JdbcTypedValue;
+
+import java.io.IOException;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Types;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class MetricJdbcSerDeser implements JdbcSerDeser<double[]> {
+    @Override
+    public double[] toJavaTypedValue(ResultSet result, Class<?> fieldType, String fieldName,
Qualifier qualifier) throws IOException {
+        try {
+            return new double[]{result.getDouble(fieldName)};
+        } catch (SQLException e) {
+            throw new IOException("Generic Metric Field: "+fieldName+", java type:"+fieldType,e);
+        }
+    }
+
+    /**
+     *
+     * @param fieldValue
+     * @param fieldType
+     * @return
+     */
+    @Override
+    public JdbcTypedValue toJdbcTypedValue(Object fieldValue, Class<?> fieldType, Qualifier
qualifier) {
+        double[] metricFieldValues = (double[]) fieldValue;
+        if(metricFieldValues == null || metricFieldValues.length == 0){
+            return null;
+        } else if(metricFieldValues.length > 1){
+            throw new IllegalArgumentException("Not support metric value length > 1: "+fieldType);
+        }else{
+            return new JdbcTypedValue(metricFieldValues[0],Types.DOUBLE);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/JdbcStorageTestBase.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/JdbcStorageTestBase.java
b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/JdbcStorageTestBase.java
new file mode 100644
index 0000000..4cee46f
--- /dev/null
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/JdbcStorageTestBase.java
@@ -0,0 +1,46 @@
+package org.apache.eagle.storage.jdbc;
+
+import org.apache.eagle.log.entity.meta.EntityDefinition;
+import org.apache.eagle.log.entity.meta.EntityDefinitionManager;
+import org.apache.eagle.log.entity.test.TestTimeSeriesAPIEntity;
+import org.apache.eagle.storage.DataStorageManager;
+import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.GregorianCalendar;
+import java.util.TimeZone;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class JdbcStorageTestBase {
+    JdbcStorage storage;
+    long baseTimestamp;
+    final static Logger LOG = LoggerFactory.getLogger(TestJdbcStorage.class);
+
+    @Before
+    public void setUp() throws Exception {
+        storage = (JdbcStorage) DataStorageManager.getDataStorageByEagleConfig();
+        storage.init();
+        GregorianCalendar gc = new GregorianCalendar();
+        gc.clear();
+        gc.set(2014, 1, 6, 1, 40, 12);
+        gc.setTimeZone(TimeZone.getTimeZone("UTC"));
+        baseTimestamp = gc.getTime().getTime();
+        System.out.println("timestamp:" + baseTimestamp);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestGenericMetricStorage.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestGenericMetricStorage.java
b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestGenericMetricStorage.java
new file mode 100644
index 0000000..524b294
--- /dev/null
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestGenericMetricStorage.java
@@ -0,0 +1,129 @@
+package org.apache.eagle.storage.jdbc;
+
+import junit.framework.Assert;
+import org.apache.eagle.common.DateTimeUtil;
+import org.apache.eagle.log.entity.GenericMetricEntity;
+import org.apache.eagle.log.entity.meta.EntityDefinition;
+import org.apache.eagle.log.entity.meta.EntityDefinitionManager;
+import org.apache.eagle.storage.exception.QueryCompileException;
+import org.apache.eagle.storage.operation.CompiledQuery;
+import org.apache.eagle.storage.operation.RawQuery;
+import org.apache.eagle.storage.result.ModifyResult;
+import org.apache.eagle.storage.result.QueryResult;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class TestGenericMetricStorage extends JdbcStorageTestBase {
+
+    EntityDefinition entityDefinition;
+    Random random = new Random();
+    String metricName = "unittest.metric.name";
+
+    @Override
+    public void setUp() throws Exception {
+        entityDefinition = EntityDefinitionManager.getEntityDefinitionByEntityClass(GenericMetricEntity.class);
+        entityDefinition.setTags(new String[]{"site","application"});
+        super.setUp();
+    }
+
+    private GenericMetricEntity newMetric(){
+        GenericMetricEntity instance = new GenericMetricEntity();
+        instance.setPrefix(metricName);
+        instance.setTags(new HashMap<String, String>() {{
+            put("site", "unittest_site");
+            put("application", "unittest_application");
+        }});
+        instance.setValue(new double[]{random.nextDouble()});
+        instance.setTimestamp(System.currentTimeMillis());
+        return instance;
+    }
+
+    @Test
+    public void testWrite1000Metrics() throws InterruptedException, IOException {
+        // Write 1000 entities
+        List<GenericMetricEntity> entityList = new ArrayList<>();
+        int i= 0;
+        while( i++ < 1000){
+            entityList.add(newMetric());
+            Thread.sleep(1);
+        }
+        ModifyResult<String> result = storage.create(entityList, entityDefinition);
+        Assert.assertTrue(result.getSize() >= 1000);
+    }
+
+    @Test
+    public void testSimpleRead() throws IOException, QueryCompileException, InterruptedException
{
+        // record insert init time
+        long startTime = System.currentTimeMillis();
+        // Write 1000 entities
+        testWrite1000Metrics();
+        // record insertion finish time
+        long endTime = System.currentTimeMillis();
+        // init read in time range [startTime, endTime)
+        RawQuery rawQuery = new RawQuery();
+        rawQuery.setQuery(GenericMetricEntity.GENERIC_METRIC_SERVICE+"[]{*}");
+        rawQuery.setMetricName(metricName);
+        rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime));
+        rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime+1000));
+        rawQuery.setPageSize(10000);
+        CompiledQuery query = new CompiledQuery(rawQuery);
+        QueryResult queryResult = storage.query(query, entityDefinition);
+        Assert.assertTrue(queryResult.getSize() >= 1000);
+    }
+
+    @Test
+    public void testSimpleGroupAggregateRead() throws IOException, InterruptedException,
QueryCompileException {
+        long startTime = System.currentTimeMillis();
+        testWrite1000Metrics();
+        long endTime = System.currentTimeMillis();
+        RawQuery rawQuery = new RawQuery();
+        rawQuery.setQuery(GenericMetricEntity.GENERIC_METRIC_SERVICE+"[@site=\"unittest_site\"
AND @application=\"unittest_application\"]<@site>{sum(value)}");
+        rawQuery.setMetricName(metricName);
+        rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime));
+        rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime+1000));
+        rawQuery.setPageSize(10000);
+        CompiledQuery query = new CompiledQuery(rawQuery);
+        QueryResult queryResult = storage.query(query, entityDefinition);
+        Assert.assertTrue(queryResult.getSize() >= 1);
+    }
+
+
+    @Test
+    public void testTimeSeriesGroupAggregateRead() throws IOException, InterruptedException,
QueryCompileException {
+        long startTime = System.currentTimeMillis();
+        testWrite1000Metrics();
+        long endTime = System.currentTimeMillis();
+        RawQuery rawQuery = new RawQuery();
+        rawQuery.setQuery(GenericMetricEntity.GENERIC_METRIC_SERVICE+"[@site=\"unittest_site\"
AND @application=\"unittest_application\"]<@site>{sum(value)}");
+        rawQuery.setMetricName(metricName);
+        rawQuery.setTimeSeries(true);
+        rawQuery.setIntervalmin(10);
+        rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime));
+        rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime + 10*60*1000));
+        rawQuery.setPageSize(10000);
+        CompiledQuery query = new CompiledQuery(rawQuery);
+        QueryResult queryResult = storage.query(query, entityDefinition);
+        Assert.assertTrue(queryResult.getSize() >= 1);
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java
b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java
index 0b4178e..cb0f0a7 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java
@@ -22,38 +22,23 @@ import org.apache.eagle.common.DateTimeUtil;
 import org.apache.eagle.log.entity.meta.EntityDefinition;
 import org.apache.eagle.log.entity.meta.EntityDefinitionManager;
 import org.apache.eagle.log.entity.test.TestTimeSeriesAPIEntity;
-import org.apache.eagle.storage.DataStorageManager;
 import org.apache.eagle.storage.exception.QueryCompileException;
 import org.apache.eagle.storage.operation.CompiledQuery;
 import org.apache.eagle.storage.operation.RawQuery;
 import org.apache.eagle.storage.result.ModifyResult;
 import org.apache.eagle.storage.result.QueryResult;
-import org.junit.Before;
 import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.*;
 
-public class TestJdbcStorage {
-    JdbcStorage storage;
+public class TestJdbcStorage extends JdbcStorageTestBase {
     EntityDefinition entityDefinition;
-    long baseTimestamp;
-    final static Logger LOG = LoggerFactory.getLogger(TestJdbcStorage.class);
-
-    @Before
+    @Override
     public void setUp() throws Exception {
         entityDefinition = EntityDefinitionManager.getEntityDefinitionByEntityClass(TestTimeSeriesAPIEntity.class);
         entityDefinition.setTags(new String[]{"cluster","datacenter","random"});
-        storage = (JdbcStorage) DataStorageManager.getDataStorageByEagleConfig();
-        storage.init();
-        GregorianCalendar gc = new GregorianCalendar();
-        gc.clear();
-        gc.set(2014, 1, 6, 1, 40, 12);
-        gc.setTimeZone(TimeZone.getTimeZone("UTC"));
-        baseTimestamp = gc.getTime().getTime();
-        System.out.println("timestamp:" + baseTimestamp);
+        super.setUp();
     }
 
     @Test
@@ -243,8 +228,10 @@ public class TestJdbcStorage {
         return instance;
     }
 
+
     @Test
     public void testInitSuccessfully() {
 
     }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-webservice/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-webservice/src/main/resources/application.conf b/eagle-webservice/src/main/resources/application.conf
index 23a7067..61a899f 100644
--- a/eagle-webservice/src/main/resources/application.conf
+++ b/eagle-webservice/src/main/resources/application.conf
@@ -16,14 +16,13 @@
 eagle {
 	service {
 		storage-type="jdbc"
-		storage-adapter="derby"
+		storage-adapter="mysql"
 		storage-username="eagle"
 		storage-password=eagle
 		storage-database=eagle
-		# Derby database location: $TOMCAT_HOME/data/eagle
-		storage-connection-url="jdbc:derby:../data/eagle;create=true"
+		storage-connection-url="jdbc:mysql://localhost:3306/eagle"
 		storage-connection-props="encoding=UTF-8"
-		storage-driver-class="org.apache.derby.jdbc.EmbeddedDriver"
+		storage-driver-class="com.mysql.jdbc.Driver"
 		storage-connection-max=8
 	}
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-webservice/src/main/webapp/app/public/feature/common/controller.js
----------------------------------------------------------------------
diff --git a/eagle-webservice/src/main/webapp/app/public/feature/common/controller.js b/eagle-webservice/src/main/webapp/app/public/feature/common/controller.js
index d87ec6d..b758431 100644
--- a/eagle-webservice/src/main/webapp/app/public/feature/common/controller.js
+++ b/eagle-webservice/src/main/webapp/app/public/feature/common/controller.js
@@ -181,7 +181,7 @@
 			];
 
 			function _loadSeries(seriesName, metricName, condition) {
-				var list = Entities.querySeries("GenericMetricService", $.extend({_metricName: metricName},
condition), "@cluster", "sum(value)", _intervalList[_intervalType][1]);
+				var list = Entities.querySeries("GenericMetricService", $.extend({_metricName: metricName},
condition), "@site", "sum(value)", _intervalList[_intervalType][1]);
 				var seriesList = nvd3.convert.eagle([list]);
 				if(!$scope[seriesName]) $scope[seriesName] = seriesList;
 				list._promise.then(function() {

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-webservice/src/main/webapp/app/public/js/app.config.js
----------------------------------------------------------------------
diff --git a/eagle-webservice/src/main/webapp/app/public/js/app.config.js b/eagle-webservice/src/main/webapp/app/public/js/app.config.js
index 876114f..4392f71 100644
--- a/eagle-webservice/src/main/webapp/app/public/js/app.config.js
+++ b/eagle-webservice/src/main/webapp/app/public/js/app.config.js
@@ -32,8 +32,8 @@
 			deleteEntity: 'rest/entities/delete?serviceName=${serviceName}&byId=true',
 			deleteEntities: 'rest/entities?query=${serviceName}[${condition}]{*}&pageSize=100000',
 
-			queryGroup: 'rest/list?query=${serviceName}[${condition}]<${groupBy}>{${values}}&pageSize=100000',
-			querySeries: 'rest/list?query=${serviceName}[${condition}]<${groupBy}>{${values}}&pageSize=100000&timeSeries=true&intervalmin=${intervalmin}',
+			queryGroup: 'rest/entities?query=${serviceName}[${condition}]<${groupBy}>{${values}}&pageSize=100000',
+			querySeries: 'rest/entities?query=${serviceName}[${condition}]<${groupBy}>{${values}}&pageSize=100000&timeSeries=true&intervalmin=${intervalmin}',
 
 			query: 'rest/',
 



Mime
View raw message