Return-Path: X-Original-To: apmail-eagle-commits-archive@minotaur.apache.org Delivered-To: apmail-eagle-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4334518F7A for ; Tue, 22 Mar 2016 07:08:51 +0000 (UTC) Received: (qmail 63697 invoked by uid 500); 22 Mar 2016 07:08:51 -0000 Delivered-To: apmail-eagle-commits-archive@eagle.apache.org Received: (qmail 63667 invoked by uid 500); 22 Mar 2016 07:08:51 -0000 Mailing-List: contact commits-help@eagle.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@eagle.incubator.apache.org Delivered-To: mailing list commits@eagle.incubator.apache.org Received: (qmail 63658 invoked by uid 99); 22 Mar 2016 07:08:51 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd4-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 22 Mar 2016 07:08:51 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd4-us-west.apache.org (ASF Mail Server at spamd4-us-west.apache.org) with ESMTP id 90D22C0CF3 for ; Tue, 22 Mar 2016 07:08:50 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd4-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -3.221 X-Spam-Level: X-Spam-Status: No, score=-3.221 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-0.001] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd4-us-west.apache.org [10.40.0.11]) (amavisd-new, port 10024) with ESMTP id dZwOpkLplFrL for ; Tue, 22 Mar 2016 07:08:43 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id 772D85FB0F for ; Tue, 22 Mar 2016 07:08:41 +0000 (UTC) Received: (qmail 63513 invoked by uid 99); 22 Mar 2016 07:08:40 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 22 Mar 2016 07:08:40 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 81435DFBD6; Tue, 22 Mar 2016 07:08:40 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: hao@apache.org To: commits@eagle.incubator.apache.org Message-Id: <134d965645a64bd589ce5452ed72c084@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: incubator-eagle git commit: EAGLE-204 Support jdbc groupBy, timeseries, aggregation Date: Tue, 22 Mar 2016 07:08:40 +0000 (UTC) Repository: incubator-eagle Updated Branches: refs/heads/master c4bdb63f8 -> 0605b34fa EAGLE-204 Support jdbc groupBy, timeseries, aggregation https://issues.apache.org/jira/browse/EAGLE-204 Support timeseries metric query in front-end Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/0605b34f Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/0605b34f Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/0605b34f Branch: refs/heads/master Commit: 0605b34faaff218a847bb8b2b7ea9e2ae114b32f Parents: c4bdb63 Author: Hao Chen Authored: Tue Mar 22 14:45:54 2016 +0800 Committer: Hao Chen Committed: Tue Mar 22 15:07:51 2016 +0800 ---------------------------------------------------------------------- .../timeseries/AbstractAggregator.java | 8 ++ .../eagle/storage/operation/CompiledQuery.java | 14 ++ .../apache/eagle/storage/jdbc/JdbcStorage.java | 1 + .../criteria/impl/QueryCriteriaBuilder.java | 48 ++++--- .../jdbc/entity/JdbcEntitySerDeserHelper.java | 4 +- .../jdbc/entity/impl/JdbcEntityReaderImpl.java | 26 +++- .../jdbc/schema/JdbcEntityDefinition.java | 42 +++++- .../schema/JdbcEntityDefinitionManager.java | 7 +- .../schema/serializer/MetricJdbcSerDeser.java | 56 ++++++++ .../eagle/storage/jdbc/JdbcStorageTestBase.java | 46 +++++++ .../storage/jdbc/TestGenericMetricStorage.java | 129 +++++++++++++++++++ .../eagle/storage/jdbc/TestJdbcStorage.java | 23 +--- .../src/main/resources/application.conf | 7 +- .../app/public/feature/common/controller.js | 2 +- .../src/main/webapp/app/public/js/app.config.js | 4 +- 15 files changed, 366 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-query-base/src/main/java/org/apache/eagle/query/aggregate/timeseries/AbstractAggregator.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-query-base/src/main/java/org/apache/eagle/query/aggregate/timeseries/AbstractAggregator.java b/eagle-core/eagle-query/eagle-query-base/src/main/java/org/apache/eagle/query/aggregate/timeseries/AbstractAggregator.java index 0837382..deb0838 100755 --- a/eagle-core/eagle-query/eagle-query-base/src/main/java/org/apache/eagle/query/aggregate/timeseries/AbstractAggregator.java +++ b/eagle-core/eagle-query/eagle-query-base/src/main/java/org/apache/eagle/query/aggregate/timeseries/AbstractAggregator.java @@ -178,6 +178,14 @@ public abstract class AbstractAggregator implements Aggregator, EntityCreationLi return new Double(0.0); } } + if(obj instanceof double[]){ + double[] value = (double[]) obj; + if(value.length > 0){ + return new Double(value[0]); + }else{ + return new Double(0.0); + } + } throw new IllegalAggregateFieldTypeException(obj.getClass().toString() + " type is not support. The aggregated field must be numeric type, int, long or double"); } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java b/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java index eacdcca..34761cb 100644 --- a/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java +++ b/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java @@ -41,6 +41,7 @@ import java.util.Set; */ public class CompiledQuery { private final static Logger LOG = LoggerFactory.getLogger(CompiledQuery.class); + private boolean timeSeries; public boolean isHasAgg() { return hasAgg; @@ -202,6 +203,7 @@ public class CompiledQuery { this.sortFunctions = compiler.sortFunctions(); this.groupByFields = compiler.groupbyFields(); this.aggregateFields = compiler.aggregateFields(); + this.timeSeries = this.getRawQuery().isTimeSeries(); final List partitionValues = compiler.getQueryPartitionValues(); if (partitionValues != null) { @@ -255,4 +257,16 @@ public class CompiledQuery { } } } + + public boolean isTimeSeries() { + return timeSeries; + } + + public void setTimeSeries(boolean timeSeries) { + this.timeSeries = timeSeries; + } + + public long getIntervalMin(){ + return this.getRawQuery().getIntervalmin(); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcStorage.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcStorage.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcStorage.java index c490dcd..6dedbf4 100644 --- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcStorage.java +++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcStorage.java @@ -18,6 +18,7 @@ package org.apache.eagle.storage.jdbc; import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity; import org.apache.eagle.log.entity.meta.EntityDefinition; +import org.apache.eagle.query.aggregate.timeseries.TimeSeriesAggregator; import org.apache.eagle.storage.DataStorageBase; import org.apache.eagle.storage.jdbc.conn.ConnectionManagerFactory; import org.apache.eagle.storage.jdbc.entity.JdbcEntityDeleter; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/criteria/impl/QueryCriteriaBuilder.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/criteria/impl/QueryCriteriaBuilder.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/criteria/impl/QueryCriteriaBuilder.java index a82d185..9687cad 100644 --- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/criteria/impl/QueryCriteriaBuilder.java +++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/criteria/impl/QueryCriteriaBuilder.java @@ -58,17 +58,32 @@ public class QueryCriteriaBuilder implements CriteriaBuilder { Criteria root = new Criteria(); SearchCondition searchCondition = query.getSearchCondition(); - // SELECT + + if(query.isHasAgg()){ - List aggFields = query.getAggregateFields(); - List aggFuncs = query.getAggregateFunctionTypes(); - for(int i=0;i aggFields = query.getAggregateFields(); + for(String field:aggFields){ + root.addSelectColumn(new ColumnImpl(this.tableName,field)); + } + + List groupByFieldsFields = query.getGroupByFields(); + for(String field:groupByFieldsFields){ + root.addSelectColumn(new ColumnImpl(this.tableName,field)); + } + } else { + List aggFields = query.getAggregateFields(); + List aggFuncs = query.getAggregateFunctionTypes(); + for (int i = 0; i < aggFuncs.size(); i++) { + AggregateFunctionType aggFunc = aggFuncs.get(i); + String aggField = aggFields.get(i); + if (aggFunc.equals(AggregateFunctionType.count)) { + root.addSelectColumn(new ColumnImpl(aggFunc.name() + "(*)")); + } else { + root.addSelectColumn(new ColumnImpl(String.format("%s(%s.%s)", aggFunc.name(), this.tableName, aggField))); + } } } } else if(searchCondition.isOutputAll()){ @@ -88,12 +103,6 @@ public class QueryCriteriaBuilder implements CriteriaBuilder { } } - // If no columns are specified, then select * by default - if(root.getSelectColumns() == null || root.getSelectColumns().size() ==0){ - // SELECT * - root.addSelectColumn(new ColumnImpl(this.tableName, "*")); - } - // FROM $tableName root.addFrom(this.tableName); @@ -119,7 +128,7 @@ public class QueryCriteriaBuilder implements CriteriaBuilder { } // TODO: GROUP BY - if(query.isHasAgg()){ + if(query.isHasAgg() && !query.isTimeSeries()){ for(String groupByField:query.getGroupByFields()){ root.addGroupByColumn(new ColumnImpl(this.tableName,groupByField)); } @@ -127,6 +136,11 @@ public class QueryCriteriaBuilder implements CriteriaBuilder { // TODO: ORDER BY + // If no columns are specified, then select * by default + if(root.getSelectColumns() == null || root.getSelectColumns().size() ==0){ + // SELECT * + root.addSelectColumn(new ColumnImpl(this.tableName, "*")); + } return root; } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/JdbcEntitySerDeserHelper.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/JdbcEntitySerDeserHelper.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/JdbcEntitySerDeserHelper.java index 3750f71..a4816dd 100644 --- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/JdbcEntitySerDeserHelper.java +++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/JdbcEntitySerDeserHelper.java @@ -22,7 +22,6 @@ import org.apache.eagle.log.entity.meta.EntityDefinition; import org.apache.eagle.log.entity.meta.Qualifier; import org.apache.eagle.storage.jdbc.JdbcConstants; import org.apache.eagle.storage.jdbc.schema.JdbcEntityDefinition; -import org.apache.eagle.storage.jdbc.schema.JdbcEntityDefinitionManager; import org.apache.eagle.storage.jdbc.schema.serializer.JdbcSerDeser; import org.apache.commons.beanutils.PropertyUtils; import org.apache.torque.ColumnImpl; @@ -211,6 +210,7 @@ public class JdbcEntitySerDeserHelper { throw new IOException(String.format("No field %s in entity %s", columnName, entityDefinition.getInternal().getEntityClass()), e); } }else{ + columnName = entityDefinition.getOriginalJavaTagName(columnName); // treat as tag or others value = resultSet.getObject(columnName); } @@ -251,7 +251,7 @@ public class JdbcEntitySerDeserHelper { Object fieldValue = getMethod.invoke(entity); Class fieldType = qualifier.getSerDeser().type(); - JdbcSerDeser jdbcSerDeser = JdbcEntityDefinitionManager.getJdbcSerDeser(fieldType); + JdbcSerDeser jdbcSerDeser = jdbcEntityDefinition.getJdbcSerDeser(displayName); JdbcTypedValue jdbcTypedValue = jdbcSerDeser.toJdbcTypedValue(fieldValue, fieldType, qualifier); columnValues.put(new ColumnImpl(tableName,displayName),jdbcTypedValue); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/impl/JdbcEntityReaderImpl.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/impl/JdbcEntityReaderImpl.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/impl/JdbcEntityReaderImpl.java index a7b93b5..5e6ca9a 100644 --- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/impl/JdbcEntityReaderImpl.java +++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/impl/JdbcEntityReaderImpl.java @@ -16,6 +16,9 @@ */ package org.apache.eagle.storage.jdbc.entity.impl; +import com.google.common.collect.Lists; +import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity; +import org.apache.eagle.query.aggregate.timeseries.TimeSeriesAggregator; import org.apache.eagle.storage.jdbc.conn.ConnectionManagerFactory; import org.apache.eagle.storage.jdbc.conn.impl.TorqueStatementPeerImpl; import org.apache.eagle.storage.jdbc.criteria.impl.PrimaryKeyCriteriaBuilder; @@ -32,7 +35,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * @since 3/27/15 @@ -57,7 +62,7 @@ public class JdbcEntityReaderImpl implements JdbcEntityReader { if(LOG.isDebugEnabled()) LOG.debug("Querying: " + displaySql); RecordMapper recordMapper; - if(query.isHasAgg()) { + if(query.isHasAgg() && !query.isTimeSeries()) { recordMapper = (RecordMapper) new AggreagteRecordMapper(query, jdbcEntityDefinition); }else{ recordMapper = new EntityRecordMapper(jdbcEntityDefinition); @@ -69,6 +74,9 @@ public class JdbcEntityReaderImpl implements JdbcEntityReader { TorqueStatementPeerImpl peer = ConnectionManagerFactory.getInstance().getStatementExecutor(); result = peer.delegate().doSelect(criteria, recordMapper); LOG.info(String.format("Read %s records in %s ms (sql: %s)",result.size(),stopWatch.getTime(),displaySql)); + if(result.size() > 0 && query.isTimeSeries()){ + result = Lists.newArrayList((E) timeseriesAggregate(result, query)); + } }catch (Exception ex){ LOG.error("Failed to query by: "+displaySql+", due to: "+ex.getMessage(),ex); throw new IOException("Failed to query by: "+displaySql,ex); @@ -78,6 +86,22 @@ public class JdbcEntityReaderImpl implements JdbcEntityReader { return result; } + private Map timeseriesAggregate(List result, CompiledQuery query) throws Exception { + TimeSeriesAggregator aggregator = new TimeSeriesAggregator(query.getGroupByFields(), + query.getAggregateFunctionTypes(), + query.getAggregateFields(), + query.getStartTime(), query.getEndTime(), + query.getIntervalMin() + ); + for(E entity: result) + aggregator.accumulate((TaggedLogAPIEntity) entity); + if(this.jdbcEntityDefinition.isGenericMetric()) { + return aggregator.getMetric(); + } else { + return aggregator.result(); + } + } + @Override public List query(List ids) throws Exception { PrimaryKeyCriteriaBuilder criteriaBuilder = new PrimaryKeyCriteriaBuilder(ids,this.jdbcEntityDefinition.getJdbcTableName()); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinition.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinition.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinition.java index eb5e874..aa95934 100644 --- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinition.java +++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinition.java @@ -48,7 +48,7 @@ public class JdbcEntityDefinition { * @return jdbc table name in lowercase */ public String getJdbcTableName(){ - if(this.internal.getService().equals(GenericMetricEntity.GENERIC_METRIC_SERVICE)){ + if(isGenericMetric()){ return this.internal.getTable().toLowerCase(); }else { return String.format("%s_%s", this.internal.getTable(),this.internal.getPrefix()).toLowerCase(); @@ -56,11 +56,11 @@ public class JdbcEntityDefinition { } public Class getColumnType(String fieldName) throws NoSuchFieldException { - if (fieldName.equals(JdbcConstants.TIMESTAMP_COLUMN_NAME)){ + if (fieldName.equalsIgnoreCase(JdbcConstants.TIMESTAMP_COLUMN_NAME)){ return Long.class; - }else if(fieldName.equals(JdbcConstants.ROW_KEY_COLUMN_NAME)) { + }else if(fieldName.equalsIgnoreCase(JdbcConstants.ROW_KEY_COLUMN_NAME)) { return String.class; - }else if(fieldName.equals(JdbcConstants.METRIC_NAME_COLUMN_NAME)){ + }else if(fieldName.equalsIgnoreCase(JdbcConstants.METRIC_NAME_COLUMN_NAME)){ return String.class; } for(String realField:internal.getDisplayNameMap().keySet()){ @@ -80,6 +80,9 @@ public class JdbcEntityDefinition { } public Integer getJdbcColumnTypeCodeOrNull(String fieldName){ + if(this.isGenericMetric() && GenericMetricEntity.VALUE_FIELD.equalsIgnoreCase(fieldName)){ + return JdbcEntityDefinitionManager.getJdbcType(double.class); + } Class columnType; try { columnType = getColumnType(fieldName); @@ -91,6 +94,10 @@ public class JdbcEntityDefinition { @SuppressWarnings("unchecked") public JdbcSerDeser getJdbcSerDeser(String columnName) { + if(this.isGenericMetric() && GenericMetricEntity.VALUE_FIELD.equalsIgnoreCase(columnName)){ + return JdbcEntityDefinitionManager.METRIC_JDBC_SERDESER; + } + Qualifier qualifier = this.getColumnQualifier(columnName); if(qualifier == null){ return JdbcEntityDefinitionManager.DEFAULT_JDBC_SERDESER; @@ -112,6 +119,33 @@ public class JdbcEntityDefinition { return false; } + /** + * TODO: Optimize with hashmap + */ + public String getJavaEntityFieldName(String jdbcColumnName){ + for(String javaEntityFieldName:this.internal.getDisplayNameMap().keySet()){ + if(javaEntityFieldName.equalsIgnoreCase(jdbcColumnName)){ + return javaEntityFieldName; + } + } + throw new IllegalArgumentException("Can't map jdbc column '"+jdbcColumnName+"' with entity: "+this.getInternal().getEntityClass()); + } + + /** + * TODO: Optimize with hashmap + * + * @param jdbcTagName + * @return + */ + public String getOriginalJavaTagName(String jdbcTagName){ + for(String tag:this.getInternal().getTags()){ + if(tag.equalsIgnoreCase(jdbcTagName)){ + return tag; + } + } + return jdbcTagName.toLowerCase(); + } + public Qualifier getColumnQualifier(String columnName) { for(Map.Entry entry:this.internal.getDisplayNameMap().entrySet()){ if(entry.getKey().equalsIgnoreCase(columnName)){ http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java index cf163fb..418f552 100644 --- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java +++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java @@ -22,6 +22,7 @@ import org.apache.eagle.log.entity.meta.EntityDefinitionManager; import org.apache.eagle.log.entity.meta.EntitySerDeser; import org.apache.eagle.storage.jdbc.schema.serializer.JdbcSerDeser; import org.apache.eagle.storage.jdbc.schema.serializer.DefaultJdbcSerDeser; +import org.apache.eagle.storage.jdbc.schema.serializer.MetricJdbcSerDeser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -85,7 +86,8 @@ public class JdbcEntityDefinitionManager { checkInit(); } - public static DefaultJdbcSerDeser DEFAULT_JDBC_SERDESER = new DefaultJdbcSerDeser(); + public static JdbcSerDeser DEFAULT_JDBC_SERDESER = new DefaultJdbcSerDeser(); + public static JdbcSerDeser METRIC_JDBC_SERDESER = new MetricJdbcSerDeser(); private final static Map,JdbcSerDeser> _columnTypeSerDeserMapping = new HashMap, JdbcSerDeser>(); /** @@ -124,7 +126,8 @@ public class JdbcEntityDefinitionManager { if(fieldType == null){ return Types.NULL; } else if(!_classJdbcType.containsKey(fieldType)){ - LOG.debug("Unable to locate simple jdbc type for: {}, return type as JAVA_OBJECT",fieldType); + if(LOG.isDebugEnabled()) + LOG.debug("Unable to locate simple jdbc type for: {}, return type as JAVA_OBJECT",fieldType); return Types.JAVA_OBJECT; } return _classJdbcType.get(fieldType); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/serializer/MetricJdbcSerDeser.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/serializer/MetricJdbcSerDeser.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/serializer/MetricJdbcSerDeser.java new file mode 100644 index 0000000..8162955 --- /dev/null +++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/serializer/MetricJdbcSerDeser.java @@ -0,0 +1,56 @@ +package org.apache.eagle.storage.jdbc.schema.serializer; + +import org.apache.eagle.log.entity.meta.Qualifier; +import org.apache.eagle.storage.jdbc.JdbcConstants; +import org.apache.eagle.storage.jdbc.schema.JdbcEntityDefinitionManager; +import org.apache.torque.util.JdbcTypedValue; + +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public class MetricJdbcSerDeser implements JdbcSerDeser { + @Override + public double[] toJavaTypedValue(ResultSet result, Class fieldType, String fieldName, Qualifier qualifier) throws IOException { + try { + return new double[]{result.getDouble(fieldName)}; + } catch (SQLException e) { + throw new IOException("Generic Metric Field: "+fieldName+", java type:"+fieldType,e); + } + } + + /** + * + * @param fieldValue + * @param fieldType + * @return + */ + @Override + public JdbcTypedValue toJdbcTypedValue(Object fieldValue, Class fieldType, Qualifier qualifier) { + double[] metricFieldValues = (double[]) fieldValue; + if(metricFieldValues == null || metricFieldValues.length == 0){ + return null; + } else if(metricFieldValues.length > 1){ + throw new IllegalArgumentException("Not support metric value length > 1: "+fieldType); + }else{ + return new JdbcTypedValue(metricFieldValues[0],Types.DOUBLE); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/JdbcStorageTestBase.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/JdbcStorageTestBase.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/JdbcStorageTestBase.java new file mode 100644 index 0000000..4cee46f --- /dev/null +++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/JdbcStorageTestBase.java @@ -0,0 +1,46 @@ +package org.apache.eagle.storage.jdbc; + +import org.apache.eagle.log.entity.meta.EntityDefinition; +import org.apache.eagle.log.entity.meta.EntityDefinitionManager; +import org.apache.eagle.log.entity.test.TestTimeSeriesAPIEntity; +import org.apache.eagle.storage.DataStorageManager; +import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.GregorianCalendar; +import java.util.TimeZone; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public class JdbcStorageTestBase { + JdbcStorage storage; + long baseTimestamp; + final static Logger LOG = LoggerFactory.getLogger(TestJdbcStorage.class); + + @Before + public void setUp() throws Exception { + storage = (JdbcStorage) DataStorageManager.getDataStorageByEagleConfig(); + storage.init(); + GregorianCalendar gc = new GregorianCalendar(); + gc.clear(); + gc.set(2014, 1, 6, 1, 40, 12); + gc.setTimeZone(TimeZone.getTimeZone("UTC")); + baseTimestamp = gc.getTime().getTime(); + System.out.println("timestamp:" + baseTimestamp); + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestGenericMetricStorage.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestGenericMetricStorage.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestGenericMetricStorage.java new file mode 100644 index 0000000..524b294 --- /dev/null +++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestGenericMetricStorage.java @@ -0,0 +1,129 @@ +package org.apache.eagle.storage.jdbc; + +import junit.framework.Assert; +import org.apache.eagle.common.DateTimeUtil; +import org.apache.eagle.log.entity.GenericMetricEntity; +import org.apache.eagle.log.entity.meta.EntityDefinition; +import org.apache.eagle.log.entity.meta.EntityDefinitionManager; +import org.apache.eagle.storage.exception.QueryCompileException; +import org.apache.eagle.storage.operation.CompiledQuery; +import org.apache.eagle.storage.operation.RawQuery; +import org.apache.eagle.storage.result.ModifyResult; +import org.apache.eagle.storage.result.QueryResult; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Random; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public class TestGenericMetricStorage extends JdbcStorageTestBase { + + EntityDefinition entityDefinition; + Random random = new Random(); + String metricName = "unittest.metric.name"; + + @Override + public void setUp() throws Exception { + entityDefinition = EntityDefinitionManager.getEntityDefinitionByEntityClass(GenericMetricEntity.class); + entityDefinition.setTags(new String[]{"site","application"}); + super.setUp(); + } + + private GenericMetricEntity newMetric(){ + GenericMetricEntity instance = new GenericMetricEntity(); + instance.setPrefix(metricName); + instance.setTags(new HashMap() {{ + put("site", "unittest_site"); + put("application", "unittest_application"); + }}); + instance.setValue(new double[]{random.nextDouble()}); + instance.setTimestamp(System.currentTimeMillis()); + return instance; + } + + @Test + public void testWrite1000Metrics() throws InterruptedException, IOException { + // Write 1000 entities + List entityList = new ArrayList<>(); + int i= 0; + while( i++ < 1000){ + entityList.add(newMetric()); + Thread.sleep(1); + } + ModifyResult result = storage.create(entityList, entityDefinition); + Assert.assertTrue(result.getSize() >= 1000); + } + + @Test + public void testSimpleRead() throws IOException, QueryCompileException, InterruptedException { + // record insert init time + long startTime = System.currentTimeMillis(); + // Write 1000 entities + testWrite1000Metrics(); + // record insertion finish time + long endTime = System.currentTimeMillis(); + // init read in time range [startTime, endTime) + RawQuery rawQuery = new RawQuery(); + rawQuery.setQuery(GenericMetricEntity.GENERIC_METRIC_SERVICE+"[]{*}"); + rawQuery.setMetricName(metricName); + rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime)); + rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime+1000)); + rawQuery.setPageSize(10000); + CompiledQuery query = new CompiledQuery(rawQuery); + QueryResult queryResult = storage.query(query, entityDefinition); + Assert.assertTrue(queryResult.getSize() >= 1000); + } + + @Test + public void testSimpleGroupAggregateRead() throws IOException, InterruptedException, QueryCompileException { + long startTime = System.currentTimeMillis(); + testWrite1000Metrics(); + long endTime = System.currentTimeMillis(); + RawQuery rawQuery = new RawQuery(); + rawQuery.setQuery(GenericMetricEntity.GENERIC_METRIC_SERVICE+"[@site=\"unittest_site\" AND @application=\"unittest_application\"]<@site>{sum(value)}"); + rawQuery.setMetricName(metricName); + rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime)); + rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime+1000)); + rawQuery.setPageSize(10000); + CompiledQuery query = new CompiledQuery(rawQuery); + QueryResult queryResult = storage.query(query, entityDefinition); + Assert.assertTrue(queryResult.getSize() >= 1); + } + + + @Test + public void testTimeSeriesGroupAggregateRead() throws IOException, InterruptedException, QueryCompileException { + long startTime = System.currentTimeMillis(); + testWrite1000Metrics(); + long endTime = System.currentTimeMillis(); + RawQuery rawQuery = new RawQuery(); + rawQuery.setQuery(GenericMetricEntity.GENERIC_METRIC_SERVICE+"[@site=\"unittest_site\" AND @application=\"unittest_application\"]<@site>{sum(value)}"); + rawQuery.setMetricName(metricName); + rawQuery.setTimeSeries(true); + rawQuery.setIntervalmin(10); + rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime)); + rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime + 10*60*1000)); + rawQuery.setPageSize(10000); + CompiledQuery query = new CompiledQuery(rawQuery); + QueryResult queryResult = storage.query(query, entityDefinition); + Assert.assertTrue(queryResult.getSize() >= 1); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java index 0b4178e..cb0f0a7 100644 --- a/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java +++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java @@ -22,38 +22,23 @@ import org.apache.eagle.common.DateTimeUtil; import org.apache.eagle.log.entity.meta.EntityDefinition; import org.apache.eagle.log.entity.meta.EntityDefinitionManager; import org.apache.eagle.log.entity.test.TestTimeSeriesAPIEntity; -import org.apache.eagle.storage.DataStorageManager; import org.apache.eagle.storage.exception.QueryCompileException; import org.apache.eagle.storage.operation.CompiledQuery; import org.apache.eagle.storage.operation.RawQuery; import org.apache.eagle.storage.result.ModifyResult; import org.apache.eagle.storage.result.QueryResult; -import org.junit.Before; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.*; -public class TestJdbcStorage { - JdbcStorage storage; +public class TestJdbcStorage extends JdbcStorageTestBase { EntityDefinition entityDefinition; - long baseTimestamp; - final static Logger LOG = LoggerFactory.getLogger(TestJdbcStorage.class); - - @Before + @Override public void setUp() throws Exception { entityDefinition = EntityDefinitionManager.getEntityDefinitionByEntityClass(TestTimeSeriesAPIEntity.class); entityDefinition.setTags(new String[]{"cluster","datacenter","random"}); - storage = (JdbcStorage) DataStorageManager.getDataStorageByEagleConfig(); - storage.init(); - GregorianCalendar gc = new GregorianCalendar(); - gc.clear(); - gc.set(2014, 1, 6, 1, 40, 12); - gc.setTimeZone(TimeZone.getTimeZone("UTC")); - baseTimestamp = gc.getTime().getTime(); - System.out.println("timestamp:" + baseTimestamp); + super.setUp(); } @Test @@ -243,8 +228,10 @@ public class TestJdbcStorage { return instance; } + @Test public void testInitSuccessfully() { } + } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-webservice/src/main/resources/application.conf ---------------------------------------------------------------------- diff --git a/eagle-webservice/src/main/resources/application.conf b/eagle-webservice/src/main/resources/application.conf index 23a7067..61a899f 100644 --- a/eagle-webservice/src/main/resources/application.conf +++ b/eagle-webservice/src/main/resources/application.conf @@ -16,14 +16,13 @@ eagle { service { storage-type="jdbc" - storage-adapter="derby" + storage-adapter="mysql" storage-username="eagle" storage-password=eagle storage-database=eagle - # Derby database location: $TOMCAT_HOME/data/eagle - storage-connection-url="jdbc:derby:../data/eagle;create=true" + storage-connection-url="jdbc:mysql://localhost:3306/eagle" storage-connection-props="encoding=UTF-8" - storage-driver-class="org.apache.derby.jdbc.EmbeddedDriver" + storage-driver-class="com.mysql.jdbc.Driver" storage-connection-max=8 } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-webservice/src/main/webapp/app/public/feature/common/controller.js ---------------------------------------------------------------------- diff --git a/eagle-webservice/src/main/webapp/app/public/feature/common/controller.js b/eagle-webservice/src/main/webapp/app/public/feature/common/controller.js index d87ec6d..b758431 100644 --- a/eagle-webservice/src/main/webapp/app/public/feature/common/controller.js +++ b/eagle-webservice/src/main/webapp/app/public/feature/common/controller.js @@ -181,7 +181,7 @@ ]; function _loadSeries(seriesName, metricName, condition) { - var list = Entities.querySeries("GenericMetricService", $.extend({_metricName: metricName}, condition), "@cluster", "sum(value)", _intervalList[_intervalType][1]); + var list = Entities.querySeries("GenericMetricService", $.extend({_metricName: metricName}, condition), "@site", "sum(value)", _intervalList[_intervalType][1]); var seriesList = nvd3.convert.eagle([list]); if(!$scope[seriesName]) $scope[seriesName] = seriesList; list._promise.then(function() { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-webservice/src/main/webapp/app/public/js/app.config.js ---------------------------------------------------------------------- diff --git a/eagle-webservice/src/main/webapp/app/public/js/app.config.js b/eagle-webservice/src/main/webapp/app/public/js/app.config.js index 876114f..4392f71 100644 --- a/eagle-webservice/src/main/webapp/app/public/js/app.config.js +++ b/eagle-webservice/src/main/webapp/app/public/js/app.config.js @@ -32,8 +32,8 @@ deleteEntity: 'rest/entities/delete?serviceName=${serviceName}&byId=true', deleteEntities: 'rest/entities?query=${serviceName}[${condition}]{*}&pageSize=100000', - queryGroup: 'rest/list?query=${serviceName}[${condition}]<${groupBy}>{${values}}&pageSize=100000', - querySeries: 'rest/list?query=${serviceName}[${condition}]<${groupBy}>{${values}}&pageSize=100000&timeSeries=true&intervalmin=${intervalmin}', + queryGroup: 'rest/entities?query=${serviceName}[${condition}]<${groupBy}>{${values}}&pageSize=100000', + querySeries: 'rest/entities?query=${serviceName}[${condition}]<${groupBy}>{${values}}&pageSize=100000&timeSeries=true&intervalmin=${intervalmin}', query: 'rest/',