eagle-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (EAGLE-928) Refine system metric schema design and fix system metric collector
Date Tue, 28 Feb 2017 06:33:45 GMT

    [ https://issues.apache.org/jira/browse/EAGLE-928?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15887396#comment-15887396
] 

ASF GitHub Bot commented on EAGLE-928:
--------------------------------------

Github user haoch commented on a diff in the pull request:

    https://github.com/apache/eagle/pull/842#discussion_r103380038
  
    --- Diff: eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/builder/MetricDescriptor.java
---
    @@ -1,201 +1,279 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one or more
    - * contributor license agreements.  See the NOTICE file distributed with
    - * this work for additional information regarding copyright ownership.
    - * The ASF licenses this file to You under the Apache License, Version 2.0
    - * (the "License"); you may not use this file except in compliance with
    - * the License.  You may obtain a copy of the License at
    - * <p>
    - * http://www.apache.org/licenses/LICENSE-2.0
    - * <p>
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - */
    -package org.apache.eagle.app.environment.builder;
    -
    -
    -import java.io.Serializable;
    -import java.util.Arrays;
    -import java.util.Calendar;
    -import java.util.List;
    -import java.util.Map;
    -
    -public class MetricDefinition implements Serializable {
    -
    -    /**
    -     * Support simple and complex name format, by default using "metric" field.
    -     */
    -    private NameSelector nameSelector = new FieldNameSelector("metric");
    -
    -    /**
    -     * Support event/system time, by default using system time.
    -     */
    -    private TimestampSelector timestampSelector = new SystemTimestampSelector();
    -
    -    /**
    -     * Metric dimension field name.
    -     */
    -    private List<String> dimensionFields;
    -
    -    /**
    -     * Metric granularity.
    -     */
    -    private int granularity = Calendar.MINUTE;
    -
    -    private String metricType = "DEFAULT";
    -
    -    /**
    -     * Metric value field name.
    -     */
    -    private String valueField = "value";
    -
    -    public NameSelector getNameSelector() {
    -        return nameSelector;
    -    }
    -
    -    public void setNameSelector(NameSelector nameSelector) {
    -        this.nameSelector = nameSelector;
    -    }
    -
    -    public String getValueField() {
    -        return valueField;
    -    }
    -
    -    public void setValueField(String valueField) {
    -        this.valueField = valueField;
    -    }
    -
    -    public List<String> getDimensionFields() {
    -        return dimensionFields;
    -    }
    -
    -    public void setDimensionFields(List<String> dimensionFields) {
    -        this.dimensionFields = dimensionFields;
    -    }
    -
    -    public TimestampSelector getTimestampSelector() {
    -        return timestampSelector;
    -    }
    -
    -    public void setTimestampSelector(TimestampSelector timestampSelector) {
    -        this.timestampSelector = timestampSelector;
    -    }
    -
    -    public int getGranularity() {
    -        return granularity;
    -    }
    -
    -    public void setGranularity(int granularity) {
    -        this.granularity = granularity;
    -    }
    -
    -    public String getMetricType() {
    -        return metricType;
    -    }
    -
    -    public void setMetricType(String metricType) {
    -        this.metricType = metricType;
    -    }
    -
    -
    -    @FunctionalInterface
    -    public interface NameSelector extends Serializable {
    -        String getMetricName(Map event);
    -    }
    -
    -    @FunctionalInterface
    -    public interface TimestampSelector extends Serializable {
    -        Long getTimestamp(Map event);
    -    }
    -
    -    public MetricDefinition namedBy(NameSelector nameSelector) {
    -        this.setNameSelector(nameSelector);
    -        return this;
    -    }
    -
    -    /**
    -     * @see java.util.Calendar
    -     */
    -    public MetricDefinition granularity(int granularity) {
    -        this.setGranularity(granularity);
    -        return this;
    -    }
    -
    -    public MetricDefinition namedByField(String nameField) {
    -        this.setNameSelector(new FieldNameSelector(nameField));
    -        return this;
    -    }
    -
    -    public static MetricDefinition metricType(String metricType) {
    -        MetricDefinition metricDefinition = new MetricDefinition();
    -        metricDefinition.setMetricType(metricType);
    -        return metricDefinition;
    -    }
    -
    -    public MetricDefinition eventTimeByField(String timestampField) {
    -        this.setTimestampSelector(new EventTimestampSelector(timestampField));
    -        return this;
    -    }
    -
    -    public MetricDefinition dimensionFields(String... dimensionFields) {
    -        this.setDimensionFields(Arrays.asList(dimensionFields));
    -        return this;
    -    }
    -
    -    public MetricDefinition valueField(String valueField) {
    -        this.setValueField(valueField);
    -        return this;
    -    }
    -
    -    public class EventTimestampSelector implements TimestampSelector {
    -        private final String timestampField;
    -
    -        EventTimestampSelector(String timestampField) {
    -            this.timestampField = timestampField;
    -        }
    -
    -        @Override
    -        public Long getTimestamp(Map event) {
    -            if (event.containsKey(timestampField)) {
    -                Object timestampValue = event.get(timestampField);
    -                if (timestampValue instanceof Integer) {
    -                    return Long.valueOf((Integer) timestampValue);
    -                }
    -                if (timestampValue instanceof String) {
    -                    return Long.valueOf((String) timestampValue);
    -                } else {
    -                    return (Long) timestampValue;
    -                }
    -            } else {
    -                throw new IllegalArgumentException("Timestamp field '" + timestampField
+ "' not exists");
    -            }
    -        }
    -    }
    -
    -    public static class SystemTimestampSelector implements TimestampSelector {
    -        @Override
    -        public Long getTimestamp(Map event) {
    -            return System.currentTimeMillis();
    -        }
    -    }
    -
    -    public static class FieldNameSelector implements NameSelector {
    -        private final String fieldName;
    -
    -        FieldNameSelector(String fieldName) {
    -            this.fieldName = fieldName;
    -        }
    -
    -        @Override
    -        public String getMetricName(Map event) {
    -            if (event.containsKey(fieldName)) {
    -                return (String) event.get(fieldName);
    -            } else {
    -                throw new IllegalArgumentException("Metric name field '" + fieldName
+ "' not exists: " + event);
    -            }
    -        }
    -    }
    +/*
    --- End diff --
    
    `dos2unix`


> Refine system metric schema design and fix system metric collector
> ------------------------------------------------------------------
>
>                 Key: EAGLE-928
>                 URL: https://issues.apache.org/jira/browse/EAGLE-928
>             Project: Eagle
>          Issue Type: Bug
>    Affects Versions: v0.5.0
>            Reporter: Hao Chen
>            Assignee: Hao Chen
>             Fix For: v0.5.0
>
>
> Refine system metric schema design and fix system metric collector
> h1. Principle
> Any metric (no mater sys metric) should at least have three fields: "metric", "group",
"timestamp", "value", 
> "group" is the metric category path separated with slash, for example "FIRST CATEGORY.SECOND
CATEGORY", which will be commonly used for well organizing metric groups in customized metric
dashboard.
> h1. Stream Name for System Metric
> {code}
> <stream>
>             <streamId>SYSTEM_METRIC_STREAM</streamId>
>             <description>System Metrics Stream including CPU, Network, Disk, etc.</description>
>             <columns>
>                 <column>
>                     <name>host</name>
>                     <type>string</type>
>                 </column>
>                 <column>
>                     <name>timestamp</name>
>                     <type>long</type>
>                 </column>
>                 <column>
>                     <name>metric</name>
>                     <type>string</type>
>                 </column>
>                 <column>
>                     <name>group</name>
>                     <type>string</type>
>                 </column>
>                 <column>
>                     <name>site</name>
>                     <type>string</type>
>                 </column>
>                 <column>
>                     <name>device</name>
>                     <type>string</type>
>                 </column>
>                 <column>
>                     <name>value</name>
>                     <type>double</type>
>                     <defaultValue>0.0</defaultValue>
>                 </column>
>             </columns>
>         </stream>
> {code}
> h1. Stream Schema for System metric
>     
>        * *metric*: [STRING] metric name string
>        * *group*: [STRING] metric group/type
>        * *timestamp*: [LONG] metric generation time
>        * *site*: [STRING] siteId
>        * *host*: [STRING] source host name
>        * *device*: [STRING] device name, like cpu
> h1. Sample CPU Metric
> {code}
> {
>     "timestamp": 1487918913569, 
>     "metric": "system.cpu.usage", 
>     "group": "system.cpu",
>     "site": "sandbox", 
>     "value": 0.058, 
>     "host": "sandbox.hortonworks.com", 
>     "device": "cpu7"
> }
> {code}
> h1. Sample Network Metrics
> {code}
> {
>     "timestamp": 1487918913569, 
>     "metric": "system.nic.transmitdrop", 
>     "group": "system.network",
>     "site": "sandbox", 
>     "value": 7724.0, 
>     "host": "sandbox.hortonworks.com", 
>     "device": "eth0"
> }
> {code}
> h1. Sample Metric Schema
> {code}
> {
>             "tags": {
>                 "site": "sandbox",
>                 "name": "system.memory.nfs_unstable.kb",
>                 "group": "system.memory"
>             },
>             "dimensionFields": [
>                 "host",
>                 "group",
>                 "site",
>                 "device"
>             ],
>             "metricFields": [
>                 "value"
>             ],
>             "granularity": "MINUTE",
>             "modifiedTimestamp": 1488190388479
>         }
> {code}
> h1. Stream Persist Topology
> {code}
>         environment.newApp(config)
>                 .fromStream("HADOOP_JMX_METRIC_STREAM")
>                 .saveAsMetric(
>                         MetricDescriptor.metricGroupAs((MetricGroupSelector) event ->
{
>                             if (event.containsKey("component")) {
>                                 return String.format("hadoop.%s", ((String) event.get("component")).toLowerCase());
>                             } else {
>                                 return "hadoop.metrics";
>                             }
>                         })
>                         .siteByField("site")
>                         .namedByField("metric")
>                         .eventTimeByField("timestamp")
>                         .dimensionFields("host", "component", "site")
>                         .granularity(Calendar.MINUTE)
>                         .valueField("value"))
>                 .fromStream("SYSTEM_METRIC_STREAM")
>                 .saveAsMetric(MetricDescriptor.metricGroupByField("group")
>                         .siteByField("site")
>                         .namedByField("metric")
>                         .eventTimeByField("timestamp")
>                         .dimensionFields("host", "group", "site", "device")
>                         .granularity(Calendar.MINUTE)
>                         .valueField("value")
>                 )
>                 .toTopology();
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Mime
View raw message