eagle-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (EAGLE-928) Refine system metric schema design and fix system metric collector
Date Tue, 28 Feb 2017 06:33:45 GMT

    [ https://issues.apache.org/jira/browse/EAGLE-928?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15887397#comment-15887397
] 

ASF GitHub Bot commented on EAGLE-928:
--------------------------------------

Github user haoch commented on a diff in the pull request:

    https://github.com/apache/eagle/pull/842#discussion_r103380011
  
    --- Diff: eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/builder/ApplicationBuilder.java
---
    @@ -1,127 +1,131 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one or more
    - * contributor license agreements.  See the NOTICE file distributed with
    - * this work for additional information regarding copyright ownership.
    - * The ASF licenses this file to You under the Apache License, Version 2.0
    - * (the "License"); you may not use this file except in compliance with
    - * the License.  You may obtain a copy of the License at
    - * <p>
    - * http://www.apache.org/licenses/LICENSE-2.0
    - * <p>
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - */
    -package org.apache.eagle.app.environment.builder;
    -
    -
    -import backtype.storm.generated.StormTopology;
    -import backtype.storm.topology.TopologyBuilder;
    -import backtype.storm.tuple.Fields;
    -import com.google.common.base.Preconditions;
    -import com.typesafe.config.Config;
    -import org.apache.eagle.app.environment.impl.StormEnvironment;
    -import org.apache.eagle.app.messaging.MetricSchemaGenerator;
    -import org.apache.eagle.app.messaging.MetricStreamPersist;
    -import org.apache.eagle.app.messaging.StormStreamSource;
    -
    -import java.util.concurrent.atomic.AtomicInteger;
    -
    -/**
    - * Storm Application Builder DSL.
    - */
    -public class ApplicationBuilder {
    -    private final StormEnvironment environment;
    -    private final Config appConfig;
    -    private final TopologyBuilder topologyBuilder;
    -    private final AtomicInteger identifier;
    -
    -    public ApplicationBuilder(Config appConfig, StormEnvironment environment) {
    -        this.appConfig = appConfig;
    -        this.environment = environment;
    -        this.identifier = new AtomicInteger(0);
    -        this.topologyBuilder = new TopologyBuilder();
    -    }
    -
    -    public class BuilderContext {
    -        public StormTopology toTopology() {
    -            return topologyBuilder.createTopology();
    -        }
    -    }
    -
    -    public abstract class InitializedStream extends BuilderContext {
    -        private String id;
    -
    -        InitializedStream(String id) {
    -            Preconditions.checkNotNull(id);
    -            this.id = id;
    -        }
    -
    -        String getId() {
    -            return this.id;
    -        }
    -
    -        /**
    -         * Persist source data stream as metric.
    -         */
    -        public BuilderContext saveAsMetric(MetricDefinition metricDefinition) {
    -            String metricDataID = generateId("MetricDataSink");
    -            String metricSchemaID = generateId("MetricSchemaGenerator");
    -            topologyBuilder.setBolt(metricDataID, new MetricStreamPersist(metricDefinition,
appConfig)).shuffleGrouping(getId());
    -            topologyBuilder.setBolt(metricSchemaID, new MetricSchemaGenerator(metricDefinition,appConfig)).fieldsGrouping(metricDataID,new
Fields(MetricStreamPersist.METRIC_NAME_FIELD));
    -            return this;
    -        }
    -
    -        public TransformedStream transformBy(TransformFunction function) {
    -            String componentId = generateId(function.getName());
    -            topologyBuilder.setBolt(componentId, new TransformFunctionBolt(function)).shuffleGrouping(getId());
    -            return new TransformedStream(componentId);
    -        }
    -    }
    -
    -    public class SourcedStream extends InitializedStream {
    -        private final Config appConfig;
    -        private final StormStreamSource streamSource;
    -
    -        private SourcedStream(SourcedStream withSourcedStream) {
    -            this(withSourcedStream.getId(), withSourcedStream.appConfig, withSourcedStream.streamSource);
    -        }
    -
    -        private SourcedStream(String componentId, Config appConfig, StormStreamSource
streamSource) {
    -            super(componentId);
    -            this.appConfig = appConfig;
    -            this.streamSource = streamSource;
    -            topologyBuilder.setSpout(componentId, streamSource);
    -        }
    -    }
    -
    -    public class TransformedStream extends InitializedStream {
    -        public TransformedStream(String id) {
    -            super(id);
    -            throw new IllegalStateException("TODO: Not implemented yet");
    -        }
    -    }
    -
    -    public TopologyBuilder getTopologyBuilder() {
    -        return this.topologyBuilder;
    -    }
    -
    -    public StormTopology createTopology() {
    -        return topologyBuilder.createTopology();
    -    }
    -
    -
    -    public SourcedStream fromStream(String streamId) {
    -        return new SourcedStream(generateId("SourcedStream-" + streamId), this.appConfig,
environment.getStreamSource(streamId, this.appConfig));
    -    }
    -
    -    public SourcedStream fromStream(SourcedStream sourcedStream) {
    -        return new SourcedStream(sourcedStream);
    -    }
    -
    -    private String generateId(String prefix) {
    -        return String.format("%s_%s", prefix, this.identifier.getAndIncrement());
    -    }
    +/*
    --- End diff --
    
    dos2unix


> Refine system metric schema design and fix system metric collector
> ------------------------------------------------------------------
>
>                 Key: EAGLE-928
>                 URL: https://issues.apache.org/jira/browse/EAGLE-928
>             Project: Eagle
>          Issue Type: Bug
>    Affects Versions: v0.5.0
>            Reporter: Hao Chen
>            Assignee: Hao Chen
>             Fix For: v0.5.0
>
>
> Refine system metric schema design and fix system metric collector
> h1. Principle
> Any metric (no mater sys metric) should at least have three fields: "metric", "group",
"timestamp", "value", 
> "group" is the metric category path separated with slash, for example "FIRST CATEGORY.SECOND
CATEGORY", which will be commonly used for well organizing metric groups in customized metric
dashboard.
> h1. Stream Name for System Metric
> {code}
> <stream>
>             <streamId>SYSTEM_METRIC_STREAM</streamId>
>             <description>System Metrics Stream including CPU, Network, Disk, etc.</description>
>             <columns>
>                 <column>
>                     <name>host</name>
>                     <type>string</type>
>                 </column>
>                 <column>
>                     <name>timestamp</name>
>                     <type>long</type>
>                 </column>
>                 <column>
>                     <name>metric</name>
>                     <type>string</type>
>                 </column>
>                 <column>
>                     <name>group</name>
>                     <type>string</type>
>                 </column>
>                 <column>
>                     <name>site</name>
>                     <type>string</type>
>                 </column>
>                 <column>
>                     <name>device</name>
>                     <type>string</type>
>                 </column>
>                 <column>
>                     <name>value</name>
>                     <type>double</type>
>                     <defaultValue>0.0</defaultValue>
>                 </column>
>             </columns>
>         </stream>
> {code}
> h1. Stream Schema for System metric
>     
>        * *metric*: [STRING] metric name string
>        * *group*: [STRING] metric group/type
>        * *timestamp*: [LONG] metric generation time
>        * *site*: [STRING] siteId
>        * *host*: [STRING] source host name
>        * *device*: [STRING] device name, like cpu
> h1. Sample CPU Metric
> {code}
> {
>     "timestamp": 1487918913569, 
>     "metric": "system.cpu.usage", 
>     "group": "system.cpu",
>     "site": "sandbox", 
>     "value": 0.058, 
>     "host": "sandbox.hortonworks.com", 
>     "device": "cpu7"
> }
> {code}
> h1. Sample Network Metrics
> {code}
> {
>     "timestamp": 1487918913569, 
>     "metric": "system.nic.transmitdrop", 
>     "group": "system.network",
>     "site": "sandbox", 
>     "value": 7724.0, 
>     "host": "sandbox.hortonworks.com", 
>     "device": "eth0"
> }
> {code}
> h1. Sample Metric Schema
> {code}
> {
>             "tags": {
>                 "site": "sandbox",
>                 "name": "system.memory.nfs_unstable.kb",
>                 "group": "system.memory"
>             },
>             "dimensionFields": [
>                 "host",
>                 "group",
>                 "site",
>                 "device"
>             ],
>             "metricFields": [
>                 "value"
>             ],
>             "granularity": "MINUTE",
>             "modifiedTimestamp": 1488190388479
>         }
> {code}
> h1. Stream Persist Topology
> {code}
>         environment.newApp(config)
>                 .fromStream("HADOOP_JMX_METRIC_STREAM")
>                 .saveAsMetric(
>                         MetricDescriptor.metricGroupAs((MetricGroupSelector) event ->
{
>                             if (event.containsKey("component")) {
>                                 return String.format("hadoop.%s", ((String) event.get("component")).toLowerCase());
>                             } else {
>                                 return "hadoop.metrics";
>                             }
>                         })
>                         .siteByField("site")
>                         .namedByField("metric")
>                         .eventTimeByField("timestamp")
>                         .dimensionFields("host", "component", "site")
>                         .granularity(Calendar.MINUTE)
>                         .valueField("value"))
>                 .fromStream("SYSTEM_METRIC_STREAM")
>                 .saveAsMetric(MetricDescriptor.metricGroupByField("group")
>                         .siteByField("site")
>                         .namedByField("metric")
>                         .eventTimeByField("timestamp")
>                         .dimensionFields("host", "group", "site", "device")
>                         .granularity(Calendar.MINUTE)
>                         .valueField("value")
>                 )
>                 .toTopology();
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Mime
View raw message