chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r765732 - in /hadoop/chukwa/branches/chukwa-0.1: bin/aggregator.sh src/java/org/apache/hadoop/chukwa/database/Aggregator.java
Date Thu, 16 Apr 2009 19:58:39 GMT
Author: eyang
Date: Thu Apr 16 19:58:39 2009
New Revision: 765732

URL: http://svn.apache.org/viewvc?rev=765732&view=rev
Log:
CHUKWA-128.  Added tools to compute aggregation in database. (Eric Yang)

Added:
    hadoop/chukwa/branches/chukwa-0.1/bin/aggregator.sh   (with props)
Modified:
    hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/database/Aggregator.java

Added: hadoop/chukwa/branches/chukwa-0.1/bin/aggregator.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/bin/aggregator.sh?rev=765732&view=auto
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/bin/aggregator.sh (added)
+++ hadoop/chukwa/branches/chukwa-0.1/bin/aggregator.sh Thu Apr 16 19:58:39 2009
@@ -0,0 +1,57 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=$$
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+if [ "$CHUKWA_IDENT_STRING" = "" ]; then
+  export CHUKWA_IDENT_STRING="$USER"
+fi
+
+CHUKWA_OPTS="-DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR}
-DDATACONFIG=${CHUKWA_CONF_DIR}/mdl.xml"
+CLASS_OPTS="-classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR}"
+JVM_OPTS="-DAPP=dbAdmin -Dlog4j.configuration=chukwa-log4j.properties ${CHUKWA_OPTS} ${CLASS_OPTS}"
+
+if [ $# -lt 3 ]; then
+  echo "$0: [cluster] [command] [date]"
+  echo "[cluster] - cluster name in jdbc.conf"
+  echo "[command] - create/aggregate"
+  echo "            create    : create database partition for given date"
+  echo "            aggregate : aggregate database partition for given date range" 
+  exit 0
+fi
+
+CLUSTER=$1
+shift
+CMD=$1
+shift
+
+if [ $CMD = "create" ]; then
+  EXP_DATE=$@
+  ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.TableCreator
${EXP_DATE} 7
+  ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.TableCreator
${EXP_DATE} 30
+  ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.TableCreator
${EXP_DATE} 91
+  ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.TableCreator
${EXP_DATE} 365
+  ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.TableCreator
${EXP_DATE} 3650
+fi
+
+if [ $CMD = "aggregate" ]; then
+  ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.Aggregator
$@
+fi

Propchange: hadoop/chukwa/branches/chukwa-0.1/bin/aggregator.sh
------------------------------------------------------------------------------
    svn:executable = *

Modified: hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/database/Aggregator.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/database/Aggregator.java?rev=765732&r1=765731&r2=765732&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/database/Aggregator.java
(original)
+++ hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/database/Aggregator.java
Thu Apr 16 19:58:39 2009
@@ -25,9 +25,11 @@
 import java.sql.ResultSet;
 import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
+import java.text.ParsePosition;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Calendar;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.regex.Matcher;
@@ -73,27 +75,40 @@
     return contents.toString();
   }
 
-  public void process(String query) {
-    ResultSet rs = null;
-    String[] columns;
-    int[] columnsType;
-    String groupBy = "";
-    long start = current;
-    long end = current;
-
+  public void process(long start, long end, String query) {
     try {
-      Macro macroProcessor = new Macro(current, current, query);
+      Macro macroProcessor = new Macro(start, end, query);
       query = macroProcessor.toString();
+      log.info(query);
       db.execute(query);
     } catch (Exception e) {
       log.error(query);
-      log.error(e.getMessage());
+      log.error(ExceptionUtil.getStackTrace(e));
     }
   }
 
+  public void process(String query) {
+    long start = current;
+    long end = current;
+    process(current, current, query);
+  }
+
   public static void main(String[] args) {
+    long startTime = 0;
+    long endTime = 0;
     long aggregatorStart = Calendar.getInstance().getTimeInMillis();
     long longest = 0;
+    if(args.length>=4) {
+      ParsePosition pp = new ParsePosition(0);
+      SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm");
+      String buffer = args[0]+" "+args[1];
+      Date tmp = format.parse(buffer, pp);
+      startTime = tmp.getTime();
+      buffer = args[2]+" "+args[3];
+      pp = new ParsePosition(0);
+      tmp = format.parse(buffer, pp);
+      endTime = tmp.getTime();
+    }
     String longQuery = null;
     log.info("Aggregator started.");
     String cluster = System.getProperty("CLUSTER");
@@ -105,21 +120,28 @@
         .getenv("CHUKWA_CONF_DIR")
         + File.separator + "aggregator.sql"));
     String[] query = queries.split("\n");
-    for (int i = 0; i < query.length; i++) {
-      if (query[i].equals("")) {
-      } else if (query[i].indexOf("#") == 0) {
-        log.debug("skipping: " + query[i]);
-      } else {
-        Aggregator dba = new Aggregator();
-        long start = Calendar.getInstance().getTimeInMillis();
-        dba.process(query[i]);
-        long end = Calendar.getInstance().getTimeInMillis();
-        long duration = end - start;
-        if (duration >= longest) {
-          longest = duration;
-          longQuery = query[i];
+    while(startTime<=endTime) {
+      for (int i = 0; i < query.length; i++) {
+        if (query[i].equals("")) {
+        } else if (query[i].indexOf("#") == 0) {
+          log.debug("skipping: " + query[i]);
+        } else {
+          Aggregator dba = new Aggregator();
+          long start = Calendar.getInstance().getTimeInMillis();
+          if(startTime!=0 && endTime!=0) {
+            dba.process(startTime, startTime, query[i]);
+          } else {
+            dba.process(query[i]);
+          }
+          long end = Calendar.getInstance().getTimeInMillis();
+          long duration = end - start;
+          if (duration >= longest) {
+            longest = duration;
+            longQuery = query[i];
+          }
         }
       }
+      startTime = startTime + 5*60000;
     }
     db.close();
     long aggregatorEnd = Calendar.getInstance().getTimeInMillis();



Mime
View raw message