chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From asrab...@apache.org
Subject svn commit: r1021625 - in /incubator/chukwa/trunk: ./ lib/ src/java/org/apache/hadoop/chukwa/conf/ src/java/org/apache/hadoop/chukwa/datacollection/agent/ src/java/org/apache/hadoop/chukwa/datacollection/collector/
Date Tue, 12 Oct 2010 04:27:32 GMT
Author: asrabkin
Date: Tue Oct 12 04:27:32 2010
New Revision: 1021625

URL: http://svn.apache.org/viewvc?rev=1021625&view=rev
Log:
CHUKWA-527  Configuration spellcheck. Contributed by Ari Rabkin.

Added:
    incubator/chukwa/trunk/lib/agent.dict
    incubator/chukwa/trunk/lib/collector.dict
    incubator/chukwa/trunk/lib/confspellcheck-LICENSE
    incubator/chukwa/trunk/lib/confspellcheck-README
    incubator/chukwa/trunk/lib/confspellcheck.jar   (with props)
Modified:
    incubator/chukwa/trunk/NOTICE.txt
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java

Modified: incubator/chukwa/trunk/NOTICE.txt
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/NOTICE.txt?rev=1021625&r1=1021624&r2=1021625&view=diff
==============================================================================
--- incubator/chukwa/trunk/NOTICE.txt (original)
+++ incubator/chukwa/trunk/NOTICE.txt Tue Oct 12 04:27:32 2010
@@ -29,3 +29,5 @@ Copyright is held by its developers.
 JChronic is under an MIT license, copyright is held by the developers.
 * Available from https://jchronic.dev.java.net/
 
+The configuration spellchecker was written by Ari Rabkin [a Chukwa committer], and is available
under a 2-clause BSD-style license. 
+* Available from http://code.google.com/p/jchord/source/browse/#svn/trunk/conf_spellchecker
\ No newline at end of file

Added: incubator/chukwa/trunk/lib/agent.dict
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/lib/agent.dict?rev=1021625&view=auto
==============================================================================
--- incubator/chukwa/trunk/lib/agent.dict (added)
+++ incubator/chukwa/trunk/lib/agent.dict Tue Oct 12 04:27:32 2010
@@ -0,0 +1,35 @@
+$CHUKWA_CONF_DIR	
+$CHUKWA_HOME	File
+$CHUKWA_PID_DIR	File
+CONF-adaptor.dirscan.intervalMs	Integral
+CONF-adaptor.memBufWrapper.size	Integral
+CONF-chukwaAgent.adaptor.context.switch.time	Time
+CONF-chukwaAgent.adaptor.fileadaptor.timeoutperiod	Time
+CONF-chukwaAgent.checkpoint.dir	File
+CONF-chukwaAgent.checkpoint.enabled	Boolean
+CONF-chukwaAgent.checkpoint.interval	Time
+CONF-chukwaAgent.checkpoint.name	
+CONF-chukwaAgent.connector.retryRate	Time
+CONF-chukwaAgent.control.port	Portno
+CONF-chukwaAgent.control.remote	Boolean
+CONF-chukwaAgent.fileTailingAdaptor.maxReadSize	Integral
+CONF-chukwaAgent.initial_adaptors	File
+CONF-chukwaAgent.sender.collectorTimeout	Integral
+CONF-chukwaAgent.sender.fastRetries	Integral
+CONF-chukwaAgent.sender.retries	Integral
+CONF-chukwaAgent.sender.retryInterval	Integral
+CONF-chukwaAgent.tags	
+CONF-chukwaCollector.asyncAcks.scanperiod	Time
+CONF-chukwaCollector.http.port	Portno
+CONF-chukwaCollector.rotateInterval	Time
+CONF-connector.commitpoll.hostfile	
+CONF-connector.commitpoll.period	Time
+CONF-connector.commitpoll.timeout	Time
+CONF-constAdaptor.minSleep	Integral
+CONF-constAdaptor.sleepVariance	Integral
+CONF-httpConnector.asyncAcks	Boolean
+CONF-httpConnector.maxPostSize	Integral
+CONF-httpConnector.minPostInterval	Time
+PROP-CHUKWA_CONF_DIR	File
+PROP-CHUKWA_HOME	
+PROP-user.name	

Added: incubator/chukwa/trunk/lib/collector.dict
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/lib/collector.dict?rev=1021625&view=auto
==============================================================================
--- incubator/chukwa/trunk/lib/collector.dict (added)
+++ incubator/chukwa/trunk/lib/collector.dict Tue Oct 12 04:27:32 2010
@@ -0,0 +1,28 @@
+$CHUKWA_CONF_DIR	
+$CHUKWA_DATA_DIR	
+$CHUKWA_HOME	
+$CHUKWA_PID_DIR	File
+CONF-chukwaCollector.asyncAcks.purgedelay	Time
+CONF-chukwaCollector.asyncAcks.scanpaths	
+CONF-chukwaCollector.asyncAcks.scanperiod	Time
+CONF-chukwaCollector.chunkSuppressBufferSize	Integral
+CONF-chukwaCollector.http.port	Integral
+CONF-chukwaCollector.http.threads	Integral
+CONF-chukwaCollector.localOutputDir	File
+CONF-chukwaCollector.minPercentFreeDisk	Integral
+CONF-chukwaCollector.outputDir	
+CONF-chukwaCollector.pipeline	ClassName list
+CONF-chukwaCollector.rotateInterval	Time
+CONF-chukwaCollector.showLogs.enabled	Boolean
+CONF-chukwaCollector.stats.period	Time
+CONF-chukwaCollector.tee.keepalive	Boolean
+CONF-chukwaCollector.tee.port	Integral
+CONF-chukwaCollector.writeChunkRetries	Integral
+CONF-chukwaCollector.writerClass	ClassName
+CONF-chukwaRootDir	
+CONF-fs.default.name	URI
+CONF-httpConnector.asyncAcks	Boolean
+CONF-localToRemoteHdfsMover.exitIfHDFSNotavailable	Boolean
+CONF-nullWriter.dataRate	Time
+CONF-writer.hdfs.filesystem	URI
+PROP-CHUKWA_HOME	

Added: incubator/chukwa/trunk/lib/confspellcheck-LICENSE
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/lib/confspellcheck-LICENSE?rev=1021625&view=auto
==============================================================================
--- incubator/chukwa/trunk/lib/confspellcheck-LICENSE (added)
+++ incubator/chukwa/trunk/lib/confspellcheck-LICENSE Tue Oct 12 04:27:32 2010
@@ -0,0 +1,26 @@
+Copyright (c) 2008-2010, Ariel Rabkin
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the
+  distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.  

Added: incubator/chukwa/trunk/lib/confspellcheck-README
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/lib/confspellcheck-README?rev=1021625&view=auto
==============================================================================
--- incubator/chukwa/trunk/lib/confspellcheck-README (added)
+++ incubator/chukwa/trunk/lib/confspellcheck-README Tue Oct 12 04:27:32 2010
@@ -0,0 +1,29 @@
+This directory holds a configuration spellcheck tool, being developed by Ari Rabkin at UC
Bekeley. (Contact asrabkin@gmail.com for details)
+
+The checker has two basic tasks. First, to make sure that every option set in a configuration
file is an option actually supported by the program in question. Second, to check that the
values specified for the option make sense. (In case, e.g., a user confuses a numeric from
boolean parameter).
+
+
+
+DICTIONARY FORMAT
+
+Each line of the dictionary files consists of a regular expression, followed by an option
type, followed by an optional annotation. These sections are separated by a single tab character.
+
+Each regular expression may match one or more valid options. (This is to cover cases like
Hadoop's fs.x.impl, where x can be any string).
+
+The list of valid option types is as follows:
+	Address
+	Boolean
+	ClassName
+	File
+	Fraction
+	Integral
+	NetworkInterface
+	Portno
+	Special
+	URI
+	URL	
+
+Most of these are self-explanatory. Special means that the option must be one of a small
number of string options. The list of valid options goes in the annotation field. It should
be comma-separated and wrapped in {}.
+
+If a dictionary includes a type not on that list, the checker will assume that all values
are acceptable for that option.
+

Added: incubator/chukwa/trunk/lib/confspellcheck.jar
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/lib/confspellcheck.jar?rev=1021625&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/chukwa/trunk/lib/confspellcheck.jar
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java?rev=1021625&r1=1021624&r2=1021625&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java
(original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java
Tue Oct 12 04:27:32 2010
@@ -27,6 +27,16 @@ import org.apache.log4j.Logger;
 public class ChukwaConfiguration extends Configuration {
   static Logger log = Logger.getLogger(ChukwaConfiguration.class);
 
+  private String chukwaHome, chukwaConf;
+  
+  public String getChukwaHome() {
+    return chukwaHome;
+  }
+  
+  public String getChukwaConf() {
+    return chukwaConf;
+  }
+  
   public ChukwaConfiguration() {
     this(true);
   }
@@ -35,7 +45,7 @@ public class ChukwaConfiguration extends
     super();
     if (loadDefaults) {
 
-      String chukwaHome = System.getenv("CHUKWA_HOME");
+      chukwaHome = System.getenv("CHUKWA_HOME");
       if (chukwaHome == null) {
         chukwaHome = ".";
       }
@@ -43,7 +53,7 @@ public class ChukwaConfiguration extends
       if (!chukwaHome.endsWith("/")) {
         chukwaHome = chukwaHome + File.separator;
       }
-      String chukwaConf = System.getenv("CHUKWA_CONF_DIR");
+      chukwaConf = System.getenv("CHUKWA_CONF_DIR");
       if (chukwaConf == null) {
         chukwaConf = chukwaHome + "conf" + File.separator;
       }

Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java?rev=1021625&r1=1021624&r2=1021625&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
(original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
Tue Oct 12 04:27:32 2010
@@ -51,12 +51,14 @@ import org.apache.hadoop.chukwa.util.Dae
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.log4j.Logger;
+
 import org.mortbay.jetty.Server;
 import org.mortbay.jetty.servlet.Context;
 import org.mortbay.jetty.servlet.ServletHolder;
 import org.mortbay.jetty.nio.SelectChannelConnector;
 import org.mortbay.thread.BoundedThreadPool;
 import com.sun.jersey.spi.container.servlet.ServletContainer;
+import edu.berkeley.confspell.*;
 
 /**
  * The local agent daemon that runs on each machine. This class is designed to
@@ -161,7 +163,9 @@ public class ChukwaAgent implements Adap
             + "[default collector URL]");
         System.exit(0);
       }
+
       Configuration conf = readConfig();
+      
       ChukwaAgent localAgent = new ChukwaAgent(conf);
 
       if (agent.anotherAgentIsRunning()) {
@@ -220,7 +224,7 @@ public class ChukwaAgent implements Adap
   public ChukwaAgent(Configuration conf) throws AlreadyRunningException {
     ChukwaAgent.agent = this;
     this.conf = conf;
-
+    
     // almost always just reading this; so use a ConcurrentHM.
     // since we wrapped the offset, it's not a structural mod.
     adaptorPositions = new ConcurrentHashMap<Adaptor, Offset>();
@@ -739,6 +743,15 @@ public class ChukwaAgent implements Adap
           .getAbsolutePath());
     conf.set("chukwaAgent.initial_adaptors", new File(chukwaConf,
         "initial_adaptors").getAbsolutePath());
+    
+    
+    try { 
+      Configuration chukwaAgentConf = new Configuration(false);
+      chukwaAgentConf.addResource(new Path(agentConf.getAbsolutePath()));
+      Checker.checkConf(new OptDictionary(new File(new File(chukwaHome, "lib"), "agent.dict")),
+          HSlurper.fromHConf(chukwaAgentConf));
+    } catch(Exception e) {e.printStackTrace();}
+    
     return conf;
   }
 

Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java?rev=1021625&r1=1021624&r2=1021625&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java
(original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java
Tue Oct 12 04:27:32 2010
@@ -27,7 +27,13 @@ import org.apache.hadoop.chukwa.datacoll
 import org.apache.hadoop.chukwa.datacollection.writer.*;
 import org.apache.hadoop.chukwa.util.DaemonWatcher;
 import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import edu.berkeley.confspell.Checker;
+import edu.berkeley.confspell.HSlurper;
+import edu.berkeley.confspell.OptDictionary;
 import javax.servlet.http.HttpServlet;
+import java.io.File;
 import java.util.*;
 
 public class CollectorStub {
@@ -49,6 +55,15 @@ public class CollectorStub {
       }
 
       ChukwaConfiguration conf = new ChukwaConfiguration();
+      
+      try {
+        Configuration collectorConf = new Configuration(false);
+        collectorConf.addResource(new Path(conf.getChukwaConf() + "/chukwa-common.xml"));
+        collectorConf.addResource(new Path(conf.getChukwaConf() + "/chukwa-collector-conf.xml"));
+        Checker.checkConf(new OptDictionary(new File(new File(conf.getChukwaHome(), "lib"),
"collector.dict")),
+            HSlurper.fromHConf(collectorConf));
+      } catch(Exception e) {e.printStackTrace();}
+      
       int portNum = conf.getInt("chukwaCollector.http.port", 9999);
       THREADS = conf.getInt("chukwaCollector.http.threads", THREADS);
 



Mime
View raw message