hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r723855 [1/23] - in /hadoop/core/trunk: ./ src/contrib/ src/contrib/chukwa/ src/contrib/chukwa/bin/ src/contrib/chukwa/conf/ src/contrib/chukwa/docs/ src/contrib/chukwa/docs/paper/ src/contrib/chukwa/hadoop-packaging/ src/contrib/chukwa/lib...
Date Fri, 05 Dec 2008 20:30:21 GMT
Author: cdouglas
Date: Fri Dec  5 12:30:14 2008
New Revision: 723855

URL: http://svn.apache.org/viewvc?rev=723855&view=rev
Log:
HADOOP-4709. Add several new features and bug fixes to Chukwa. Contributed by Jerome Boulon, Eric Yang, Andy Konwinski, and Ariel Rabkin.

Added:
    hadoop/core/trunk/src/contrib/chukwa/bin/README
    hadoop/core/trunk/src/contrib/chukwa/bin/dailyRolling.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/dbAdmin.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/dbLoader.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/dbSetup.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/dumpArchive.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/dumpDataType.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/dumpRecord.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/hourlyRolling.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/netstat.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/start-agents.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/stop-agents.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/validateDemux.sh
    hadoop/core/trunk/src/contrib/chukwa/conf/README
    hadoop/core/trunk/src/contrib/chukwa/conf/aggregator.sql
    hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agents.template
    hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-demux-conf.xml
    hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-hadoop-metrics-log4j.properties
    hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-log4j.properties
    hadoop/core/trunk/src/contrib/chukwa/conf/database_create_tables
    hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-log4j.properties
    hadoop/core/trunk/src/contrib/chukwa/conf/jdbc.conf
    hadoop/core/trunk/src/contrib/chukwa/conf/jdbc.conf.template
    hadoop/core/trunk/src/contrib/chukwa/lib/jstl-LICENSE.txt
    hadoop/core/trunk/src/contrib/chukwa/lib/jstl-README.txt
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/database/
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/database/Aggregator.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/database/Consolidator.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/database/DataExpiration.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/database/DatabaseConfig.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/database/MetricsAggregation.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/database/TableCreator.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/FileAdaptor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TerminatorThread.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/ClientAck.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/WriterException.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/archive/ChukwaArchiveBuilder.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/archive/ChukwaArchiveDataTypeOutputFormat.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/archive/ChukwaArchiveDataTypePartitioner.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/archive/ChukwaArchiveStreamNameOutputFormat.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/archive/ChukwaArchiveStreamNamePartitioner.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/database/DBException.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/database/DBPlugin.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/database/MRJobCounters.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/DailyChukwaRecordRolling.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/HourlyChukwaRecordRolling.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/MoveToRepository.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/RecordMerger.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/ChukwaOutputCollector.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/Util.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/ChunkSaver.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/DefaultProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Df.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobConfProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Log4jJobHistoryProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/MapProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/MapProcessorFactory.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/PbsNodes.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/SysLog.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Torque.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TsProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/YWatch.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/reducer/
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/reducer/DuplicateReduceProcessorException.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/reducer/IdentityReducer.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/reducer/JobLogHistoryReduceProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/reducer/MRJobReduceProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/reducer/ReduceProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/reducer/ReduceProcessorFactory.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/reducer/SystemMetrics.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/reducer/UnknownReduceTypeException.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/ChukwaRecordKey.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/RecordUtil.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/Token.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/record/ChukwaDSInternalResult.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/record/ChukwaRecordDataSource.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/hicc/Chart.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/hicc/Iframe.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/DumpArchive.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/DumpDataType.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/DumpRecord.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/writer/
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/writer/TestClientAck.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/ChukwaTestOutputCollector.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestAbtractProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestHadoopLogProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/validationframework/
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/validationframework/ChukwaAgentToCollectorValidator.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/validationframework/DemuxDirectoryValidator.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/validationframework/TestDemux.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/validationframework/interceptor/
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/validationframework/interceptor/ChunkDumper.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/validationframework/interceptor/ChunkQueueInterceptor.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/validationframework/interceptor/SetupTestClasses.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/validationframework/util/
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/validationframework/util/DataOperations.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/validationframework/util/MD5.java
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/WEB-INF/
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/WEB-INF/jsp2/
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/WEB-INF/jsp2/taglib.tld
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/WEB-INF/web.xml
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/css/
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/css/default.css
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/css/flexigrid/
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/css/flexigrid/flexigrid.css
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/css/iui.css
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/css/menu.css
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/css/tab.css
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/css/timeframe.css
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/css/timeline.css
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/cluster_disk.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/cluster_disk_pcnt.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/cluster_metrics.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/cluster_metrics_pcnt.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/cluster_selector.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/debug.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/dfs_datanode.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/dfs_fsnamesystem.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/dfs_namenode.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/dfs_throughput.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/disk.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/disk_pcnt.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/event_viewer.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/hadoop_jvm.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/hadoop_mapred.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/hod_job.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/hod_job_chart.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/hod_job_list.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/host_selector.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/node_activity_chart.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/node_utilization_by_job.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/rpc_metrics.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/system_metrics.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/system_metrics_pcnt.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/descriptors/time.descriptor
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/index.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/ajaxtree.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/excanvas.pack.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/flexigrid.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/flexigrid.pack.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/flot.extend.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/host_selector.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/iui.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/jquery-1.2.6.min.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/jquery.flot.pack.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/json.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/org/
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/org/tool-man/
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/org/tool-man/cookies.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/org/tool-man/coordinates.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/org/tool-man/core.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/org/tool-man/core.js.old
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/org/tool-man/css.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/org/tool-man/drag.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/org/tool-man/dragsort.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/org/tool-man/events.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/time.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/timeframe.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/treeview-min.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/workspace/
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/workspace/builder.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/workspace/controls.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/workspace/dragdrop.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/workspace/effects.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/workspace/prototype.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/workspace/scriptaculous.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/workspace/slider.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/workspace/sound.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/workspace/workspace.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/js/yahoo-dom-event.js
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/chart.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/cluster_selector.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/debug.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/event.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/event2_viewer.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/event_viewer.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/event_viewer_data.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/event_viewer_helper.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/event_wrapper.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/events-xml.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/get_data_source.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/get_timezone_list.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/get_user_list.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/hod_job.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/hod_job_list.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/host_selector.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/session.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/single-series-chart-javascript.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/time.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/time_frame.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/time_range.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/time_slider.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/time_slider_wrapper.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/workspace/
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/workspace/manage_view.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/jsp/workspace/time_iphone.jsp
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/views/
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/views/default.view
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/views/newview.tpl
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/views/view.permission
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/views/workspace_view_list.cache
    hadoop/core/trunk/src/contrib/chukwa/src/web/hicc/welcome.html
    hadoop/core/trunk/src/contrib/chukwa/tools/init.d/
    hadoop/core/trunk/src/contrib/chukwa/tools/init.d/chukwa-agent
    hadoop/core/trunk/src/contrib/chukwa/tools/init.d/chukwa-collector
    hadoop/core/trunk/src/contrib/chukwa/tools/init.d/chukwa-data-processors
    hadoop/core/trunk/src/contrib/chukwa/tools/init.d/chukwa-system-metrics
    hadoop/core/trunk/src/contrib/chukwa/tools/packetcapture.sh
    hadoop/core/trunk/src/contrib/chukwa/tools/startpacketcapture.sh
    hadoop/core/trunk/src/contrib/chukwa/tools/stoppacketcapture.sh
Removed:
    hadoop/core/trunk/src/contrib/chukwa/chukwafy_jobhist.awk
    hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-slaves.template
    hadoop/core/trunk/src/contrib/chukwa/conf/mdl.xml
    hadoop/core/trunk/src/contrib/chukwa/hadoop-packaging/build-tarball.sh
    hadoop/core/trunk/src/contrib/chukwa/hadoop-packaging/log4j.properties.templ
    hadoop/core/trunk/src/contrib/chukwa/hadoop-packaging/patchhadoop.patch
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/Consolidator.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/database/DataExpiration.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/database/DatabaseHelper.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/DFProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/PbsNodesProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/YWatchProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/DatabaseRecord.java
Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/contrib/build.xml
    hadoop/core/trunk/src/contrib/chukwa/bin/VERSION
    hadoop/core/trunk/src/contrib/chukwa/bin/agent.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/buildDailyArchive.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/buildHourlyArchive.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemon.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/jettyCollector.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/nodeActivityDataLoader.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/processSinkFiles.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/slaves.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/start-all.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/start-probes.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/stop-all.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/stop-data-processors.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/stop-probes.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh
    hadoop/core/trunk/src/contrib/chukwa/bin/torqueDataLoader.sh
    hadoop/core/trunk/src/contrib/chukwa/build.xml
    hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml.template
    hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh.template
    hadoop/core/trunk/src/contrib/chukwa/conf/collectors.template
    hadoop/core/trunk/src/contrib/chukwa/conf/hadoop-metrics.properties
    hadoop/core/trunk/src/contrib/chukwa/conf/log4j.properties
    hadoop/core/trunk/src/contrib/chukwa/conf/mdl.xml.template
    hadoop/core/trunk/src/contrib/chukwa/conf/system-data-loader.properties
    hadoop/core/trunk/src/contrib/chukwa/docs/README
    hadoop/core/trunk/src/contrib/chukwa/docs/paper/chukwa_08.tex
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/Chunk.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/ChunkImpl.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/ChunkQueue.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/DataFactory.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/Adaptor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/ExecAdaptor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailer.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/AdaptorFactory.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/AgentControlSocketListener.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/MemLimitQueue.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/collector/servlet/ServletCollector.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/collector/servlet/ServletDiagnostics.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/connector/Connector.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/connector/http/HttpConnector.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/sender/ChukwaHttpSender.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/sender/RetryListOfCollectors.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/test/ConsoleOutConnector.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/ChukwaWriter.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/ConsoleWriter.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/InMemoryWriter.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/SeqFileWriter.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/archive/ChukwaArchiveDailyOutputFormat.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/database/DatabaseLoader.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/database/MetricDataLoader.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/ChukwaRecordOutputFormat.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/ChukwaRecordPartitioner.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/Demux.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/MoveOrMergeRecordFile.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/AbstractProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/DFInvalidRecord.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/DebugOutputProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/DuplicateProcessorException.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/HadoopLogProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/HadoopMetricsProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Iostat.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLogHistoryProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/PbsInvalidEntry.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Sar.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Top.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/UnknownRecordTypeException.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/YwatchInvalidEntry.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/ChukwaRecord.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/ChukwaSearchResult.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/ChukwaSearchService.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/Record.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/SearchResult.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/SearchService.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/DataSource.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/DataSourceFactory.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/DsDirectory.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/database/DatabaseDS.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/record/ChukwaSequenceFileParser.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/record/RecordDS.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/hicc/ClusterConfig.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/hicc/DatasetMapper.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/hicc/TimeHandler.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/hicc/Workspace.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/Log4JMetricsContext.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/mdl/TorqueInfoProcessor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/ConstRateAdaptor.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/DatabaseWriter.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/MaxRateSender.java
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/PidFile.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/TestChunkBuilder.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestFileTailingAdaptors.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestStartAtOffset.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/connector/ChunkCatcherConnector.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/inputtools/log4j/TestChukwaAppender.java
    hadoop/core/trunk/src/contrib/chukwa/src/web/collector/WEB-INF/web.xml
    hadoop/core/trunk/src/contrib/chukwa/tools/expire.sh

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Fri Dec  5 12:30:14 2008
@@ -63,6 +63,16 @@
     HADOOP-4661. Add DistCh, a new tool for distributed ch{mod,own,grp}.
     (szetszwo)
 
+    HADOOP-4709. Add several new features and bug fixes to Chukwa.
+    (Jerome Boulon, Eric Yang, Andy Konwinski, Ariel Rabkin via cdouglas)
+      Added Hadoop Infrastructure Care Center (UI for visualize data collected by Chukwa)
+      Added FileAdaptor for streaming small file in one chunk
+      Added compression to archive and demux output
+      Added unit tests and validation for agent, collector, and demux map reduce job
+      Added database loader for loading demux output (sequence file) to jdbc connected database
+      Added algorithm to distribute collector load more evenly
+
+
   IMPROVEMENTS
 
     HADOOP-4234. Fix KFS "glue" layer to allow applications to interface

Modified: hadoop/core/trunk/src/contrib/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/build.xml?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/build.xml (original)
+++ hadoop/core/trunk/src/contrib/build.xml Fri Dec  5 12:30:14 2008
@@ -50,6 +50,7 @@
       <fileset dir="." includes="streaming/build.xml"/>
       <fileset dir="." includes="fairscheduler/build.xml"/>
       <fileset dir="." includes="capacity-scheduler/build.xml"/>
+      <fileset dir="." includes="chukwa/build.xml"/>
     </subant>
   </target>
   

Added: hadoop/core/trunk/src/contrib/chukwa/bin/README
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/README?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/README (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/README Fri Dec  5 12:30:14 2008
@@ -0,0 +1,24 @@
+Check for an updated copy of this README at http://wiki.apache.org/hadoop/Chukwa_Startup_and_Shutdown_Scripts
+
+start-all.sh - runs start-collectors.sh, start-agents.sh, start-probes.sh, start-data-processors.sh
+
+start-collectors.sh - start the chukwa collector daemon (jettyCollector.sh) on hosts listed in conf/collectors
+stop-collectors.sh - stop the chukwa collector daemon (jettyCollector.sh) on hosts listed in conf/collectors
+
+jettyCollector.sh - start the chukwa collector daemon on the current host
+
+start-agents.sh - start chukwa agent daemon (agent.sh) on all hosts listed in conf/chukwa-agents
+stop-agents.sh - stop chukwa agent daemon (agent.sh) on all hosts listed in conf/chukwa-agents
+
+agent.sh - start the chukwa agent on the current host
+
+start-probes.sh - runs, in this order, systemDataLoader.sh, torqueDataLoader.sh, nodeActivityDataLoader.sh
+
+systemDataLoader.sh - every 60 seconds run: sar, iostat, top, top, df, netstat. //TODO: figure out where these guys send their output
+torqueDataLoader.sh - //TODO: FILL THIS IN!
+nodeActivityDataLoader.sh - //TODO: FILL THIS IN!
+
+slaves.sh <command command_args ...> - run arbitrary commands on all hosts in conf/slaves
+
+jettycollector.sh - start a jetty based version of the Chukwa collector
+agent.sh - start the chukwa agent on the local machine

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/VERSION
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/VERSION?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/VERSION (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/VERSION Fri Dec  5 12:30:14 2008
@@ -1 +1 @@
-0.1.0
+0.1.1

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/agent.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/agent.sh?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/agent.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/agent.sh Fri Dec  5 12:30:14 2008
@@ -21,6 +21,16 @@
 
 . "$bin"/chukwa-config.sh
 
-echo "${pid}" > "$CHUKWA_HOME/var/run/Agent.pid"
+echo "hadoop jar for agent is " ${HADOOP_JAR}
+trap '${JPS} | grep ChukwaAgent | cut -f 1 -d" " | xargs kill -TERM ; exit 0' 1 2 15
 
-${JAVA_HOME}/bin/java -DCHUKWA_HOME=${CHUKWA_HOME} -classpath ${CLASSPATH}:${chukwaCore}:${chukwaAgent}:${HADOOP_JAR}:${CHUKWA_HOME}/conf:${HADOOP_HOME}/conf:${common} org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent $@
+if [ "X$1" = "Xstop" ]; then
+  echo -n "Shutting down agent..."
+  JETTY_PID=`${JPS} | grep ChukwaAgent | cut -f 1 -d" "`
+  kill -TERM ${JETTY_PID} >&/dev/null
+  echo "done"
+  exit 0
+fi
+
+
+${JAVA_HOME}/bin/java -Xms32M -Xmx64M -DAPP=agent -Dlog4j.configuration=chukwa-log4j.properties -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -classpath ${CLASSPATH}:${CHUKWA_AGENT}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent $@

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/buildDailyArchive.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/buildDailyArchive.sh?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/buildDailyArchive.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/buildDailyArchive.sh Fri Dec  5 12:30:14 2008
@@ -20,11 +20,45 @@
 bin=`cd "$bin"; pwd`
 . "$bin"/chukwa-config.sh
 
-echo "${pid}" > "$CHUKWA_HOME/var/run/BuildDailyArchive.pid"
+echo "${pid}" > "$CHUKWA_HOME/var/run/buildDailyArchive.pid"
 
 HADOOP_CONF_DIR="${HADOOP_HOME}/conf/"
 HADOOP_CMDE="${HADOOP_HOME}/bin/hadoop "
 
+while [ 1 ]
+ do
+  now=`date +%s`
+  strDate=`date +%m/%d/%y%n`
+  srcHourly="/chukwa/postprocess/srcDaily$now/"
 
-  $HADOOP_CMDE jar ${chukwaCore} org.apache.hadoop.chukwa.extraction.archive.ChuckwaArchiveBuilder Daily $srcEventHdfsDir ${chuwaRecordsRepository}
- 
\ No newline at end of file
+  echo "Running $strDate $now" >> "${CHUKWA_LOG_DIR}/daily.log"
+
+  echo "srcHourly: $srcHourly " >> "${CHUKWA_LOG_DIR}/daily.log"
+
+  $HADOOP_CMDE dfs -mkdir $srcHourly/hourly
+  echo "done with mkdir" >> "${CHUKWA_LOG_DIR}/daily.log"
+ 
+  $HADOOP_CMDE dfs -mv "/chukwa/archives/hourly/*.arc" ${srcHourly}/hourly/
+  echo "done with mv archives" >> "${CHUKWA_LOG_DIR}/daily.log"
+ 
+  # Build the archive
+  $HADOOP_CMDE jar ${CHUKWA_CORE} org.apache.hadoop.chukwa.extraction.archive.ChuckwaArchiveBuilder Daily $srcHourly/hourly $srcHourly/daily
+  echo "done with chuckwaArchiveBuilder" >> "${CHUKWA_LOG_DIR}/daily.log"
+  
+   ## Hourly Archive available call all processors
+   ##############  ############## 
+  
+   ##############  ############## 
+  
+  
+  ############## MERGE or MOVE ##############
+  
+  ############## MERGE or MOVE ##############
+  
+  
+  now=`date +%s`
+  strDate=`date +%m/%d/%y%n`
+  echo "Stopping ${strDate} ${now}" >> "${CHUKWA_LOG_DIR}/daily.log"
+
+  sleep 36000
+done

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/buildHourlyArchive.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/buildHourlyArchive.sh?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/buildHourlyArchive.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/buildHourlyArchive.sh Fri Dec  5 12:30:14 2008
@@ -20,11 +20,45 @@
 bin=`cd "$bin"; pwd`
 . "$bin"/chukwa-config.sh
 
-echo "${pid}" > "$CHUKWA_HOME/var/run/BuildDailyArchive.pid"
+echo "${pid}" > "$CHUKWA_HOME/var/run/buildDailyArchive.pid"
 
 HADOOP_CONF_DIR="${HADOOP_HOME}/conf/"
 HADOOP_CMDE="${HADOOP_HOME}/bin/hadoop "
 
+while [ 1 ]
+ do
+  now=`date +%s`
+  strDate=`date +%m/%d/%y%n`
+  srcHourly="/chukwa/postprocess/srcHourly$now/"
 
-  $HADOOP_CMDE jar ${chukwaCore} org.apache.hadoop.chukwa.extraction.archive.ChuckwaArchiveBuilder Hourly $srcEventHdfsDir ${chuwaRecordsRepository}
- 
\ No newline at end of file
+  echo "Running $strDate $now" >> "${CHUKWA_LOG_DIR}/hourly.log"
+
+  echo "srcHourly: $srcHourly " >> "${CHUKWA_LOG_DIR}/hourly.log"
+
+  $HADOOP_CMDE dfs -mkdir $srcHourly/raw
+  echo "done with mkdir" >> "${CHUKWA_LOG_DIR}/hourly.log"
+ 
+  $HADOOP_CMDE dfs -mv "/chukwa/archives/raw/*.arc" ${srcHourly}/raw/
+  echo "done with mv archives" >> "${CHUKWA_LOG_DIR}/hourly.log"
+ 
+  # Build the archive
+  $HADOOP_CMDE jar ${CHUKWA_CORE} org.apache.hadoop.chukwa.extraction.archive.ChuckwaArchiveBuilder Hourly $srcHourly/arcFiles $srcHourly/hourly
+  echo "done with chuckwaArchiveBuilder" >> "${CHUKWA_LOG_DIR}/hourly.log"
+  
+   ## Hourly Archive available call all processors
+   ##############  ############## 
+  
+   ##############  ############## 
+  
+  
+  ############## MERGE or MOVE ##############
+  
+  ############## MERGE or MOVE ##############
+  
+  
+  now=`date +%s`
+  strDate=`date +%m/%d/%y%n`
+  echo "Stopping ${strDate} ${now}" >> "${CHUKWA_LOG_DIR}/hourly.log"
+
+  sleep 36000
+done

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh Fri Dec  5 12:30:14 2008
@@ -75,7 +75,13 @@
     fi
 fi
 
-export CHUKWA_LOG_DIR="$CHUKWA_HOME/logs"
+if [ -z ${CHUKWA_LOG_DIR} ]; then
+    export CHUKWA_LOG_DIR="$CHUKWA_HOME/logs"
+fi
+
+if [ -z ${CHUKWA_PID_DIR} ]; then
+    export CHUKWA_PID_DIR="${CHUKWA_HOME}/var/run"
+fi
 
 CHUKWA_VERSION=`cat ${CHUKWA_HOME}/bin/VERSION`
 
@@ -90,15 +96,35 @@
 fi
 
 export DATACONFIG=${CHUKWA_CONF_DIR}/mdl.xml
-COMMON=`ls ${CHUKWA_HOME}/lib/*.jar`
+COMMON=`ls ${CHUKWA_HOME}/lib/*.jar ${CHUKWA_HOME}/hadoopjars/commons*.jar`
 export COMMON=`echo ${COMMON} | sed 'y/ /:/'`
-export CHUKWA_CORE=${CHUKWA_HOME}/chukwa-core-${CHUKWA_VERSION}.jar
-export CHUKWA_AGENT=${CHUKWA_HOME}/chukwa-agent-${CHUKWA_VERSION}.jar
-export HADOOP_JAR=`ls ${HADOOP_HOME}/hadoop-*-core.jar`
+export CHUKWA_CORE=${CHUKWA_HOME}/build/chukwa-core-${CHUKWA_VERSION}.jar
+export CHUKWA_AGENT=${CHUKWA_HOME}/build/chukwa-agent-${CHUKWA_VERSION}.jar
 export CURRENT_DATE=`date +%Y%m%d%H%M`
 
+if [ -z ${HADOOP_JAR} ]; then
+  if [ -z ${HADOOP_HOME} ]; then
+        export HADOOP_HOME=../../..
+    fi
+    if [ -d ${HADOOP_HOME} ]; then
+        export HADOOP_JAR=`ls ${HADOOP_HOME}/build/hadoop-*-core.jar`
+        if [ -z ${HADOOP_JAR} ]; then
+            echo "Please make sure hadoop-*-core.jar exists in ${HADOOP_HOME}/build"
+            exit -1
+        fi
+    else
+        if [ -d ${CHUKWA_HOME}/hadoopjars ]; then
+            echo "WARNING: neither HADOOP_HOME nor HADOOP_JAR is set we we are reverting to defaults in $CHUKWA_HOME/hadoopjars dir"
+            export HADOOP_JAR=`ls ${CHUKWA_HOME}/hadoopjars/hadoop-*-core.jar`
+        else
+            echo "Please make sure hadoop-*-core.jar exists in ${CHUKWA_HOME}/hadoopjars"
+            exit -1
+        fi
+    fi
+fi
+
 if [ -z "$JAVA_HOME" ] ; then
-    export JAVA_HOME=/usr/lib/j2sdk1.5-sun
+  echo ERROR! You forgot to set JAVA_HOME in conf/chukwa-env.sh   
 fi
 
 export JPS=${JAVA_HOME}/bin/jps

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemon.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemon.sh?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemon.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-daemon.sh Fri Dec  5 12:30:14 2008
@@ -98,25 +98,31 @@
   (start)
     MAIL=`cat ${CHUKWA_HOME}/conf/alert.conf`
 
+    RANDOM=`date '+%s'`
+    PARTROL_HOUR=$[($RANDOM % 24)]
+    if [ ${PARTROL_HOUR} -gt 12 ]; then
+        PARTROL_HOUR2=$[${PARTROL_HOUR}-12]
+    else 
+        PARTROL_HOUR2=$[${PARTROL_HOUR}+12]
+    fi
     if [ "${WATCHDOG}" != "" ]; then
         mkdir -p ${CHUKWA_HOME}/var/tmp >&/dev/null
         crontab -l > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
         crontest=$?
 
         if [ "X${crontest}" != "X0" ]; then
-          cat > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} << CRON
-MAILTO=${MAIL}
-1,30 * * * * ${CHUKWA_HOME}/bin/watchdog.sh
-CRON
+          echo "MAILTO=${MAIL}" > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
         else
-          grep -v "${CHUKWA_HOME}/bin/watchdog.sh" ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} | grep -v MAILTO > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2
+          grep -v "${CHUKWA_HOME}/bin/watchdog.sh" ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} | grep -v MAILTO | grep -v "cat ${CHUKWA_HOME}/var/run/watchdog.out" | grep -v ${CHUKWA_HOME}/tools/expire.sh > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2
           echo "MAILTO=${MAIL}" > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
           cat ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2 >> ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
           rm -f ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2
-          cat >> ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} << CRON
-1,30 * * * * ${CHUKWA_HOME}/bin/watchdog.sh
-CRON
         fi
+        cat >> ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} << CRON
+*/5 * * * * ${CHUKWA_HOME}/bin/watchdog.sh > ${CHUKWA_HOME}/var/run/watchdog.out
+1 ${PARTROL_HOUR},${PARTROL_HOUR2} * * * /bin/bash -c "cat ${CHUKWA_HOME}/var/run/watchdog.out; cat /dev/null > ${CHUKWA_HOME}/var/run/watchdog.out"
+15 3 * * * ${CHUKWA_HOME}/tools/expire.sh 10 ${CHUKWA_LOG_DIR} nowait
+CRON
 
         # save crontab
         echo -n "Registering watchdog.."
@@ -152,7 +158,7 @@
 
     if [ "${WATCHDOG}" != "" ]; then
         # remove watchdog
-        crontab -l | grep -v ${CHUKWA_HOME}/bin/watchdog.sh > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+        crontab -l | grep -v ${CHUKWA_HOME}/bin/watchdog.sh | grep -v ${CHUKWA_HOME}/var/run/watchdog.out | grep -v ${CHUKWA_HOME}/tools/expire.sh > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
         crontab ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
         rm -f ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
     fi

Added: hadoop/core/trunk/src/contrib/chukwa/bin/dailyRolling.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/dailyRolling.sh?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/dailyRolling.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/dailyRolling.sh Fri Dec  5 12:30:14 2008
@@ -0,0 +1,32 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=$$
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+. "$bin"/chukwa-config.sh
+
+HADOOP_CONF_DIR="${HADOOP_HOME}/conf/"
+HADOOP_CMDE="${HADOOP_HOME}/bin/hadoop "
+
+  $HADOOP_CMDE jar ${CHUKWA_CORE} org.apache.hadoop.chukwa.extraction.demux.DailyChukwaRecordRolling rollInSequence true deleteRawdata true
+
+  previousDay=`date --date="2 day ago" +%Y%m%d`
+  #previousDay=`date -v -2d +%Y%m%d`
+  echo "deleting /chukwa/postprocess/srcSink${previousDay}_*"
+  $HADOOP_CMDE dfs -rmr "/chukwa/postprocess/srcSink${previousDay}_*"
+

Added: hadoop/core/trunk/src/contrib/chukwa/bin/dbAdmin.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/dbAdmin.sh?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/dbAdmin.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/dbAdmin.sh Fri Dec  5 12:30:14 2008
@@ -0,0 +1,63 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=$$
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+if [ "$CHUKWA_IDENT_STRING" = "" ]; then
+  export CHUKWA_IDENT_STRING="$USER"
+fi
+
+trap 'rm -f $CHUKWA_HOME/var/run/chukwa-$CHUKWA_IDENT_STRING-dbAdmin.sh.pid ${CHUKWA_HOME}/var/run/dbAdmin.pid; exit 0' 1 2 15
+EXP_DATE=`date +%Y-%m-%d`
+
+JVM_OPTS="-DAPP=dbAdmin -Dlog4j.configuration=chukwa-log4j.properties -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DDATACONFIG=${CHUKWA_CONF_DIR}/mdl.xml -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR}"
+
+echo "${pid}" > "${CHUKWA_HOME}/var/run/dbAdmin.pid"
+while [ 1 ]
+  do
+    start=`date +%s`
+    cat ${CHUKWA_CONF_DIR}/jdbc.conf | \
+    while read LINE; do
+        CLUSTER=`echo ${LINE} | cut -f 1 -d'='`
+        ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.TableCreator ${EXP_DATE} 7 
+        ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.TableCreator ${EXP_DATE} 30
+        ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.TableCreator ${EXP_DATE} 91
+        ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.TableCreator ${EXP_DATE} 365
+        ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.TableCreator ${EXP_DATE} 3650
+        ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.Aggregator 
+        ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.Consolidator 
+        ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.DataExpiration ${EXP_DATE} 7
+        ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.DataExpiration ${EXP_DATE} 30
+        ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.DataExpiration ${EXP_DATE} 91
+        ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.DataExpiration ${EXP_DATE} 365
+        ${JAVA_HOME}/bin/java -DCLUSTER=${CLUSTER} ${JVM_OPTS} org.apache.hadoop.chukwa.database.DataExpiration ${EXP_DATE} 3650
+    done
+    end=`date +%s`
+    duration=$(( $end - $start ))
+    if [ $duration -lt 300 ]; then
+        sleep=$(( 300 - $duration ))
+        SLEEP_COUNTER=`expr $sleep / 5`
+        while [ $SLEEP_COUNTER -gt 1 ]; do
+            sleep 5
+            SLEEP_COUNTER=`expr $SLEEP_COUNTER - 1`
+        done
+    fi
+done

Added: hadoop/core/trunk/src/contrib/chukwa/bin/dbLoader.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/dbLoader.sh?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/dbLoader.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/dbLoader.sh Fri Dec  5 12:30:14 2008
@@ -0,0 +1,25 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=$$
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+echo "${pid}" > "$CHUKWA_HOME/var/run/dbLoader.pid"
+${JAVA_HOME}/bin/java -DDATACONFIG=${CHUKWA_CONF_DIR}/mdl.xml -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.extraction.database.MetricDataLoader $1

Added: hadoop/core/trunk/src/contrib/chukwa/bin/dbSetup.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/dbSetup.sh?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/dbSetup.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/dbSetup.sh Fri Dec  5 12:30:14 2008
@@ -0,0 +1,32 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=$$
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+EXP_DATE=`date +%Y-%m-%d`
+echo -n "SETUP Database partition..."
+echo "${pid}" > "$CHUKWA_HOME/var/run/dbSetup.pid"
+${JAVA_HOME}/bin/java -DCLUSTER=$1 -DDATACONFIG=${CHUKWA_CONF_DIR}/mdl.xml -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.database.TableCreator ${EXP_DATE} 7 #>/dev/null 2>&1
+${JAVA_HOME}/bin/java -DCLUSTER=$1 -DDATACONFIG=${CHUKWA_CONF_DIR}/mdl.xml -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.database.TableCreator ${EXP_DATE} 30 >/dev/null 2>&1
+${JAVA_HOME}/bin/java -DCLUSTER=$1 -DDATACONFIG=${CHUKWA_CONF_DIR}/mdl.xml -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.database.TableCreator ${EXP_DATE} 91 >/dev/null 2>&1
+${JAVA_HOME}/bin/java -DCLUSTER=$1 -DDATACONFIG=${CHUKWA_CONF_DIR}/mdl.xml -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.database.TableCreator ${EXP_DATE} 365 >/dev/null 2>&1
+echo "done"
+rm -f "$CHUKWA_HOME/var/run/dbSetup.pid"

Added: hadoop/core/trunk/src/contrib/chukwa/bin/dumpArchive.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/dumpArchive.sh?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/dumpArchive.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/dumpArchive.sh Fri Dec  5 12:30:14 2008
@@ -0,0 +1,24 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=$$
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+${JAVA_HOME}/bin/java -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.util.DumpArchive $1

Added: hadoop/core/trunk/src/contrib/chukwa/bin/dumpDataType.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/dumpDataType.sh?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/dumpDataType.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/dumpDataType.sh Fri Dec  5 12:30:14 2008
@@ -0,0 +1,24 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=$$
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+${JAVA_HOME}/bin/java -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.util.DumpDataType $@

Added: hadoop/core/trunk/src/contrib/chukwa/bin/dumpRecord.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/dumpRecord.sh?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/dumpRecord.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/dumpRecord.sh Fri Dec  5 12:30:14 2008
@@ -0,0 +1,24 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=$$
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+${JAVA_HOME}/bin/java -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.util.DumpRecord $1

Added: hadoop/core/trunk/src/contrib/chukwa/bin/hourlyRolling.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/hourlyRolling.sh?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/hourlyRolling.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/hourlyRolling.sh Fri Dec  5 12:30:14 2008
@@ -0,0 +1,28 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=$$
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+. "$bin"/chukwa-config.sh
+
+HADOOP_CONF_DIR="${HADOOP_HOME}/conf/"
+HADOOP_CMDE="${HADOOP_HOME}/bin/hadoop "
+
+  $HADOOP_CMDE jar ${CHUKWA_CORE} org.apache.hadoop.chukwa.extraction.demux.HourlyChukwaRecordRolling rollInSequence true deleteRawdata true
+
+

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/jettyCollector.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/jettyCollector.sh?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/jettyCollector.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/jettyCollector.sh Fri Dec  5 12:30:14 2008
@@ -21,11 +21,17 @@
 
 . "$bin"/chukwa-config.sh
 
-echo "${pid}" > "${CHUKWA_PID_DIR}/Collector.pid"
+trap 'stop; exit 0' 1 2 15
 
+function stop {
+  echo -n "Shutting down Collector..."
+  ${JPS} | grep CollectorStub | cut -f 1 -d" " | xargs kill -TERM >&/dev/null
+  echo "done"
+  exit 0
+}
 
-if [ $# -lt 1 ]; then 
-${JAVA_HOME}/bin/java -classpath ${CLASSPATH}:${chukwaCore}:${common}:${HADOOP_JAR}:${CHUKWA_HOME}/conf:${HADOOP_HOME}/conf org.apache.hadoop.chukwa.datacollection.collector.CollectorStub 9310
-else
-${JAVA_HOME}/bin/java -classpath ${CLASSPATH}:${chukwaCore}:${common}:${HADOOP_JAR}:${CHUKWA_HOME}/conf:${HADOOP_HOME}/conf org.apache.hadoop.chukwa.datacollection.collector.CollectorStub $@
+if [ "X$1" = "Xstop" ]; then
+  stop
 fi
+
+${JAVA_HOME}/bin/java -DAPP=collector -Dlog4j.configuration=chukwa-log4j.properties -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -classpath ${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.datacollection.collector.CollectorStub 

Added: hadoop/core/trunk/src/contrib/chukwa/bin/netstat.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/netstat.sh?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/netstat.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/netstat.sh Fri Dec  5 12:30:14 2008
@@ -0,0 +1,2 @@
+#!/bin/bash
+netstat -a | grep ESTABLISH | grep -v '        0      0'

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/nodeActivityDataLoader.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/nodeActivityDataLoader.sh?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/nodeActivityDataLoader.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/nodeActivityDataLoader.sh Fri Dec  5 12:30:14 2008
@@ -40,7 +40,7 @@
 fi
 
 if [ ${EXISTS} -lt 1 ]; then
-    ${JAVA_HOME}/bin/java -DPERIOD=600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=PbsNodes -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec "${nodeActivityCmde}" &
+    ${JAVA_HOME}/bin/java -DPERIOD=600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=PbsNodes -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec "${nodeActivityCmde}" &
 fi
 
 

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/processSinkFiles.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/processSinkFiles.sh?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/processSinkFiles.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/processSinkFiles.sh Fri Dec  5 12:30:14 2008
@@ -20,40 +20,137 @@
 bin=`cd "$bin"; pwd`
 . "$bin"/chukwa-config.sh
 
-echo "${pid}" > "${CHUKWA_PID_DIR}/ProcessSinkFiles.pid"
+if [ "$CHUKWA_IDENT_STRING" = "" ]; then
+  export CHUKWA_IDENT_STRING="$USER"
+fi
+
+trap 'remove_cron;rm -f $CHUKWA_HOME/var/run/chukwa-$CHUKWA_IDENT_STRING-processSinkFiles.sh.pid ${CHUKWA_HOME}/var/run/ProcessSinkFiles.pid; exit 0' 1 2 15
+echo "${pid}" > "$CHUKWA_HOME/var/run/ProcessSinkFiles.pid"
 
 HADOOP_CONF_DIR="${HADOOP_HOME}/conf/"
 HADOOP_CMDE="${HADOOP_HOME}/bin/hadoop "
 
+function remove_cron {
+    mkdir -p ${CHUKWA_HOME}/var/tmp >&/dev/null
+    crontab -l | grep -v ${CHUKWA_HOME}/bin/hourlyRolling.sh > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+    cat /tmp/cron.${CURRENT_DATE} | grep -v ${CHUKWA_HOME}/bin/dailyRolling.sh > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2
+    crontab ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2
+    rm -f ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+    rm -f ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2
+}
+
+function add_cron {
+    mkdir -p ${CHUKWA_HOME}/var/tmp >&/dev/null
+    crontab -l > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+    crontest=$?
+
+    if [ "X${crontest}" != "X0" ]; then
+      cat > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} << CRON
+16 * * * * ${CHUKWA_HOME}/bin/hourlyRolling.sh >& ${CHUKWA_HOME}/logs/hourly.log
+30 1 * * * ${CHUKWA_HOME}/bin/dailyRolling.sh >& ${CHUKWA_HOME}/logs/dailyRolling.log
+CRON
+    else
+      grep -v "${CHUKWA_HOME}/bin/hourlyRolling.sh" ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}  | grep -v "${CHUKWA_HOME}/bin/dailyRolling.sh" > ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2
+      mv ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}.2 ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+      cat >> ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} << CRON
+16 * * * * ${CHUKWA_HOME}/bin/hourlyRolling.sh >& ${CHUKWA_HOME}/logs/hourly.log
+30 1 * * * ${CHUKWA_HOME}/bin/dailyRolling.sh >& ${CHUKWA_HOME}/logs/dailyRolling.log
+CRON
+    fi
+
+    # save crontab
+    echo -n "Registering cron jobs.."
+    crontab ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE} > /dev/null 2>&1
+    rm -f ${CHUKWA_HOME}/var/tmp/cron.${CURRENT_DATE}
+    echo "done"
+}
+
+if [ "X$1" = "Xstop" ]; then
+  echo -n "Shutting down processSinkFiles.sh..."
+  kill -TERM `cat ${CHUKWA_HOME}/var/run/ProcessSinkFiles.pid`
+  echo "done"
+  exit 0
+fi
+
+if [ "X$1" = "Xwatchdog" ]; then
+  add_cron
+fi
+
 while [ 1 ]
-do 
+ do
+  debugDate=`date `
+  startTime=`date +%s`
+  now=`date +%Y%m%d_%H_%M%S`
+  strDate=`date +%Y%m%d_%H_%M%S`
+  srcDoneHdfsDir="/chukwa/postprocess/srcSink$now/"
   
   
-  now=`date +%s`
-  strDate=`date +%m/%d/%y%n`
-  srcDoneHdfsDir="/chukwa/tmp/srcDone$now/"
-  srcEventHdfsDir="/chukwa/tmp/srcEvent$now/"
+  destArchiveDir=`date +%Y%m%d/%H/%M%S`
 
   echo "Running $strDate $now" >> "${CHUKWA_LOG_DIR}/mr.log"
 
-  echo "srcDoneHdfsDir: $srcDoneHdfsDir srcEventHdfsDir: $srcEventHdfsDir" >> "${MR_LOG}"
+  echo "srcDoneHdfsDir: $srcDoneHdfsDir " >> "${CHUKWA_LOG_DIR}/mr.log"
 
-  $HADOOP_CMDE dfs -mkdir $srcDoneHdfsDir
+  $HADOOP_CMDE dfs -mkdir $srcDoneHdfsDir/doneFile
   echo "done with mkdir" >> "${CHUKWA_LOG_DIR}/mr.log"
  
-  $HADOOP_CMDE dfs -mv "/chukwa/logs/*/*.done" $srcDoneHdfsDir
-  echo "done with mv logs" >> "${CHUKWA_LOG_DIR}/mr.log"
+  $HADOOP_CMDE dfs -mv "/chukwa/logs/*/*.done" ${srcDoneHdfsDir}/doneFile
+  endMoveTime=`date +%s`
+  moveDuration=$(( $endMoveTime - $startTime))
+  echo "moveDuration $moveDuration" >> "${CHUKWA_LOG_DIR}/mr.log"
+  debugDate=`date `
+  echo "$debugDate done with mv logs" >> "${CHUKWA_LOG_DIR}/mr.log"
  
-  $HADOOP_CMDE jar ${chukwaCore} org.apache.hadoop.chukwa.extraction.demux.Demux -r 2 $srcDoneHdfsDir $srcEventHdfsDir
-  echo "done with demux job" >> "${CHUKWA_LOG_DIR}/mr.log"
- 
-  $HADOOP_CMDE jar ${chukwaCore} org.apache.hadoop.chukwa.extraction.demux.MoveOrMergeLogFile $srcEventHdfsDir ${chuwaRecordsRepository}
-  echo "done with MoveOrMergeLogFile" >> "${CHUKWA_LOG_DIR}/mr.log"
-
+  # Build the archive
+  $HADOOP_CMDE jar  ${CHUKWA_CORE} org.apache.hadoop.chukwa.extraction.archive.ChukwaArchiveBuilder Stream ${srcDoneHdfsDir}/doneFile /chukwa/archives/raw/${destArchiveDir}
+  endArchiveTime=`date +%s`
+  archiveDuration=$(( $endArchiveTime - $endMoveTime))
+  echo "archiveDuration $archiveDuration" >> "${CHUKWA_LOG_DIR}/mr.log"
+  debugDate=`date `
+  echo "$debugDate done with chuckwaArchiveBuilder" >> "${CHUKWA_LOG_DIR}/mr.log"
+  
+  
+  ## Archive available call all processors
+  
+  
+  $HADOOP_CMDE jar  ${CHUKWA_CORE} org.apache.hadoop.chukwa.extraction.demux.Demux -Dmapred.compress.map.output=true -Dmapred.map.output.compression.codec=org.apache.hadoop.io.compress.LzoCodec -Dmapred.output.compress=true -Dmapred.output.compression.type=BLOCK -r 4 /chukwa/archives/raw/${destArchiveDir} ${srcDoneHdfsDir}/demux
+  endDemuxTime=`date +%s`
+  demuxDuration=$(( $endDemuxTime - $endArchiveTime))
+  echo "demuxDuration $demuxDuration" >> "${CHUKWA_LOG_DIR}/mr.log"
+  debugDate=`date `
+  echo "$debugDate done with demux job" >> "${CHUKWA_LOG_DIR}/mr.log"
+   
+  ${JAVA_HOME}/bin/java -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -Dlog4j.configuration=log4j.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools}:${CHUKWA_HOME}/conf org.apache.hadoop.chukwa.extraction.database.DatabaseLoader "${srcDoneHdfsDir}/demux" SystemMetrics Df Hadoop_dfs Hadoop_jvm Hadoop_mapred Hadoop_rpc MSSRGraph MRJobCounters NodeActivity HodJob HodMachine Hadoop_dfs_FSDirectory Hadoop_dfs_FSNamesystem Hadoop_dfs_datanode Hadoop_dfs_namenode Hadoop_jvm_metrics Hadoop_mapred_job Hadoop_mapred_jobtracker Hadoop_mapred_shuffleOutput Hadoop_mapred_tasktracker Hadoop_rpc_metrics
+  endDbLoaderTime=`date +%s`
+  dbLoaderDuration=$(( $endDbLoaderTime - $endDemuxTime))
+  echo "dbLoaderDuration $dbLoaderDuration" >> "${CHUKWA_LOG_DIR}/mr.log"
+  debugDate=`date `
+  echo "$debugDate done with dbLoader job" >> "${CHUKWA_LOG_DIR}/mr.log"
+   
+  $HADOOP_CMDE jar ${CHUKWA_CORE} org.apache.hadoop.chukwa.extraction.demux.MoveToRepository ${srcDoneHdfsDir}/demux ${chuwaRecordsRepository}
+  endMoveToRepoTime=`date +%s`
+  moveToRepoDuration=$(( $endMoveToRepoTime - $endDbLoaderTime))
+  echo "moveToRepoDuration $moveToRepoDuration" >> "${CHUKWA_LOG_DIR}/mr.log"
+  debugDate=`date `
+  echo "$debugDate done with MoveToRepository" >> "${CHUKWA_LOG_DIR}/mr.log"
+  
   now=`date +%s`
   strDate=`date +%m/%d/%y%n`
-  echo "Stopping ${strDate} ${now}" >> "${CHUKWA_LOG_DIR}/mr.log"
-
- sleep 300
-
+  debugDate=`date `
+  echo "$debugDate Stopping ${strDate} ${now}" >> "${CHUKWA_LOG_DIR}/mr.log"
+  
+  endTime=`date +%s`
+  duration=$(( $endTime - $startTime))
+  echo "Duration: $duration s" >> "${CHUKWA_LOG_DIR}/mr.log"
+  
+  if [ $duration -lt 300 ]; then
+   sleepTime=$(( 300 - $duration)) 
+   echo "Sleep: $sleepTime s" >> "${CHUKWA_LOG_DIR}/mr.log"
+   SLEEP_COUNTER=`expr $sleepTime / 5`
+   while [ $SLEEP_COUNTER -gt 1 ]; do
+       sleep 5
+       SLEEP_COUNTER=`expr $SLEEP_COUNTER - 1`
+   done
+  fi
 done
+

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/slaves.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/slaves.sh?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/slaves.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/slaves.sh Fri Dec  5 12:30:14 2008
@@ -21,7 +21,7 @@
 # Environment Variables
 #
 #   CHUKWA_SLAVES    File naming remote hosts.
-#     Default is ${CHUKWA_CONF_DIR}/chukwa-slaves.
+#     Default is ${CHUKWA_CONF_DIR}/chukwa-agents.
 #   CHUKWA_CONF_DIR  Alternate conf dir. Default is ${CHUKWA_HOME}/conf.
 #   CHUKWA_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
 #   CHUKWA_SSH_OPTS Options passed to ssh when running remote commands.
@@ -51,7 +51,7 @@
 
 if [ "$HOSTLIST" = "" ]; then
   if [ "$CHUKWA_SLAVES" = "" ]; then
-    export HOSTLIST="${CHUKWA_CONF_DIR}/chukwa-slaves"
+    export HOSTLIST="${CHUKWA_CONF_DIR}/chukwa-agents"
   else
     export HOSTLIST="${CHUKWA_SLAVES}"
   fi

Added: hadoop/core/trunk/src/contrib/chukwa/bin/start-agents.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/start-agents.sh?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/start-agents.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/start-agents.sh Fri Dec  5 12:30:14 2008
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This script is modeled after the parallel script in Hadoop
+# Start chukwa agent daemons on cluster slaves
+# Run this on a collector node.
+
+usage="Usage: start-agent.sh"
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+# start chukwa agents
+"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR --hosts slaves start agent.sh

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/start-all.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/start-all.sh?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/start-all.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/start-all.sh Fri Dec  5 12:30:14 2008
@@ -23,11 +23,14 @@
 
 . "$bin"/chukwa-config.sh
 
+# start collectors
+"$bin"/start-collectors.sh --config $CHUKWA_CONF_DIR
+
+# start agents
+"$bin"/start-agents.sh --config $CHUKWA_CONF_DIR
+
 # start probes
 "$bin"/start-probes.sh --config $CHUKWA_CONF_DIR
 
 # start data processors
 "$bin"/start-data-processors.sh --config $CHUKWA_CONF_DIR
-
-# start collectors
-"$bin"/start-collectors.sh --config $CHUKWA_CONF_DIR

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/start-probes.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/start-probes.sh?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/start-probes.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/start-probes.sh Fri Dec  5 12:30:14 2008
@@ -27,15 +27,13 @@
 
 . "$bin"/chukwa-config.sh
 
-# start agents
-"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR --watchdog start agent.sh
 # start system data loader daemons
 "$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR start systemDataLoader.sh
 
 # start torque data loader daemons
-if [ ${TORQUE_HOME} != "" ]; then
+if [ "x${TORQUE_HOME}" != "x" ]; then
   "$bin"/chukwa-daemon.sh --config $CHUKWA_CONF_DIR start torqueDataLoader.sh
 fi
-if [ ${nodeActivityCmde} != "" ]; then
+if [ "x${nodeActivityCmde}" != "x" ]; then
   "$bin"/chukwa-daemon.sh --config $CHUKWA_CONF_DIR start nodeActivityDataLoader.sh
 fi

Added: hadoop/core/trunk/src/contrib/chukwa/bin/stop-agents.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/stop-agents.sh?rev=723855&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/stop-agents.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/stop-agents.sh Fri Dec  5 12:30:14 2008
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Stop collectors.  Run this on master node.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR --hosts slaves --watchdog stop agent.sh

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/stop-all.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/stop-all.sh?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/stop-all.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/stop-all.sh Fri Dec  5 12:30:14 2008
@@ -25,4 +25,5 @@
 
 "$bin"/stop-probes.sh --config $CHUKWA_CONF_DIR
 "$bin"/stop-data-processors.sh --config $CHUKWA_CONF_DIR
+"$bin"/stop-agents.sh --config $CHUKWA_CONF_DIR
 "$bin"/stop-collectors.sh --config $CHUKWA_CONF_DIR

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/stop-data-processors.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/stop-data-processors.sh?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/stop-data-processors.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/stop-data-processors.sh Fri Dec  5 12:30:14 2008
@@ -22,24 +22,52 @@
 . "$bin"/chukwa-config.sh
 
 # stop processSinkFiles.sh
-pidFile=$CHUKWA_HOME/var/run/ProcessSinkFiles.pid
+pidFile=$CHUKWA_PID_DIR/ProcessSinkFiles.pid
 if [ -f $pidFile ]; then  
    echo -n "Shutting down Data Processors.."
    DP_PID=`head ${pidFile}`
    kill -TERM ${DP_PID}
-   rm ${pidFile}
+   for i in 1 2 5; do
+       test_pid=`ps ax | grep ${DP_PID} | grep -v grep | grep processSinkFiles.sh | wc -l`
+       if [ $test_pid -ge 1 ]; then
+           sleep $i
+           kill -TERM ${DP_PID}
+       else
+           break
+       fi
+   done
+   test_pid=`ps ax | grep ${DP_PID} | grep -v grep | grep processSinkFiles.sh | wc -l`
+   if [ $test_pid -ge 1 ]; then
+       kill -9 ${DBADMIN_PID} &>/dev/null
+   fi
+   rm -f ${pidFile}
+   rm -f $CHUKWA_PID_DIR/chukwa-$CHUKWA_IDENT_STRING-processSinkFiles.sh.pid
    echo "done"
 else
   echo " no $pidFile"
 fi
 
 # stop dbAdmin.sh
-pidFile=$CHUKWA_HOME/var/run/dbAdmin.pid
+pidFile=$CHUKWA_PID_DIR/dbAdmin.pid
 if [ -f $pidFile ]; then  
    echo -n "Shutting down Database Admin.."
    DBADMIN_PID=`head ${pidFile}`
    kill -TERM ${DBADMIN_PID}
-   rm ${pidFile}
+   for i in 1 2 5; do
+       test_pid=`ps ax | grep ${DBADMIN_PID} | grep -v grep | grep dbAdmin.sh | wc -l`
+       if [ $test_pid -ge 1 ]; then
+           sleep $i
+           kill -TERM ${DBADMIN_PID}
+       else
+           break
+       fi
+   done
+   test_pid=`ps ax | grep ${DBADMIN_PID} | grep -v grep | grep dbAdmin.sh | wc -l`
+   if [ $test_pid -ge 1 ]; then
+       kill -9 ${DBADMIN_PID} &>/dev/null
+   fi
+   rm -f ${pidFile}
+   rm -f $CHUKWA_PID_DIR/chukwa-$CHUKWA_IDENT_STRING-dbAdmin.sh.pid
    echo "done"
 else
   echo " no $pidFile"

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/stop-probes.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/stop-probes.sh?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/stop-probes.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/stop-probes.sh Fri Dec  5 12:30:14 2008
@@ -26,4 +26,3 @@
 "$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR stop systemDataLoader.sh
 "$bin"/chukwa-daemon.sh --config $CHUKWA_CONF_DIR stop torqueDataLoader.sh
 "$bin"/chukwa-daemon.sh --config $CHUKWA_CONF_DIR stop nodeActivityDataLoader.sh
-"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR --watchdog stop agent.sh

Modified: hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh?rev=723855&r1=723854&r2=723855&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh (original)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh Fri Dec  5 12:30:14 2008
@@ -22,19 +22,34 @@
 
 JVM_OPTS="-Xms4M -Xmx4M"
 
-if [ "X$1" = "Xstop" ]; then
+trap 'shutdown' 1 2 15
+
+function shutdown {
   echo -n "Shutting down System Data Loader..."
   if [ -f ${CHUKWA_HOME}/var/run/Sar-data-loader.pid ]; then
-    kill -TERM `cat ${CHUKWA_HOME}/var/run/Sar-data-loader.pid`
+    kill -9 `cat ${CHUKWA_HOME}/var/run/Sar-data-loader.pid`
   fi
   if [ -f ${CHUKWA_HOME}/var/run/Iostat-data-loader.pid ]; then
-    kill -TERM `cat ${CHUKWA_HOME}/var/run/Iostat-data-loader.pid`
+    kill -9 `cat ${CHUKWA_HOME}/var/run/Iostat-data-loader.pid`
   fi
   if [ -f ${CHUKWA_HOME}/var/run/Top-data-loader.pid ]; then
-    kill -TERM `cat ${CHUKWA_HOME}/var/run/Top-data-loader.pid`
+    kill -9 `cat ${CHUKWA_HOME}/var/run/Top-data-loader.pid`
   fi
   if [ -f ${CHUKWA_HOME}/var/run/Df-data-loader.pid ]; then
-    kill -TERM `cat ${CHUKWA_HOME}/var/run/Df-data-loader.pid`
+    kill -9 `cat ${CHUKWA_HOME}/var/run/Df-data-loader.pid`
+  fi
+  if [ -f ${CHUKWA_HOME}/var/run/Netstat-data-loader.pid ]; then
+    kill -9 `cat ${CHUKWA_HOME}/var/run/Netstat-data-loader.pid`
+  fi
+  rm -f $CHUKWA_HOME/var/run/chukwa-$CHUKWA_IDENT_STRING-systemDataLoader.sh.pid
+  echo "done"
+  exit 0
+}
+
+if [ "X$1" = "Xstop" ]; then
+  echo -n "Shutting down System Data Loader..."
+  if [ -f $CHUKWA_HOME/var/run/chukwa-$CHUKWA_IDENT_STRING-systemDataLoader.sh.pid ]; then
+    kill -TERM `head $CHUKWA_HOME/var/run/chukwa-$CHUKWA_IDENT_STRING-systemDataLoader.sh.pid`
   fi
   echo "done"
   exit 0
@@ -58,7 +73,7 @@
 fi
 
 if [ ${EXISTS} -lt 1 ]; then
-    ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Sar -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec sar -q -r -n FULL 55 &
+    ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=Sar -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec sar -q -r -n FULL 55 &
 fi
 
 EXISTS=0
@@ -72,7 +87,7 @@
 fi
 
 if [ ${EXISTS} -lt 1 ]; then
-  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Iostat -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec iostat -x 55 2 &
+  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=Iostat -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec iostat -x 55 2 &
 fi
 
 EXISTS=0
@@ -86,7 +101,7 @@
 fi
 
 if [ ${EXISTS} -lt 1 ]; then
-  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Top -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec top -b -n 1 -c &
+  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=Top -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec top -b -n 1 -c &
 fi
 
 EXISTS=0
@@ -100,7 +115,27 @@
 fi
 
 if [ ${EXISTS} -lt 1 ]; then
-  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DRECORD_TYPE=Df -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec df -l &
+  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=Df -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec df -l &
+fi
+
+EXISTS=0
+pidFile="${CHUKWA_HOME}/var/run/Netstat-data-loader.pid"
+if [ -f $pidFile ]; then
+  pid=`head ${pidFile}`
+  ChildPIDRunningStatus=`${JPS} | grep ${pid} | grep Exec | grep -v grep | wc -l`
+  if [ $ChildPIDRunningStatus -ge 1 ]; then
+    EXISTS=1
+  fi
+fi
+
+if [ ${EXISTS} -lt 1 ]; then
+  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=60 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=Netstat -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec ${CHUKWA_HOME}/bin/netstat.sh &
 fi
 
 echo "done"
+
+while [ 1 ]
+do
+    # sleep until shutdown signal has been sent.
+    sleep 5
+done



Mime
View raw message