Return-Path:
Delivered-To: apmail-hadoop-core-commits-archive@www.apache.org
Received: (qmail 49429 invoked from network); 18 Sep 2008 01:41:37 -0000
Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2)
by minotaur.apache.org with SMTP; 18 Sep 2008 01:41:37 -0000
Received: (qmail 73637 invoked by uid 500); 18 Sep 2008 01:41:34 -0000
Delivered-To: apmail-hadoop-core-commits-archive@hadoop.apache.org
Received: (qmail 73612 invoked by uid 500); 18 Sep 2008 01:41:34 -0000
Mailing-List: contact core-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
List-Help:
List-Unsubscribe:
List-Post:
List-Id:
Reply-To: core-dev@hadoop.apache.org
Delivered-To: mailing list core-commits@hadoop.apache.org
Received: (qmail 73601 invoked by uid 99); 18 Sep 2008 01:41:34 -0000
Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136)
by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 17 Sep 2008 18:41:34 -0700
X-ASF-Spam-Status: No, hits=-2000.0 required=10.0
tests=ALL_TRUSTED
X-Spam-Check-By: apache.org
Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4)
by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 18 Sep 2008 01:40:43 +0000
Received: by eris.apache.org (Postfix, from userid 65534)
id 85CB7238896F; Wed, 17 Sep 2008 18:41:16 -0700 (PDT)
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r696533 - in /hadoop/core/trunk: ./
src/core/org/apache/hadoop/filecache/ src/core/org/apache/hadoop/fs/
src/core/org/apache/hadoop/io/compress/ src/core/org/apache/hadoop/util/
Date: Thu, 18 Sep 2008 01:41:16 -0000
To: core-commits@hadoop.apache.org
From: omalley@apache.org
X-Mailer: svnmailer-1.0.8
Message-Id: <20080918014116.85CB7238896F@eris.apache.org>
X-Virus-Checked: Checked by ClamAV on apache.org
Author: omalley
Date: Wed Sep 17 18:41:15 2008
New Revision: 696533
URL: http://svn.apache.org/viewvc?rev=696533&view=rev
Log:
HADOOP-4184. Break the module dependencies between core, hdfs, and
mapred.
From: Tom White
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/core/org/apache/hadoop/filecache/DistributedCache.java
hadoop/core/trunk/src/core/org/apache/hadoop/fs/FileSystem.java
hadoop/core/trunk/src/core/org/apache/hadoop/io/compress/CompressionCodecFactory.java
hadoop/core/trunk/src/core/org/apache/hadoop/util/NativeCodeLoader.java
hadoop/core/trunk/src/core/org/apache/hadoop/util/Shell.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=696533&r1=696532&r2=696533&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Sep 17 18:41:15 2008
@@ -332,6 +332,9 @@
HADOOP-4186. Factor LineReader out of LineRecordReader. (tomwhite via
omalley)
+ HADOOP-4184. Break the module dependencies between core, hdfs, and
+ mapred. (tomwhite via omalley)
+
OPTIMIZATIONS
HADOOP-3556. Removed lock contention in MD5Hash by changing the
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/filecache/DistributedCache.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/filecache/DistributedCache.java?rev=696533&r1=696532&r2=696533&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/filecache/DistributedCache.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/filecache/DistributedCache.java Wed Sep 17 18:41:15 2008
@@ -24,10 +24,6 @@
import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.*;
import org.apache.hadoop.fs.*;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.Reducer;
import java.net.URI;
@@ -39,7 +35,8 @@
*
*
* Applications specify the files, via urls (hdfs:// or http://) to be cached
- * via the {@link JobConf}. The DistributedCache
assumes that the
+ * via the {@link org.apache.hadoop.mapred.JobConf}.
+ * The DistributedCache
assumes that the
* files specified via hdfs:// urls are already present on the
* {@link FileSystem} at the path specified by the url.
*
@@ -85,7 +82,8 @@
* DistributedCache.addCacheArchive(new URI("/myapp/mytgz.tgz", job);
* DistributedCache.addCacheArchive(new URI("/myapp/mytargz.tar.gz", job);
*
- * 3. Use the cached files in the {@link Mapper} or {@link Reducer}:
+ * 3. Use the cached files in the {@link org.apache.hadoop.mapred.Mapper}
+ * or {@link org.apache.hadoop.mapred.Reducer}:
*
* public static class MapClass extends MapReduceBase
* implements Mapper<K, V, K, V> {
@@ -111,8 +109,8 @@
*
*
*
- * @see JobConf
- * @see JobClient
+ * @see org.apache.hadoop.mapred.JobConf
+ * @see org.apache.hadoop.mapred.JobClient
*/
public class DistributedCache {
// cacheID to cacheStatus mapping
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/FileSystem.java?rev=696533&r1=696532&r2=696533&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/FileSystem.java Wed Sep 17 18:41:15 2008
@@ -84,7 +84,9 @@
* Parse the cmd-line args, starting at i. Remove consumed args
* from array. We expect param in the form:
* '-local | -dfs '
+ * @deprecated Consider using {@link GenericOptionsParser} instead.
*/
+ @Deprecated
public static FileSystem parseArgs(String argv[], int i, Configuration conf) throws IOException {
/**
if (argv.length - i < 1) {
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/io/compress/CompressionCodecFactory.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/io/compress/CompressionCodecFactory.java?rev=696533&r1=696532&r2=696533&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/io/compress/CompressionCodecFactory.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/io/compress/CompressionCodecFactory.java Wed Sep 17 18:41:15 2008
@@ -185,7 +185,7 @@
* @param args
*/
public static void main(String[] args) throws Exception {
- Configuration conf = new org.apache.hadoop.mapred.JobConf();
+ Configuration conf = new Configuration();
CompressionCodecFactory factory = new CompressionCodecFactory(conf);
boolean encode = false;
for(int i=0; i < args.length; ++i) {
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/util/NativeCodeLoader.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/util/NativeCodeLoader.java?rev=696533&r1=696532&r2=696533&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/util/NativeCodeLoader.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/util/NativeCodeLoader.java Wed Sep 17 18:41:15 2008
@@ -18,8 +18,9 @@
package org.apache.hadoop.util;
-import org.apache.commons.logging.*;
-import org.apache.hadoop.mapred.JobConf;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
/**
* A helper to load the native hadoop code i.e. libhadoop.so.
@@ -65,24 +66,24 @@
/**
* Return if native hadoop libraries, if present, can be used for this job.
- * @param jobConf job configuration
+ * @param conf configuration
*
* @return true
if native hadoop libraries, if present, can be
* used for this job; false
otherwise.
*/
- public boolean getLoadNativeLibraries(JobConf jobConf) {
- return jobConf.getBoolean("hadoop.native.lib", true);
+ public boolean getLoadNativeLibraries(Configuration conf) {
+ return conf.getBoolean("hadoop.native.lib", true);
}
/**
* Set if native hadoop libraries, if present, can be used for this job.
*
- * @param jobConf job configuration
+ * @param conf configuration
* @param loadNativeLibraries can native hadoop libraries be loaded
*/
- public void setLoadNativeLibraries(JobConf jobConf,
+ public void setLoadNativeLibraries(Configuration conf,
boolean loadNativeLibraries) {
- jobConf.setBoolean("hadoop.native.lib", loadNativeLibraries);
+ conf.setBoolean("hadoop.native.lib", loadNativeLibraries);
}
}
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/util/Shell.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/util/Shell.java?rev=696533&r1=696532&r2=696533&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/util/Shell.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/util/Shell.java Wed Sep 17 18:41:15 2008
@@ -17,15 +17,15 @@
*/
package org.apache.hadoop.util;
-import java.util.Map;
+import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.io.BufferedReader;
+import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.conf.Configuration;
/**
* A base class for running a Unix command.
@@ -65,19 +65,19 @@
*
* It also checks to ensure that we are running on a *nix platform else
* (e.g. in Cygwin/Windows) it returns null
.
- * @param job job configuration
+ * @param conf configuration
* @return a String[]
with the ulimit command arguments or
* null
if we are running on a non *nix platform or
* if the limit is unspecified.
*/
- public static String[] getUlimitMemoryCommand(JobConf job) {
+ public static String[] getUlimitMemoryCommand(Configuration conf) {
// ulimit isn't supported on Windows
if (WINDOWS) {
return null;
}
- // get the memory limit from the JobConf
- String ulimit = job.get("mapred.child.ulimit");
+ // get the memory limit from the configuration
+ String ulimit = conf.get("mapred.child.ulimit");
if (ulimit == null) {
return null;
}