Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0B895E722 for ; Fri, 30 Nov 2012 19:59:33 +0000 (UTC) Received: (qmail 65788 invoked by uid 500); 30 Nov 2012 19:59:32 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 65708 invoked by uid 500); 30 Nov 2012 19:59:32 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 65700 invoked by uid 99); 30 Nov 2012 19:59:32 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 30 Nov 2012 19:59:32 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED,NORMAL_HTTP_TO_IP X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 30 Nov 2012 19:59:30 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 580722388C6C; Fri, 30 Nov 2012 19:59:04 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1415815 [3/3] - in /hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project: ./ conf/ dev-support/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/h... Date: Fri, 30 Nov 2012 19:58:54 -0000 To: mapreduce-commits@hadoop.apache.org From: suresh@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20121130195904.580722388C6C@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java Fri Nov 30 19:58:09 2012 @@ -22,7 +22,6 @@ import static org.apache.hadoop.yarn.web import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; -import static org.apache.hadoop.yarn.webapp.view.JQueryUI.THEMESWITCHER_ID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.postInitID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit; @@ -54,7 +53,6 @@ public class HsView extends TwoColumnLay protected void commonPreHead(Page.HTML<_> html) { set(ACCORDION_ID, "nav"); set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}"); - set(THEMESWITCHER_ID, "themeswitcher"); } /* @@ -84,17 +82,15 @@ public class HsView extends TwoColumnLay */ private String jobsTableInit() { return tableInit(). + append(", 'aaData': jobsTableData"). + append(", bDeferRender: true"). + append(", bProcessing: true"). + // Sort by id upon page load append(", aaSorting: [[2, 'desc']]"). append(", aoColumnDefs:["). - // Maps Total - append("{'sType':'numeric', 'bSearchable': false, 'aTargets': [ 7 ] }"). - // Maps Completed - append(",{'sType':'numeric', 'bSearchable': false, 'aTargets': [ 8 ] }"). - // Reduces Total - append(",{'sType':'numeric', 'bSearchable': false, 'aTargets': [ 9 ] }"). - // Reduces Completed - append(",{'sType':'numeric', 'bSearchable': false, 'aTargets': [ 10 ] }"). + // Maps Total, Maps Completed, Reduces Total and Reduces Completed + append("{'sType':'numeric', 'bSearchable': false, 'aTargets': [ 7, 8, 9, 10 ] }"). append("]}"). toString(); } Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/HistoryInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/HistoryInfo.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/HistoryInfo.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/HistoryInfo.java Fri Nov 30 19:58:09 2012 @@ -22,17 +22,20 @@ import javax.xml.bind.annotation.XmlAcce import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer; import org.apache.hadoop.util.VersionInfo; @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) public class HistoryInfo { + protected long startedOn; protected String hadoopVersion; protected String hadoopBuildVersion; protected String hadoopVersionBuiltOn; public HistoryInfo() { + this.startedOn = JobHistoryServer.historyServerTimeStamp; this.hadoopVersion = VersionInfo.getVersion(); this.hadoopBuildVersion = VersionInfo.getBuildVersion(); this.hadoopVersionBuiltOn = VersionInfo.getDate(); @@ -50,4 +53,8 @@ public class HistoryInfo { return this.hadoopVersionBuiltOn; } + public long getStartedOn() { + return this.startedOn; + } + } Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java Fri Nov 30 19:58:09 2012 @@ -36,6 +36,7 @@ import org.apache.hadoop.mapreduce.v2.ap import org.apache.hadoop.mapreduce.v2.app.job.Job; import org.apache.hadoop.mapreduce.v2.hs.HistoryContext; import org.apache.hadoop.mapreduce.v2.hs.JobHistory; +import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer; import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobsInfo; import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.yarn.Clock; @@ -344,21 +345,24 @@ public class TestHsWebServices extends J } public void verifyHsInfoGeneric(String hadoopVersionBuiltOn, - String hadoopBuildVersion, String hadoopVersion) { + String hadoopBuildVersion, String hadoopVersion, long startedon) { WebServicesTestUtils.checkStringMatch("hadoopVersionBuiltOn", VersionInfo.getDate(), hadoopVersionBuiltOn); WebServicesTestUtils.checkStringMatch("hadoopBuildVersion", VersionInfo.getBuildVersion(), hadoopBuildVersion); WebServicesTestUtils.checkStringMatch("hadoopVersion", VersionInfo.getVersion(), hadoopVersion); + assertEquals("startedOn doesn't match: ", + JobHistoryServer.historyServerTimeStamp, startedon); } public void verifyHSInfo(JSONObject info, TestAppContext ctx) throws JSONException { - assertEquals("incorrect number of elements", 3, info.length()); + assertEquals("incorrect number of elements", 4, info.length()); verifyHsInfoGeneric(info.getString("hadoopVersionBuiltOn"), - info.getString("hadoopBuildVersion"), info.getString("hadoopVersion")); + info.getString("hadoopBuildVersion"), info.getString("hadoopVersion"), + info.getLong("startedOn")); } public void verifyHSInfoXML(String xml, TestAppContext ctx) @@ -376,7 +380,8 @@ public class TestHsWebServices extends J verifyHsInfoGeneric( WebServicesTestUtils.getXmlString(element, "hadoopVersionBuiltOn"), WebServicesTestUtils.getXmlString(element, "hadoopBuildVersion"), - WebServicesTestUtils.getXmlString(element, "hadoopVersion")); + WebServicesTestUtils.getXmlString(element, "hadoopVersion"), + WebServicesTestUtils.getXmlLong(element, "startedOn")); } } Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java Fri Nov 30 19:58:09 2012 @@ -50,37 +50,40 @@ public class TestNLineInputFormat extend Job job = Job.getInstance(conf); Path file = new Path(workDir, "test.txt"); - int seed = new Random().nextInt(); - Random random = new Random(seed); - localFs.delete(workDir, true); FileInputFormat.setInputPaths(job, workDir); int numLinesPerMap = 5; NLineInputFormat.setNumLinesPerSplit(job, numLinesPerMap); - // for a variety of lengths for (int length = 0; length < MAX_LENGTH; - length += random.nextInt(MAX_LENGTH / 10) + 1) { + length += 1) { + // create a file with length entries Writer writer = new OutputStreamWriter(localFs.create(file)); try { for (int i = 0; i < length; i++) { - writer.write(Integer.toString(i)); + writer.write(Integer.toString(i)+" some more text"); writer.write("\n"); } } finally { writer.close(); } - checkFormat(job, numLinesPerMap); + int lastN = 0; + if (length != 0) { + lastN = length % 5; + if (lastN == 0) { + lastN = 5; + } + } + checkFormat(job, numLinesPerMap, lastN); } } - void checkFormat(Job job, int expectedN) + void checkFormat(Job job, int expectedN, int lastN) throws IOException, InterruptedException { NLineInputFormat format = new NLineInputFormat(); List splits = format.getSplits(job); - // check all splits except last one int count = 0; - for (int i = 0; i < splits.size() -1; i++) { + for (int i = 0; i < splits.size(); i++) { assertEquals("There are no split locations", 0, splits.get(i).getLocations().length); TaskAttemptContext context = MapReduceTestUtil. @@ -104,8 +107,13 @@ public class TestNLineInputFormat extend } finally { reader.close(); } - assertEquals("number of lines in split is " + expectedN , - expectedN, count); + if ( i == splits.size() - 1) { + assertEquals("number of lines in split(" + i + ") is wrong" , + lastN, count); + } else { + assertEquals("number of lines in split(" + i + ") is wrong" , + expectedN, count); + } } } Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java Fri Nov 30 19:58:09 2012 @@ -35,26 +35,28 @@ import org.apache.hadoop.hdfs.MiniDFSClu import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.SleepJob; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; +import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; -@SuppressWarnings("deprecation") -@Ignore public class TestBinaryTokenFile { + private static final String KEY_SECURITY_TOKEN_FILE_NAME = "key-security-token-file"; + private static final String DELEGATION_TOKEN_KEY = "Hdfs"; + // my sleep class static class MySleepMapper extends SleepJob.SleepMapper { /** @@ -63,29 +65,65 @@ public class TestBinaryTokenFile { @Override public void map(IntWritable key, IntWritable value, Context context) throws IOException, InterruptedException { - // get token storage and a key - Credentials ts = context.getCredentials(); - Collection> dts = ts.getAllTokens(); + // get context token storage: + final Credentials contextCredentials = context.getCredentials(); + final Collection> contextTokenCollection = contextCredentials.getAllTokens(); + for (Token t : contextTokenCollection) { + System.out.println("Context token: [" + t + "]"); + } + if (contextTokenCollection.size() != 2) { // one job token and one delegation token + // fail the test: + throw new RuntimeException("Exactly 2 tokens are expected in the contextTokenCollection: " + + "one job token and one delegation token, but was found " + contextTokenCollection.size() + " tokens."); + } - if(dts.size() != 2) { // one job token and one delegation token - throw new RuntimeException("tokens are not available"); // fail the test + final Token dt = contextCredentials.getToken(new Text(DELEGATION_TOKEN_KEY)); + if (dt == null) { + throw new RuntimeException("Token for key ["+DELEGATION_TOKEN_KEY+"] not found in the job context."); } - Token dt = ts.getToken(new Text("Hdfs")); + String tokenFile0 = context.getConfiguration().get(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY); + if (tokenFile0 != null) { + throw new RuntimeException("Token file key ["+MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY+"] found in the configuration. It should have been removed from the configuration."); + } - //Verify that dt is same as the token in the file - String tokenFile = context.getConfiguration().get( - "mapreduce.job.credentials.binary"); - Credentials cred = new Credentials(); - cred.readTokenStorageStream(new DataInputStream(new FileInputStream( + final String tokenFile = context.getConfiguration().get(KEY_SECURITY_TOKEN_FILE_NAME); + if (tokenFile == null) { + throw new RuntimeException("Token file key ["+KEY_SECURITY_TOKEN_FILE_NAME+"] not found in the job configuration."); + } + final Credentials binaryCredentials = new Credentials(); + binaryCredentials.readTokenStorageStream(new DataInputStream(new FileInputStream( tokenFile))); - for (Token t : cred.getAllTokens()) { - if (!dt.equals(t)) { - throw new RuntimeException( - "Delegation token in job is not same as the token passed in file." - + " tokenInFile=" + t + ", dt=" + dt); - } + final Collection> binaryTokenCollection = binaryCredentials.getAllTokens(); + if (binaryTokenCollection.size() != 1) { + throw new RuntimeException("The token collection read from file ["+tokenFile+"] must have size = 1."); + } + final Token binTok = binaryTokenCollection.iterator().next(); + System.out.println("The token read from binary file: t = [" + binTok + "]"); + // Verify that dt is same as the token in the file: + if (!dt.equals(binTok)) { + throw new RuntimeException( + "Delegation token in job is not same as the token passed in file:" + + " tokenInFile=[" + binTok + "], dt=[" + dt + "]."); + } + + // Now test the user tokens. + final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + // Print all the UGI tokens for diagnostic purposes: + final Collection> ugiTokenCollection = ugi.getTokens(); + for (Token t: ugiTokenCollection) { + System.out.println("UGI token: [" + t + "]"); + } + final Token ugiToken + = ugi.getCredentials().getToken(new Text(DELEGATION_TOKEN_KEY)); + if (ugiToken == null) { + throw new RuntimeException("Token for key ["+DELEGATION_TOKEN_KEY+"] not found among the UGI tokens."); + } + if (!ugiToken.equals(binTok)) { + throw new RuntimeException( + "UGI token is not same as the token passed in binary file:" + + " tokenInBinFile=[" + binTok + "], ugiTok=[" + ugiToken + "]."); } super.map(key, value, context); @@ -118,13 +156,20 @@ public class TestBinaryTokenFile { TokenCache.obtainTokensForNamenodesInternal(cred1, new Path[] { p1 }, job.getConfiguration()); for (Token t : cred1.getAllTokens()) { - cred2.addToken(new Text("Hdfs"), t); + cred2.addToken(new Text(DELEGATION_TOKEN_KEY), t); } DataOutputStream os = new DataOutputStream(new FileOutputStream( binaryTokenFileName.toString())); - cred2.writeTokenStorageToStream(os); - os.close(); - job.getConfiguration().set("mapreduce.job.credentials.binary", + try { + cred2.writeTokenStorageToStream(os); + } finally { + os.close(); + } + job.getConfiguration().set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, + binaryTokenFileName.toString()); + // NB: the MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY key now gets deleted from config, + // so it's not accessible in the job's config. So, we use another key to pass the file name into the job configuration: + job.getConfiguration().set(KEY_SECURITY_TOKEN_FILE_NAME, binaryTokenFileName.toString()); } catch (IOException e) { Assert.fail("Exception " + e); @@ -132,39 +177,53 @@ public class TestBinaryTokenFile { } } - private static MiniMRCluster mrCluster; + private static MiniMRYarnCluster mrCluster; private static MiniDFSCluster dfsCluster; + private static final Path TEST_DIR = new Path(System.getProperty("test.build.data","/tmp")); private static final Path binaryTokenFileName = new Path(TEST_DIR, "tokenFile.binary"); - private static int numSlaves = 1; - private static JobConf jConf; + + private static final int numSlaves = 1; // num of data nodes + private static final int noOfNMs = 1; + private static Path p1; @BeforeClass public static void setUp() throws Exception { - Configuration conf = new Configuration(); - dfsCluster = new MiniDFSCluster(conf, numSlaves, true, null); - jConf = new JobConf(conf); - mrCluster = new MiniMRCluster(0, 0, numSlaves, - dfsCluster.getFileSystem().getUri().toString(), 1, null, null, null, - jConf); + final Configuration conf = new Configuration(); + + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); + conf.set(YarnConfiguration.RM_PRINCIPAL, "jt_id/" + SecurityUtil.HOSTNAME_PATTERN + "@APACHE.ORG"); + + final MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); + builder.checkExitOnShutdown(true); + builder.numDataNodes(numSlaves); + builder.format(true); + builder.racks(null); + dfsCluster = builder.build(); + + mrCluster = new MiniMRYarnCluster(TestBinaryTokenFile.class.getName(), noOfNMs); + mrCluster.init(conf); + mrCluster.start(); - NameNodeAdapter.getDtSecretManager(dfsCluster.getNamesystem()).startThreads(); - FileSystem fs = dfsCluster.getFileSystem(); + NameNodeAdapter.getDtSecretManager(dfsCluster.getNamesystem()).startThreads(); + FileSystem fs = dfsCluster.getFileSystem(); p1 = new Path("file1"); p1 = fs.makeQualified(p1); } @AfterClass public static void tearDown() throws Exception { - if(mrCluster != null) - mrCluster.shutdown(); - mrCluster = null; - if(dfsCluster != null) + if(mrCluster != null) { + mrCluster.stop(); + mrCluster = null; + } + if(dfsCluster != null) { dfsCluster.shutdown(); - dfsCluster = null; + dfsCluster = null; + } } /** @@ -173,31 +232,24 @@ public class TestBinaryTokenFile { */ @Test public void testBinaryTokenFile() throws IOException { - - System.out.println("running dist job"); - - // make sure JT starts - jConf = mrCluster.createJobConf(); + Configuration conf = mrCluster.getConfig(); // provide namenodes names for the job to get the delegation tokens for - String nnUri = dfsCluster.getURI(0).toString(); - jConf.set(MRJobConfig.JOB_NAMENODES, nnUri + "," + nnUri); - // job tracker principla id.. - jConf.set(JTConfig.JT_USER_NAME, "jt_id"); + final String nnUri = dfsCluster.getURI(0).toString(); + conf.set(MRJobConfig.JOB_NAMENODES, nnUri + "," + nnUri); // using argument to pass the file name - String[] args = { + final String[] args = { "-m", "1", "-r", "1", "-mt", "1", "-rt", "1" }; - int res = -1; try { - res = ToolRunner.run(jConf, new MySleepJob(), args); + res = ToolRunner.run(conf, new MySleepJob(), args); } catch (Exception e) { - System.out.println("Job failed with" + e.getLocalizedMessage()); + System.out.println("Job failed with " + e.getLocalizedMessage()); e.printStackTrace(System.out); fail("Job failed"); } - assertEquals("dist job res is not 0", res, 0); + assertEquals("dist job res is not 0:", 0, res); } } Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java Fri Nov 30 19:58:09 2012 @@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.RunningJ import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -52,6 +53,8 @@ public class TestEncryptedShuffle { private static final String BASEDIR = System.getProperty("test.build.dir", "target/test-dir") + "/" + TestEncryptedShuffle.class.getSimpleName(); + + private String classpathDir; @BeforeClass public static void setUp() throws Exception { @@ -62,27 +65,12 @@ public class TestEncryptedShuffle { @Before public void createCustomYarnClasspath() throws Exception { - String classpathDir = - KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class); - - URL url = Thread.currentThread().getContextClassLoader(). - getResource("mrapp-generated-classpath"); - File f = new File(url.getPath()); - BufferedReader reader = new BufferedReader(new FileReader(f)); - String cp = reader.readLine(); - cp = cp + ":" + classpathDir; - f = new File(classpathDir, "mrapp-generated-classpath"); - Writer writer = new FileWriter(f); - writer.write(cp); - writer.close(); + classpathDir = KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class); new File(classpathDir, "core-site.xml").delete(); } @After public void cleanUpMiniClusterSpecialConfig() throws Exception { - String classpathDir = - KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class); - new File(classpathDir, "mrapp-generated-classpath").delete(); new File(classpathDir, "core-site.xml").delete(); String keystoresDir = new File(BASEDIR).getAbsolutePath(); KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, classpathDir); @@ -98,6 +86,9 @@ public class TestEncryptedShuffle { conf.set("dfs.block.access.token.enable", "false"); conf.set("dfs.permissions", "true"); conf.set("hadoop.security.authentication", "simple"); + String cp = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH) + + File.pathSeparator + classpathDir; + conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, cp); dfsCluster = new MiniDFSCluster(conf, 1, true, null); FileSystem fileSystem = dfsCluster.getFileSystem(); fileSystem.mkdirs(new Path("/tmp")); @@ -113,8 +104,6 @@ public class TestEncryptedShuffle { mrCluster = MiniMRClientClusterFactory.create(this.getClass(), 1, conf); // so the minicluster conf is avail to the containers. - String classpathDir = - KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class); Writer writer = new FileWriter(classpathDir + "/core-site.xml"); mrCluster.getConfig().writeXml(writer); writer.close(); Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java Fri Nov 30 19:58:09 2012 @@ -37,6 +37,7 @@ import java.io.RandomAccessFile; import java.net.InetSocketAddress; import java.net.URL; import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -45,6 +46,7 @@ import java.util.concurrent.ConcurrentHa import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; import javax.crypto.SecretKey; @@ -106,6 +108,7 @@ import org.jboss.netty.handler.ssl.SslHa import org.jboss.netty.handler.stream.ChunkedWriteHandler; import org.jboss.netty.util.CharsetUtil; +import com.google.common.base.Charsets; import com.google.common.util.concurrent.ThreadFactoryBuilder; public class ShuffleHandler extends AbstractService @@ -119,10 +122,16 @@ public class ShuffleHandler extends Abst public static final String SHUFFLE_READAHEAD_BYTES = "mapreduce.shuffle.readahead.bytes"; public static final int DEFAULT_SHUFFLE_READAHEAD_BYTES = 4 * 1024 * 1024; + // pattern to identify errors related to the client closing the socket early + // idea borrowed from Netty SslHandler + private static final Pattern IGNORABLE_ERROR_MESSAGE = Pattern.compile( + "^.*(?:connection.*reset|connection.*closed|broken.*pipe).*$", + Pattern.CASE_INSENSITIVE); + private int port; private ChannelFactory selector; private final ChannelGroup accepted = new DefaultChannelGroup(); - private HttpPipelineFactory pipelineFact; + protected HttpPipelineFactory pipelineFact; private int sslFileBufferSize; /** @@ -318,13 +327,17 @@ public class ShuffleHandler extends Abst } } + protected Shuffle getShuffle(Configuration conf) { + return new Shuffle(conf); + } + class HttpPipelineFactory implements ChannelPipelineFactory { final Shuffle SHUFFLE; private SSLFactory sslFactory; public HttpPipelineFactory(Configuration conf) throws Exception { - SHUFFLE = new Shuffle(conf); + SHUFFLE = getShuffle(conf); if (conf.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY, MRConfig.SHUFFLE_SSL_ENABLED_DEFAULT)) { sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf); @@ -464,7 +477,7 @@ public class ShuffleHandler extends Abst lastMap.addListener(ChannelFutureListener.CLOSE); } - private void verifyRequest(String appid, ChannelHandlerContext ctx, + protected void verifyRequest(String appid, ChannelHandlerContext ctx, HttpRequest request, HttpResponse response, URL requestUri) throws IOException { SecretKey tokenSecret = secretManager.retrieveTokenSecret(appid); @@ -490,7 +503,8 @@ public class ShuffleHandler extends Abst SecureShuffleUtils.verifyReply(urlHashStr, enc_str, tokenSecret); // verification passed - encode the reply String reply = - SecureShuffleUtils.generateHash(urlHashStr.getBytes(), tokenSecret); + SecureShuffleUtils.generateHash(urlHashStr.getBytes(Charsets.UTF_8), + tokenSecret); response.setHeader(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH, reply); if (LOG.isDebugEnabled()) { int len = reply.length(); @@ -564,12 +578,12 @@ public class ShuffleHandler extends Abst return writeFuture; } - private void sendError(ChannelHandlerContext ctx, + protected void sendError(ChannelHandlerContext ctx, HttpResponseStatus status) { sendError(ctx, "", status); } - private void sendError(ChannelHandlerContext ctx, String message, + protected void sendError(ChannelHandlerContext ctx, String message, HttpResponseStatus status) { HttpResponse response = new DefaultHttpResponse(HTTP_1_1, status); response.setHeader(CONTENT_TYPE, "text/plain; charset=UTF-8"); @@ -588,6 +602,16 @@ public class ShuffleHandler extends Abst if (cause instanceof TooLongFrameException) { sendError(ctx, BAD_REQUEST); return; + } else if (cause instanceof IOException) { + if (cause instanceof ClosedChannelException) { + LOG.debug("Ignoring closed channel error", cause); + return; + } + String message = String.valueOf(cause.getMessage()); + if (IGNORABLE_ERROR_MESSAGE.matcher(message).matches()) { + LOG.debug("Ignoring client socket close", cause); + return; + } } LOG.error("Shuffle error: ", cause); Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java Fri Nov 30 19:58:09 2012 @@ -17,17 +17,35 @@ */ package org.apache.hadoop.mapred; +import static org.apache.hadoop.test.MetricsAsserts.assertCounter; +import static org.apache.hadoop.test.MetricsAsserts.assertGauge; +import static org.apache.hadoop.test.MetricsAsserts.getMetrics; +import static org.apache.hadoop.test.MockitoMaker.make; +import static org.apache.hadoop.test.MockitoMaker.stub; +import static org.jboss.netty.buffer.ChannelBuffers.wrappedBuffer; +import static org.junit.Assert.assertEquals; + +import java.io.DataInputStream; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.mapreduce.task.reduce.ShuffleHeader; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.MetricsSource; -import org.apache.hadoop.metrics2.impl.MetricsSystemImpl; import org.apache.hadoop.metrics2.MetricsSystem; -import static org.apache.hadoop.test.MetricsAsserts.*; - +import org.apache.hadoop.metrics2.impl.MetricsSystemImpl; +import org.jboss.netty.channel.Channel; import org.jboss.netty.channel.ChannelFuture; - +import org.jboss.netty.channel.ChannelHandlerContext; +import org.jboss.netty.handler.codec.http.HttpRequest; +import org.jboss.netty.handler.codec.http.HttpResponse; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.junit.Assert; import org.junit.Test; -import static org.junit.Assert.*; -import static org.apache.hadoop.test.MockitoMaker.*; public class TestShuffleHandler { static final long MiB = 1024 * 1024; @@ -69,4 +87,76 @@ public class TestShuffleHandler { assertCounter("ShuffleOutputsOK", succeeded, rb); assertGauge("ShuffleConnections", connections, rb); } + + @Test + public void testClientClosesConnection() throws Exception { + final ArrayList failures = new ArrayList(1); + Configuration conf = new Configuration(); + conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); + ShuffleHandler shuffleHandler = new ShuffleHandler() { + @Override + protected Shuffle getShuffle(Configuration conf) { + // replace the shuffle handler with one stubbed for testing + return new Shuffle(conf) { + @Override + protected void verifyRequest(String appid, ChannelHandlerContext ctx, + HttpRequest request, HttpResponse response, URL requestUri) + throws IOException { + } + @Override + protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, + Channel ch, String user, String jobId, String mapId, int reduce) + throws IOException { + // send a shuffle header and a lot of data down the channel + // to trigger a broken pipe + ShuffleHeader header = + new ShuffleHeader("attempt_12345_1_m_1_0", 5678, 5678, 1); + DataOutputBuffer dob = new DataOutputBuffer(); + header.write(dob); + ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength())); + dob = new DataOutputBuffer(); + for (int i=0; i<100000; ++i) { + header.write(dob); + } + return ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength())); + } + @Override + protected void sendError(ChannelHandlerContext ctx, + HttpResponseStatus status) { + if (failures.size() == 0) { + failures.add(new Error()); + ctx.getChannel().close(); + } + } + @Override + protected void sendError(ChannelHandlerContext ctx, String message, + HttpResponseStatus status) { + if (failures.size() == 0) { + failures.add(new Error()); + ctx.getChannel().close(); + } + } + }; + } + }; + shuffleHandler.init(conf); + shuffleHandler.start(); + + // simulate a reducer that closes early by reading a single shuffle header + // then closing the connection + URL url = new URL("http://127.0.0.1:" + + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + + "/mapOutput?job=job_12345_1&reduce=1&map=attempt_12345_1_m_1_0"); + HttpURLConnection conn = (HttpURLConnection)url.openConnection(); + conn.connect(); + DataInputStream input = new DataInputStream(conn.getInputStream()); + Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + ShuffleHeader header = new ShuffleHeader(); + header.readFields(input); + input.close(); + + shuffleHandler.stop(); + Assert.assertTrue("sendError called when client closed connection", + failures.size() == 0); + } } Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/findbugs-exclude.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/findbugs-exclude.xml?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/findbugs-exclude.xml (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/findbugs-exclude.xml Fri Nov 30 19:58:09 2012 @@ -60,4 +60,10 @@ + + + + + + Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml Fri Nov 30 19:58:09 2012 @@ -103,6 +103,11 @@ hsqldb provided + + com.google.guava + guava + provided + Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java Fri Nov 30 19:58:09 2012 @@ -22,7 +22,9 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.io.PrintStream; +import java.io.PrintWriter; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -50,6 +52,8 @@ import org.apache.hadoop.mapreduce.lib.o import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import com.google.common.base.Charsets; + /** * A map/reduce program that uses Bailey-Borwein-Plouffe to compute exact * digits of Pi. @@ -151,7 +155,8 @@ public class BaileyBorweinPlouffe extend LOG.info("Writing text output to " + outfile); final OutputStream outputstream = fs.create(outfile); try { - final PrintStream out = new PrintStream(outputstream, true); + final PrintWriter out = new PrintWriter( + new OutputStreamWriter(outputstream, Charsets.UTF_8), true); // write hex text print(out, hex.iterator(), "Pi = 0x3.", "%02X", 5, 5); out.println("Total number of hexadecimal digits is " @@ -184,7 +189,7 @@ public class BaileyBorweinPlouffe extend } /** Print out elements in a nice format. */ - private static void print(PrintStream out, Iterator iterator, + private static void print(PrintWriter out, Iterator iterator, String prefix, String format, int elementsPerGroup, int groupsPerLine) { final StringBuilder sb = new StringBuilder("\n"); for (int i = 0; i < prefix.length(); i++) Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMean.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMean.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMean.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMean.java Fri Nov 30 19:58:09 2012 @@ -37,6 +37,8 @@ import org.apache.hadoop.mapreduce.lib.o import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import com.google.common.base.Charsets; + public class WordMean extends Configured implements Tool { private double mean = 0; @@ -125,7 +127,7 @@ public class WordMean extends Configured // average = total sum / number of elements; try { - br = new BufferedReader(new InputStreamReader(fs.open(file))); + br = new BufferedReader(new InputStreamReader(fs.open(file), Charsets.UTF_8)); long count = 0; long length = 0; @@ -151,7 +153,9 @@ public class WordMean extends Configured System.out.println("The mean is: " + theMean); return theMean; } finally { - br.close(); + if (br != null) { + br.close(); + } } } Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMedian.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMedian.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMedian.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMedian.java Fri Nov 30 19:58:09 2012 @@ -38,6 +38,8 @@ import org.apache.hadoop.mapreduce.lib.o import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import com.google.common.base.Charsets; + public class WordMedian extends Configured implements Tool { private double median = 0; @@ -127,7 +129,7 @@ public class WordMedian extends Configur BufferedReader br = null; try { - br = new BufferedReader(new InputStreamReader(fs.open(file))); + br = new BufferedReader(new InputStreamReader(fs.open(file), Charsets.UTF_8)); int num = 0; String line; @@ -157,7 +159,9 @@ public class WordMedian extends Configur } } } finally { - br.close(); + if (br != null) { + br.close(); + } } // error, no median found return -1; Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordStandardDeviation.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordStandardDeviation.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordStandardDeviation.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordStandardDeviation.java Fri Nov 30 19:58:09 2012 @@ -37,6 +37,8 @@ import org.apache.hadoop.mapreduce.lib.o import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import com.google.common.base.Charsets; + public class WordStandardDeviation extends Configured implements Tool { private double stddev = 0; @@ -135,7 +137,7 @@ public class WordStandardDeviation exten double stddev = 0; BufferedReader br = null; try { - br = new BufferedReader(new InputStreamReader(fs.open(file))); + br = new BufferedReader(new InputStreamReader(fs.open(file), Charsets.UTF_8)); long count = 0; long length = 0; long square = 0; @@ -166,7 +168,9 @@ public class WordStandardDeviation exten stddev = Math.sqrt((term - mean)); System.out.println("The standard deviation is: " + stddev); } finally { - br.close(); + if (br != null) { + br.close(); + } } return stddev; } Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java Fri Nov 30 19:58:09 2012 @@ -33,6 +33,8 @@ import org.apache.hadoop.mapreduce.lib.i import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.*; +import com.google.common.base.Charsets; + /** * Launch a distributed pentomino solver. * It generates a complete list of prefixes of length N with each unique prefix @@ -137,9 +139,9 @@ public class DistributedPentomino extend fs.mkdirs(dir); List splits = pent.getSplits(depth); Path input = new Path(dir, "part1"); - PrintStream file = - new PrintStream(new BufferedOutputStream - (fs.create(input), 64*1024)); + PrintWriter file = + new PrintWriter(new OutputStreamWriter(new BufferedOutputStream + (fs.create(input), 64*1024), Charsets.UTF_8)); for(int[] prefix: splits) { for(int i=0; i < prefix.length; ++i) { if (i != 0) { Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Sudoku.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Sudoku.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Sudoku.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Sudoku.java Fri Nov 30 19:58:09 2012 @@ -21,6 +21,8 @@ package org.apache.hadoop.examples.danci import java.io.*; import java.util.*; +import com.google.common.base.Charsets; + /** * This class uses the dancing links algorithm from Knuth to solve sudoku * puzzles. It has solved 42x42 puzzles in 1.02 seconds. @@ -133,7 +135,8 @@ public class Sudoku { * @param stream The input stream to read the data from */ public Sudoku(InputStream stream) throws IOException { - BufferedReader file = new BufferedReader(new InputStreamReader(stream)); + BufferedReader file = new BufferedReader( + new InputStreamReader(stream, Charsets.UTF_8)); String line = file.readLine(); List result = new ArrayList(); while (line != null) { Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java Fri Nov 30 19:58:09 2012 @@ -19,9 +19,11 @@ package org.apache.hadoop.examples.pi; import java.io.BufferedReader; import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.util.ArrayList; import java.util.Collections; @@ -32,6 +34,8 @@ import java.util.TreeMap; import org.apache.hadoop.examples.pi.math.Bellard; import org.apache.hadoop.examples.pi.math.Bellard.Parameter; +import com.google.common.base.Charsets; + /** A class for parsing outputs */ public final class Parser { static final String VERBOSE_PROPERTY = "pi.parser.verbose"; @@ -71,7 +75,8 @@ public final class Parser { for(Parameter p : Parameter.values()) m.put(p, new ArrayList()); - final BufferedReader in = new BufferedReader(new FileReader(f)); + final BufferedReader in = new BufferedReader( + new InputStreamReader(new FileInputStream(f), Charsets.UTF_8)); try { for(String line; (line = in.readLine()) != null; ) try { @@ -127,7 +132,8 @@ public final class Parser { Collections.sort(results); final PrintWriter out = new PrintWriter( - new FileWriter(new File(outputdir, p + ".txt")), true); + new OutputStreamWriter(new FileOutputStream( + new File(outputdir, p + ".txt")), Charsets.UTF_8), true); try { for(int i = 0; i < results.size(); i++) out.println(DistSum.taskResult2string(p + "." + i, results.get(i))); Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Util.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Util.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Util.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Util.java Fri Nov 30 19:58:09 2012 @@ -19,9 +19,10 @@ package org.apache.hadoop.examples.pi; import java.io.BufferedReader; import java.io.File; -import java.io.FileWriter; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.io.OutputStreamWriter; import java.io.PrintStream; import java.io.PrintWriter; import java.text.SimpleDateFormat; @@ -46,6 +47,8 @@ import org.apache.hadoop.fs.permission.F import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.util.ToolRunner; +import com.google.common.base.Charsets; + /** Utility methods */ public class Util { /** Output stream */ @@ -81,7 +84,7 @@ public class Util { final long t = System.currentTimeMillis(); final long delta = t - (isAccumulative? start: previous); if (s != null) { - out.format("%15dms (=%-15s: %s\n", delta, millis2String(delta) + ")", s); + out.format("%15dms (=%-15s: %s%n", delta, millis2String(delta) + ")", s); out.flush(); } previous = t; @@ -203,16 +206,16 @@ public class Util { throw new IllegalArgumentException("dir (=" + dir + ") is not a directory."); } - private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("-yyyyMMdd-HHmmssSSS"); /** Create a writer of a local file. */ public static PrintWriter createWriter(File dir, String prefix) throws IOException { checkDirectory(dir); - + + SimpleDateFormat dateFormat = new SimpleDateFormat("-yyyyMMdd-HHmmssSSS"); for(;;) { final File f = new File(dir, - prefix + DATE_FORMAT.format(new Date(System.currentTimeMillis())) + ".txt"); + prefix + dateFormat.format(new Date(System.currentTimeMillis())) + ".txt"); if (!f.exists()) - return new PrintWriter(new FileWriter(f)); + return new PrintWriter(new OutputStreamWriter(new FileOutputStream(f), Charsets.UTF_8)); try {Thread.sleep(10);} catch (InterruptedException e) {} } @@ -286,7 +289,8 @@ public class Util { final List results = new ArrayList(); for(FileStatus status : fs.listStatus(outdir)) { if (status.getPath().getName().startsWith("part-")) { - final BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(status.getPath()))); + final BufferedReader in = new BufferedReader( + new InputStreamReader(fs.open(status.getPath()), Charsets.UTF_8)); try { for(String line; (line = in.readLine()) != null; ) results.add(TaskResult.valueOf(line)); @@ -305,7 +309,7 @@ public class Util { static void writeResults(String name, List results, FileSystem fs, String dir) throws IOException { final Path outfile = new Path(dir, name + ".txt"); Util.out.println(name + "> writing results to " + outfile); - final PrintStream out = new PrintStream(fs.create(outfile), true); + final PrintWriter out = new PrintWriter(new OutputStreamWriter(fs.create(outfile), Charsets.UTF_8), true); try { for(TaskResult r : results) out.println(r); Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java Fri Nov 30 19:58:09 2012 @@ -29,6 +29,8 @@ import org.apache.hadoop.mapreduce.Input import org.apache.hadoop.mapreduce.lib.input.FileSplit; import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; +import com.google.common.base.Charsets; + class TeraScheduler { static String USE = "mapreduce.terasort.use.terascheduler"; private static final Log LOG = LogFactory.getLog(TeraScheduler.class); @@ -73,7 +75,8 @@ class TeraScheduler { List readFile(String filename) throws IOException { List result = new ArrayList(10000); - BufferedReader in = new BufferedReader(new FileReader(filename)); + BufferedReader in = new BufferedReader( + new InputStreamReader(new FileInputStream(filename), Charsets.UTF_8)); String line = in.readLine(); while (line != null) { result.add(line);