hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject svn commit: r1596354 - in /hbase/branches/0.98: hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
Date Tue, 20 May 2014 18:27:32 GMT
Author: tedyu
Date: Tue May 20 18:27:32 2014
New Revision: 1596354

URL: http://svn.apache.org/r1596354
Log:
HBASE-11104 IntegrationTestImportTsv#testRunFromOutputCommitter misses credential initialization
(Vandana)


Modified:
    hbase/branches/0.98/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java
    hbase/branches/0.98/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java

Modified: hbase/branches/0.98/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.98/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java?rev=1596354&r1=1596353&r2=1596354&view=diff
==============================================================================
--- hbase/branches/0.98/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java
(original)
+++ hbase/branches/0.98/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java
Tue May 20 18:27:32 2014
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.client.Sc
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobClient;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.OutputCommitter;
@@ -60,6 +61,7 @@ import org.apache.hadoop.mapreduce.lib.i
 import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
@@ -257,7 +259,8 @@ public class IntegrationTestImportTsv im
       // look a lot like TestImportTsv#testMROnTable.
       final String table = format("%s-%s-child", NAME, context.getJobID());
       final String cf = "FAM";
-
+      String fileLocation = System.getenv(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION);
+      conf.set(ImportTsv.CREDENTIALS_LOCATION, fileLocation);
       String[] args = {
           "-D" + ImportTsv.COLUMNS_CONF_KEY + "=HBASE_ROW_KEY,FAM:A,FAM:B",
           "-D" + ImportTsv.SEPARATOR_CONF_KEY + "=\u001b",
@@ -271,7 +274,9 @@ public class IntegrationTestImportTsv im
       } catch (Exception e) {
         throw new IOException("Underlying MapReduce job failed. Aborting commit.", e);
       } finally {
-        util.deleteTable(table);
+        if (util.getHBaseAdmin().tableExists(table)) {
+          util.deleteTable(table);
+        }
       }
     }
   }
@@ -362,6 +367,10 @@ public class IntegrationTestImportTsv im
     JobLaunchingOutputFormat.setOutputPath(job, outputPath);
     TableMapReduceUtil.addDependencyJars(job);
     addTestDependencyJars(job.getConfiguration());
+    TableMapReduceUtil.initCredentials(job);
+    JobClient jc = new JobClient(job.getConfiguration());
+    job.getCredentials().addToken(new Text("my_mr_token"),
+      jc.getDelegationToken(new Text("renewer")));
 
     // Job launched by the OutputCommitter will fail if dependency jars are
     // not shipped properly.

Modified: hbase/branches/0.98/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.98/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java?rev=1596354&r1=1596353&r2=1596354&view=diff
==============================================================================
--- hbase/branches/0.98/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
(original)
+++ hbase/branches/0.98/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
Tue May 20 18:27:32 2014
@@ -20,11 +20,13 @@ package org.apache.hadoop.hbase.mapreduc
 
 import static java.lang.String.format;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Set;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -49,6 +51,7 @@ import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
@@ -77,6 +80,9 @@ public class ImportTsv extends Configure
   public final static String BULK_OUTPUT_CONF_KEY = "importtsv.bulk.output";
   public final static String TIMESTAMP_CONF_KEY = "importtsv.timestamp";
   public final static String JOB_NAME_CONF_KEY = "mapred.job.name";
+  //This config is used to propagate credentials from parent MR jobs which launch
+  //ImportTSV jobs. SEE IntegrationTestImportTsv.
+  public final static String CREDENTIALS_LOCATION = "credentials_location";
   // TODO: the rest of these configs are used exclusively by TsvImporterMapper.
   // Move them out of the tool and let the mapper handle its own validation.
   public final static String SKIP_LINES_CONF_KEY = "importtsv.skip.bad.lines";
@@ -394,7 +400,6 @@ public class ImportTsv extends Configure
       throws IOException, ClassNotFoundException {
 
     HBaseAdmin admin = new HBaseAdmin(conf);
-
     // Support non-XML supported characters
     // by re-encoding the passed separator as a Base64 string.
     String actualSeparator = conf.get(SEPARATOR_CONF_KEY);
@@ -416,9 +421,14 @@ public class ImportTsv extends Configure
     FileInputFormat.setInputPaths(job, inputDir);
     job.setInputFormatClass(TextInputFormat.class);
     job.setMapperClass(mapperClass);
-
     String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY);
     String columns[] = conf.getStrings(COLUMNS_CONF_KEY);
+    if(StringUtils.isNotEmpty(conf.get(CREDENTIALS_LOCATION))) {
+      String fileLoc = conf.get(CREDENTIALS_LOCATION);
+      Credentials cred = Credentials.readTokenStorageFile(new File(fileLoc), conf);
+      job.getCredentials().addAll(cred);
+    }
+
     if (hfileOutPath != null) {
       if (!admin.tableExists(tableName)) {
         LOG.warn(format("Table '%s' does not exist.", tableName));



Mime
View raw message