hbase-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Jean-Daniel Cryans <jdcry...@apache.org>
Subject Re: can't load configuration successfully with the newest mapreduce package.
Date Thu, 25 Feb 2010 02:54:04 GMT
Steven,

I would recommend putting 1 region server with each datanode and use 3
or 5 quorum servers, 4 means that your quorum is already crippled of
one member.

I would also recommend reviewing the following doc:
http://hadoop.apache.org/hbase/docs/r0.20.3/api/org/apache/hadoop/hbase/mapreduce/package-summary.html

See how the classpath is handled and try to do the same. It's how I
personally do it.

J-D

On Wed, Feb 24, 2010 at 6:46 PM, steven zhuang <zhuangxin8448@gmail.com> wrote:
> hello,
>       I have a program loading formated data into HTable, but it crashed
> when connecting to the quorum server because of wrong configuration, seems
> that it load configuration from hbase-default.xml, not from hbase-site.xml.
>
>       my hadoop cluster has 35 nodes, and the administrator has set up 4
> quorum server and 5 region server(too few?). we have copied the Hbase
> configuration folder to every hadoop task tracker.
>
>       the job failed during the reduce phase, raised exceptions "
> NoServerForRegionException", I have checked the job log on some nodes, the
> connection string is set to "localhost:port" instead of the configuration in
> hbase-site.xml.
>
>       my code is as follows, I think the problem may be from the *doJob
> method(when I do the configuration):*
>       please help.
> *
> *
> *import java.io.IOException;*
> *import java.util.Iterator;*
> *
> *
> *import org.apache.hadoop.conf.Configuration;*
> *import org.apache.hadoop.conf.Configured;*
> *import org.apache.hadoop.hbase.HBaseConfiguration;*
> *import org.apache.hadoop.hbase.HColumnDescriptor;*
> *import org.apache.hadoop.hbase.HTableDescriptor;*
> *import org.apache.hadoop.hbase.TableExistsException;*
> *import org.apache.hadoop.hbase.client.HBaseAdmin;*
> *import org.apache.hadoop.hbase.client.Put;*
> *import org.apache.hadoop.hbase.io.HbaseMapWritable;*
> *import org.apache.hadoop.hbase.io.ImmutableBytesWritable;*
> *import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;*
> *import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;*
> *import org.apache.hadoop.hbase.mapreduce.TableReducer;*
> *import org.apache.hadoop.hbase.util.Bytes;*
> *import org.apache.hadoop.io.LongWritable;*
> *import org.apache.hadoop.io.Text;*
> *import org.apache.hadoop.mapred.JobClient;*
> *import org.apache.hadoop.mapred.JobConf;*
> *import org.apache.hadoop.mapreduce.Job;*
> *import org.apache.hadoop.mapreduce.Mapper;*
> *import org.apache.hadoop.mapreduce.Mapper.Context;*
> *import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;*
> *import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;*
> *import org.apache.hadoop.util.Tool;*
> *
> *
> *public class FasterMidImport extends Configured {*
> *
> *
> *......*
> * **private static Configuration conf = new HBaseConfiguration();*
> *
> *
> * **public static class InnerMap extends Mapper<LongWritable, Text, Text,
> Text> {*
> *
> *
> * **protected void map(LongWritable key, Text value, Context context)*
> * **throws IOException, InterruptedException {*
> * ......*
> * **context.write(new Text(rowKey), new Text(columnid + "\t" + value));*
> * **}*
> * **}*
> *
> *
> * **public static class MyTableReducer extends TableReducer<Text, Text,
> Text> {*
> * **protected void reduce(Text key, Iterable<Text> values, Context context)*
> * **throws IOException, InterruptedException {*
> * **Iterator<Text> iter = values.iterator();*
> * **Put put = new Put(key.toString().getBytes());*
> * **int cellCount = 0;*
> * **while (iter.hasNext()) {*
> * **String[] splits = iter.next().toString().split("\t");*
> * **if (splits.length < 2) {*
> * **continue;*
> * **}*
> * **String query = splits[0].substring(splits[0].indexOf(':') + 1);*
> * **put.add("queries".getBytes(), query.getBytes(), splits[1]*
> * **.getBytes());*
> * **if (++cellCount > 10000) {*
> * **cellCount = 0;*
> * **context.write(key, put);*
> * **put = new Put(key.toString().getBytes());*
> * **}*
> * **}*
> * **if (0 < cellCount) {*
> * **context.write(key, put);*
> * **}*
> * **}*
> * **}*
> *
> *
> * **public void doJob(String[] args) throws IOException {*
> * **conf.set("input.dir", args[0]);*
> * **conf.set("table.name", args[1]);*
> * **try {*
> * **String tblName = conf.get("table.name");*
> * **try {*
> * **HBaseAdmin admin = new HBaseAdmin((HBaseConfiguration) conf);*
> * **if (!admin.tableExists(tblName)) {*
> * **Thread.currentThread().sleep(2000);*
> * **HTableDescriptor tblDesc = new HTableDescriptor(tblName);*
> * **tblDesc.addFamily(new HColumnDescriptor("queries"));*
> * **admin.createTable(tblDesc);*
> * **}*
> * **} catch (TableExistsException e) {*
> * **} catch (InterruptedException e) {*
> * **e.printStackTrace();*
> * **}*
> * **} catch (IOException e) {*
> * **e.printStackTrace();*
> * **}*
> * **Job job = new Job(conf);*
> * **// Job job = new Job(new Configuration());*
> * **job.setJarByClass(FasterMidImport.class);*
> * **job.setJobName("Faster-midImport: import " + conf.get("input.dir")*
> * **+ "into" + conf.get("input.dir"));*
> *
> *
> * **job.setOutputKeyClass(Text.class);*
> * **job.setOutputValueClass(Text.class);*
> * **job.setOutputFormatClass(TableOutputFormat.class);*
> *
> *
> * **job.setInputFormatClass(TextInputFormat.class);*
> * **FileInputFormat.setInputPaths(job, conf.get("input.dir"));*
> *
> *
> * **job.setMapperClass(InnerMap.class);*
> *
> *
> * **job.setMapOutputKeyClass(Text.class);*
> * **job.setMapOutputValueClass(Text.class);*
> *
> *
> * **job.setNumReduceTasks(20);*
> * **//
> //////////////////////////////////////////////////////////////////////////////////////////////////////////////
> *
> * **TableMapReduceUtil.initTableReducerJob(conf.get("table.name"),*
> * **MyTableReducer.class, job);*
> * **try {*
> * **job.submit();*
> * **while (!job.isComplete()) {*
> * **Thread.currentThread().sleep(1000);*
> * **System.out*
> * **.println("Map: " + (job.mapProgress() * 100)*
> * **+ "% ... Reduce: "*
> * **+ (job.reduceProgress() * 100) + "%");*
> *
> *
> * **}*
> * **} catch (InterruptedException e) {*
> * **e.printStackTrace();*
> * **} catch (ClassNotFoundException e) {*
> * **e.printStackTrace();*
> * **}*
> * **}*
> * **static int printUsage() {*
> * **System.err.println("Usage: FasterMidImport <input-dir> <table_name>");*
> * **return -1;*
> * **}*
> * **/***
> * ** * * Run*
> * ** * */*
> * **public int run(String[] args) throws Exception {*
> * **if (args.length < 2) {*
> * **return printUsage();*
> * **}*
> * **this.doJob(args);*
> * **return 0;*
> * **}*
> *
> *
> * **public static int main(String[] args) throws Exception {*
> * **new FasterMidImport().run(args);*
> * **return 0;*
> * **}*
> *}*
>
>
> --
>       best wishes.
>                            steven
>

Mime
View raw message