Return-Path: X-Original-To: apmail-hadoop-common-user-archive@www.apache.org Delivered-To: apmail-hadoop-common-user-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id CE3477D13 for ; Thu, 28 Jul 2011 11:05:27 +0000 (UTC) Received: (qmail 74046 invoked by uid 500); 28 Jul 2011 11:05:21 -0000 Delivered-To: apmail-hadoop-common-user-archive@hadoop.apache.org Received: (qmail 73859 invoked by uid 500); 28 Jul 2011 11:05:12 -0000 Mailing-List: contact common-user-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-user@hadoop.apache.org Delivered-To: mailing list common-user@hadoop.apache.org Received: (qmail 73844 invoked by uid 99); 28 Jul 2011 11:05:09 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Jul 2011 11:05:09 +0000 X-ASF-Spam-Status: No, hits=1.5 required=5.0 tests=FREEMAIL_FROM,HTML_MESSAGE,RCVD_IN_DNSWL_LOW,SPF_PASS,T_TO_NO_BRKTS_FREEMAIL X-Spam-Check-By: apache.org Received-SPF: pass (athena.apache.org: domain of cnweike@gmail.com designates 209.85.213.176 as permitted sender) Received: from [209.85.213.176] (HELO mail-yx0-f176.google.com) (209.85.213.176) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Jul 2011 11:05:03 +0000 Received: by yxl11 with SMTP id 11so2006466yxl.35 for ; Thu, 28 Jul 2011 04:04:42 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=gamma; h=mime-version:in-reply-to:references:date:message-id:subject:from:to :content-type; bh=T7S4+JCyJT1yD9wZnQzrzNQWluu2/WNuqXNM2pw3gCk=; b=BvCwEXbgc+lJ13UnymngXdTzgQ2YkCJIynjmPPyVPQBEFBm5jrV/KtpWG67F8znSU2 ggeTAPRXmR6MPZIw2eQ9Zg7JFwNtfV3frCuUD3+uvcpOchgwDTlyA4Fz+1pJTlq35NT4 DXMCsWNJEpiuAAXknSP3Gpa0L7IDmDG3ZGVsU= MIME-Version: 1.0 Received: by 10.236.200.200 with SMTP id z48mr903691yhn.261.1311851082687; Thu, 28 Jul 2011 04:04:42 -0700 (PDT) Received: by 10.147.113.16 with HTTP; Thu, 28 Jul 2011 04:04:42 -0700 (PDT) In-Reply-To: References: Date: Thu, 28 Jul 2011 19:04:42 +0800 Message-ID: Subject: Fwd: HBase Mapreduce cannot find Map class From: air To: common-user@hadoop.apache.org Content-Type: multipart/alternative; boundary=20cf30549a13c825e804a91f21f8 --20cf30549a13c825e804a91f21f8 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: quoted-printable ---------- Forwarded message ---------- From: air Date: 2011/7/28 Subject: HBase Mapreduce cannot find Map class To: CDH Users import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.mapred.TableMapReduceUtil; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.mapred.Mapper; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.lib.NullOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; public class LoadToHBase extends Configured implements Tool{ public static class XMap extends MapReduceBase implements Mapper{ private JobConf conf; @Override public void configure(JobConf conf){ this.conf =3D conf; try{ this.table =3D new HTable(new HBaseConfiguration(conf), "observations"); }catch(IOException e){ throw new RuntimeException("Failed HTable construction", e)= ; } } @Override public void close() throws IOException{ super.close(); table.close(); } private HTable table; public void map(LongWritable key, Text value, OutputCollector output, Reporter reporter) throws IOException{ String[] valuelist =3D value.toString().split("\t"); SimpleDateFormat sdf =3D new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); Date addtime =3D null; // =E7=94=A8=E6=88=B7=E6=B3=A8=E5=86=8C= =E6=97=B6=E9=97=B4 Date ds =3D null; Long delta_days =3D null; String uid =3D valuelist[0]; try { addtime =3D sdf.parse(valuelist[1]); } catch (ParseException e) { // TODO Auto-generated catch block e.printStackTrace(); } String ds_str =3D conf.get("load.hbase.ds", null); if (ds_str !=3D null){ try { ds =3D sdf.parse(ds_str); } catch (ParseException e) { // TODO Auto-generated catch block e.printStackTrace(); } }else{ ds_str =3D "2011-07-28"; } if (addtime !=3D null && ds !=3D null){ delta_days =3D (ds.getTime() - addtime.getTime()) / (24 * 6= 0 * 60 * 1000); } if (delta_days !=3D null){ byte[] rowKey =3D uid.getBytes(); Put p =3D new Put(rowKey); p.add("content".getBytes(), "attr1".getBytes(), delta_days.toString().getBytes()); table.put(p); } } } /** * @param args * @throws Exception */ public static void main(String[] args) throws Exception { // TODO Auto-generated method stub int exitCode =3D ToolRunner.run(new HBaseConfiguration(), new LoadToHBase(), args); System.exit(exitCode); } @Override public int run(String[] args) throws Exception { // TODO Auto-generated method stub JobConf conf =3D new JobConf(getClass()); TableMapReduceUtil.addDependencyJars(conf); FileInputFormat.addInputPath(conf, new Path(args[0])); conf.setJobName("LoadToHBase"); conf.setJarByClass(getClass()); conf.setMapperClass(XMap.class); conf.setNumReduceTasks(0); conf.setOutputFormat(NullOutputFormat.class); JobClient.runJob(conf); return 0; } } execute it using hbase LoadToHBase /user/hive/warehouse/datamining.db/xxx/ and it says: ...... 11/07/28 17:20:29 INFO mapred.JobClient: Task Id : attempt_201107261532_2625_m_000004_1, Status : FAILED java.lang.RuntimeException: Error in configuring object at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93) at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64) at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117= ) at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:387) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:325) at org.apache.hadoop.mapred.Child$4.run(Child.java:270) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:396) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.j= ava:1127) at org.apache.hadoop.mapred.Child.main(Child.java:264) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:3= 9) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImp= l.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88) ... 9 more Caused by: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.ClassNotFoundException: LoadToHBase$XMap at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1028) at org.apache.hadoop.mapred.JobConf.getMapperClass(JobConf.java:968= ) at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34) ... 14 more Caused by: java.lang.RuntimeException: java.lang.ClassNotFoundException: LoadToHBase$XMap at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:996) at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1020) ... 16 more Caused by: java.lang.ClassNotFoundException: LoadToHBase$XMap at java.net.URLClassLoader$1.run(URLClassLoader.java:202) at java.security.AccessController.doPrivileged(Native Method) at java.net.URLClassLoader.findClass(URLClassLoader.java:190) at java.lang.ClassLoader.loadClass(ClassLoader.java:306) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301) at java.lang.ClassLoader.loadClass(ClassLoader.java:247) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:247) at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:943) at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:994) ... 17 more 11/07/28 17:20:29 INFO mapred.JobClient: Task Id : attempt_201107261532_2625_m_000006_1, Status : FAILED java.lang.RuntimeException: Error in configuring object at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93) at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64) at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117= ) at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:387) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:325) at org.apache.hadoop.mapred.Child$4.run(Child.java:270) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:396) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.j= ava:1127) at org.apache.hadoop.mapred.Child.main(Child.java:264) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:3= 9) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImp= l.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88) ... 9 more Caused by: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.ClassNotFoundException: LoadToHBase$XMap at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1028) at org.apache.hadoop.mapred.JobConf.getMapperClass(JobConf.java:968= ) at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34) ... 14 more Caused by: java.lang.RuntimeException: java.lang.ClassNotFoundException: LoadToHBase$XMap at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:996) at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1020) ... 16 more Caused by: java.lang.ClassNotFoundException: LoadToHBase$XMap at java.net.URLClassLoader$1.run(URLClassLoader.java:202) at java.security.AccessController.doPrivileged(Native Method) at java.net.URLClassLoader.findClass(URLClassLoader.java:190) at java.lang.ClassLoader.loadClass(ClassLoader.java:306) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301) at java.lang.ClassLoader.loadClass(ClassLoader.java:247) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:247) at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:943) at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:994) 11/07/28 17:20:29 INFO mapred.JobClient: Job complete: job_201107261532_262= 5 11/07/28 17:20:29 INFO mapred.JobClient: Counters: 8 11/07/28 17:20:29 INFO mapred.JobClient: Job Counters 11/07/28 17:20:29 INFO mapred.JobClient: SLOTS_MILLIS_MAPS=3D1504081 11/07/28 17:20:29 INFO mapred.JobClient: Total time spent by all reduce= s waiting after reserving slots (ms)=3D0 11/07/28 17:20:29 INFO mapred.JobClient: Total time spent by all maps waiting after reserving slots (ms)=3D0 11/07/28 17:20:29 INFO mapred.JobClient: Rack-local map tasks=3D187 11/07/28 17:20:29 INFO mapred.JobClient: Launched map tasks=3D350 11/07/28 17:20:29 INFO mapred.JobClient: Data-local map tasks=3D163 11/07/28 17:20:29 INFO mapred.JobClient: SLOTS_MILLIS_REDUCES=3D0 11/07/28 17:20:29 INFO mapred.JobClient: Failed map tasks=3D1 11/07/28 17:20:29 INFO mapred.JobClient: Job Failed: NA Exception in thread "main" java.io.IOException: Job failed! at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:1246) at LoadToHBase.run(LoadToHBase.java:106) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:65) at LoadToHBase.main(LoadToHBase.java:91) why it cannot find XMap, it's a inner class, it should be found ... I am really confused with this situation... thank you for you help --=20 Knowledge Mangement . --=20 Knowledge Mangement . --20cf30549a13c825e804a91f21f8--