FAQ

---------- Forwarded message ----------
From: air <cnweike@gmail.com>
Date: 2011/7/28
Subject: HBase Mapreduce cannot find Map class
To: CDH Users <cdh-user@cloudera.org>
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;

import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.mapred.TableMapReduceUtil;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.lib.NullOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;


public class LoadToHBase extends Configured implements Tool{
public static class XMap<K, V> extends MapReduceBase implements
Mapper<LongWritable, Text, K, V>{
private JobConf conf;

@Override
public void configure(JobConf conf){
this.conf = conf;
try{
this.table = new HTable(new HBaseConfiguration(conf),
"observations");
}catch(IOException e){
throw new RuntimeException("Failed HTable construction", e);
}
}

@Override
public void close() throws IOException{
super.close();
table.close();
}

private HTable table;
public void map(LongWritable key, Text value, OutputCollector
output, Reporter reporter) throws IOException{
String[] valuelist = value.toString().split("\t");
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd
HH:mm:ss");
Date addtime = null; // 用户注册时间
Date ds = null;
Long delta_days = null;
String uid = valuelist[0];
try {
addtime = sdf.parse(valuelist[1]);
} catch (ParseException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}

String ds_str = conf.get("load.hbase.ds", null);
if (ds_str != null){
try {
ds = sdf.parse(ds_str);
} catch (ParseException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}else{
ds_str = "2011-07-28";
}

if (addtime != null && ds != null){
delta_days = (ds.getTime() - addtime.getTime()) / (24 * 60 *
60 * 1000);
}

if (delta_days != null){
byte[] rowKey = uid.getBytes();
Put p = new Put(rowKey);
p.add("content".getBytes(), "attr1".getBytes(),
delta_days.toString().getBytes());
table.put(p);
}
}
}
/**
* @param args
* @throws Exception
*/
public static void main(String[] args) throws Exception {
// TODO Auto-generated method stub
int exitCode = ToolRunner.run(new HBaseConfiguration(), new
LoadToHBase(), args);
System.exit(exitCode);
}

@Override
public int run(String[] args) throws Exception {
// TODO Auto-generated method stub
JobConf conf = new JobConf(getClass());
TableMapReduceUtil.addDependencyJars(conf);
FileInputFormat.addInputPath(conf, new Path(args[0]));
conf.setJobName("LoadToHBase");
conf.setJarByClass(getClass());
conf.setMapperClass(XMap.class);
conf.setNumReduceTasks(0);
conf.setOutputFormat(NullOutputFormat.class);
JobClient.runJob(conf);
return 0;
}

}

execute it using hbase LoadToHBase /user/hive/warehouse/datamining.db/xxx/
and it says:

......
11/07/28 17:20:29 INFO mapred.JobClient: Task Id :
attempt_201107261532_2625_m_000004_1, Status : FAILED
java.lang.RuntimeException: Error in configuring object
at
org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93)
at
org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64)
at
org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:387)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:325)
at org.apache.hadoop.mapred.Child$4.run(Child.java:270)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1127)
at org.apache.hadoop.mapred.Child.main(Child.java:264)
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
... 9 more
Caused by: java.lang.RuntimeException: java.lang.RuntimeException:
java.lang.ClassNotFoundException: LoadToHBase$XMap
at
org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1028)
at org.apache.hadoop.mapred.JobConf.getMapperClass(JobConf.java:968)
at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34)
... 14 more
Caused by: java.lang.RuntimeException: java.lang.ClassNotFoundException:
LoadToHBase$XMap
at
org.apache.hadoop.conf.Configuration.getClass(Configuration.java:996)
at
org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1020)
... 16 more
Caused by: java.lang.ClassNotFoundException: LoadToHBase$XMap
at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:247)
at
org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:943)
at
org.apache.hadoop.conf.Configuration.getClass(Configuration.java:994)
... 17 more

11/07/28 17:20:29 INFO mapred.JobClient: Task Id :
attempt_201107261532_2625_m_000006_1, Status : FAILED
java.lang.RuntimeException: Error in configuring object
at
org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93)
at
org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64)
at
org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:387)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:325)
at org.apache.hadoop.mapred.Child$4.run(Child.java:270)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1127)
at org.apache.hadoop.mapred.Child.main(Child.java:264)
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
... 9 more
Caused by: java.lang.RuntimeException: java.lang.RuntimeException:
java.lang.ClassNotFoundException: LoadToHBase$XMap
at
org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1028)
at org.apache.hadoop.mapred.JobConf.getMapperClass(JobConf.java:968)
at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34)
... 14 more
Caused by: java.lang.RuntimeException: java.lang.ClassNotFoundException:
LoadToHBase$XMap
at
org.apache.hadoop.conf.Configuration.getClass(Configuration.java:996)
at
org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1020)
... 16 more
Caused by: java.lang.ClassNotFoundException: LoadToHBase$XMap
at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:247)
at
org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:943)
at
org.apache.hadoop.conf.Configuration.getClass(Configuration.java:994)

11/07/28 17:20:29 INFO mapred.JobClient: Job complete: job_201107261532_2625
11/07/28 17:20:29 INFO mapred.JobClient: Counters: 8
11/07/28 17:20:29 INFO mapred.JobClient: Job Counters
11/07/28 17:20:29 INFO mapred.JobClient: SLOTS_MILLIS_MAPS=1504081
11/07/28 17:20:29 INFO mapred.JobClient: Total time spent by all reduces
waiting after reserving slots (ms)=0
11/07/28 17:20:29 INFO mapred.JobClient: Total time spent by all maps
waiting after reserving slots (ms)=0
11/07/28 17:20:29 INFO mapred.JobClient: Rack-local map tasks=187
11/07/28 17:20:29 INFO mapred.JobClient: Launched map tasks=350
11/07/28 17:20:29 INFO mapred.JobClient: Data-local map tasks=163
11/07/28 17:20:29 INFO mapred.JobClient: SLOTS_MILLIS_REDUCES=0
11/07/28 17:20:29 INFO mapred.JobClient: Failed map tasks=1
11/07/28 17:20:29 INFO mapred.JobClient: Job Failed: NA
Exception in thread "main" java.io.IOException: Job failed!
at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:1246)
at LoadToHBase.run(LoadToHBase.java:106)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:65)
at LoadToHBase.main(LoadToHBase.java:91)


why it cannot find XMap, it's a inner class, it should be found ... I am
really confused with this situation... thank you for you help
--
Knowledge Mangement .




--
Knowledge Mangement .

Search Discussions

  • Stack at Jul 28, 2011 at 3:48 pm
    See http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/mapreduce/package-summary.html#package_description
    for some help.
    St.Ack
    On Thu, Jul 28, 2011 at 4:04 AM, air wrote:
    ---------- Forwarded message ----------
    From: air <cnweike@gmail.com>
    Date: 2011/7/28
    Subject: HBase Mapreduce cannot find Map class
    To: CDH Users <cdh-user@cloudera.org>


    import java.io.IOException;
    import java.text.ParseException;
    import java.text.SimpleDateFormat;
    import java.util.Date;

    import org.apache.hadoop.conf.Configured;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.hbase.HBaseConfiguration;
    import org.apache.hadoop.hbase.client.HTable;
    import org.apache.hadoop.hbase.client.Put;
    import org.apache.hadoop.hbase.mapred.TableMapReduceUtil;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapred.JobClient;
    import org.apache.hadoop.mapred.JobConf;
    import org.apache.hadoop.mapred.MapReduceBase;
    import org.apache.hadoop.mapred.Mapper;
    import org.apache.hadoop.mapred.OutputCollector;
    import org.apache.hadoop.mapred.Reporter;
    import org.apache.hadoop.mapred.FileInputFormat;
    import org.apache.hadoop.mapred.lib.NullOutputFormat;
    import org.apache.hadoop.util.Tool;
    import org.apache.hadoop.util.ToolRunner;


    public class LoadToHBase extends Configured implements Tool{
    public static class XMap<K, V> extends MapReduceBase implements
    Mapper<LongWritable, Text, K, V>{
    private JobConf conf;

    @Override
    public void configure(JobConf conf){
    this.conf = conf;
    try{
    this.table = new HTable(new HBaseConfiguration(conf),
    "observations");
    }catch(IOException e){
    throw new RuntimeException("Failed HTable construction", e);
    }
    }

    @Override
    public void close() throws IOException{
    super.close();
    table.close();
    }

    private HTable table;
    public void map(LongWritable key, Text value, OutputCollector
    output, Reporter reporter) throws IOException{
    String[] valuelist = value.toString().split("\t");
    SimpleDateFormat sdf = new  SimpleDateFormat("yyyy-MM-dd
    HH:mm:ss");
    Date addtime = null; // 用户注册时间
    Date ds = null;
    Long delta_days = null;
    String uid = valuelist[0];
    try {
    addtime = sdf.parse(valuelist[1]);
    } catch (ParseException e) {
    // TODO Auto-generated catch block
    e.printStackTrace();
    }

    String ds_str = conf.get("load.hbase.ds", null);
    if (ds_str != null){
    try {
    ds = sdf.parse(ds_str);
    } catch (ParseException e) {
    // TODO Auto-generated catch block
    e.printStackTrace();
    }
    }else{
    ds_str = "2011-07-28";
    }

    if (addtime != null && ds != null){
    delta_days = (ds.getTime() - addtime.getTime()) / (24 * 60 *
    60 * 1000);
    }

    if (delta_days != null){
    byte[] rowKey = uid.getBytes();
    Put p = new Put(rowKey);
    p.add("content".getBytes(), "attr1".getBytes(),
    delta_days.toString().getBytes());
    table.put(p);
    }
    }
    }
    /**
    * @param args
    * @throws Exception
    */
    public static void main(String[] args) throws Exception {
    // TODO Auto-generated method stub
    int exitCode = ToolRunner.run(new HBaseConfiguration(), new
    LoadToHBase(), args);
    System.exit(exitCode);
    }

    @Override
    public int run(String[] args) throws Exception {
    // TODO Auto-generated method stub
    JobConf conf = new JobConf(getClass());
    TableMapReduceUtil.addDependencyJars(conf);
    FileInputFormat.addInputPath(conf, new Path(args[0]));
    conf.setJobName("LoadToHBase");
    conf.setJarByClass(getClass());
    conf.setMapperClass(XMap.class);
    conf.setNumReduceTasks(0);
    conf.setOutputFormat(NullOutputFormat.class);
    JobClient.runJob(conf);
    return 0;
    }

    }

    execute it using hbase LoadToHBase /user/hive/warehouse/datamining.db/xxx/
    and it says:

    ......
    11/07/28 17:20:29 INFO mapred.JobClient: Task Id :
    attempt_201107261532_2625_m_000004_1, Status : FAILED
    java.lang.RuntimeException: Error in configuring object
    at
    org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93)
    at
    org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64)
    at
    org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117)
    at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:387)
    at org.apache.hadoop.mapred.MapTask.run(MapTask.java:325)
    at org.apache.hadoop.mapred.Child$4.run(Child.java:270)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:396)
    at
    org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1127)
    at org.apache.hadoop.mapred.Child.main(Child.java:264)
    Caused by: java.lang.reflect.InvocationTargetException
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at
    sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
    at
    sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at
    org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
    ... 9 more
    Caused by: java.lang.RuntimeException: java.lang.RuntimeException:
    java.lang.ClassNotFoundException: LoadToHBase$XMap
    at
    org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1028)
    at org.apache.hadoop.mapred.JobConf.getMapperClass(JobConf.java:968)
    at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34)
    ... 14 more
    Caused by: java.lang.RuntimeException: java.lang.ClassNotFoundException:
    LoadToHBase$XMap
    at
    org.apache.hadoop.conf.Configuration.getClass(Configuration.java:996)
    at
    org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1020)
    ... 16 more
    Caused by: java.lang.ClassNotFoundException: LoadToHBase$XMap
    at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
    at java.security.AccessController.doPrivileged(Native Method)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
    at java.lang.Class.forName0(Native Method)
    at java.lang.Class.forName(Class.java:247)
    at
    org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:943)
    at
    org.apache.hadoop.conf.Configuration.getClass(Configuration.java:994)
    ... 17 more

    11/07/28 17:20:29 INFO mapred.JobClient: Task Id :
    attempt_201107261532_2625_m_000006_1, Status : FAILED
    java.lang.RuntimeException: Error in configuring object
    at
    org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93)
    at
    org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64)
    at
    org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117)
    at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:387)
    at org.apache.hadoop.mapred.MapTask.run(MapTask.java:325)
    at org.apache.hadoop.mapred.Child$4.run(Child.java:270)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:396)
    at
    org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1127)
    at org.apache.hadoop.mapred.Child.main(Child.java:264)
    Caused by: java.lang.reflect.InvocationTargetException
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at
    sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
    at
    sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at
    org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
    ... 9 more
    Caused by: java.lang.RuntimeException: java.lang.RuntimeException:
    java.lang.ClassNotFoundException: LoadToHBase$XMap
    at
    org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1028)
    at org.apache.hadoop.mapred.JobConf.getMapperClass(JobConf.java:968)
    at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34)
    ... 14 more
    Caused by: java.lang.RuntimeException: java.lang.ClassNotFoundException:
    LoadToHBase$XMap
    at
    org.apache.hadoop.conf.Configuration.getClass(Configuration.java:996)
    at
    org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1020)
    ... 16 more
    Caused by: java.lang.ClassNotFoundException: LoadToHBase$XMap
    at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
    at java.security.AccessController.doPrivileged(Native Method)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
    at java.lang.Class.forName0(Native Method)
    at java.lang.Class.forName(Class.java:247)
    at
    org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:943)
    at
    org.apache.hadoop.conf.Configuration.getClass(Configuration.java:994)

    11/07/28 17:20:29 INFO mapred.JobClient: Job complete: job_201107261532_2625
    11/07/28 17:20:29 INFO mapred.JobClient: Counters: 8
    11/07/28 17:20:29 INFO mapred.JobClient:   Job Counters
    11/07/28 17:20:29 INFO mapred.JobClient:     SLOTS_MILLIS_MAPS=1504081
    11/07/28 17:20:29 INFO mapred.JobClient:     Total time spent by all reduces
    waiting after reserving slots (ms)=0
    11/07/28 17:20:29 INFO mapred.JobClient:     Total time spent by all maps
    waiting after reserving slots (ms)=0
    11/07/28 17:20:29 INFO mapred.JobClient:     Rack-local map tasks=187
    11/07/28 17:20:29 INFO mapred.JobClient:     Launched map tasks=350
    11/07/28 17:20:29 INFO mapred.JobClient:     Data-local map tasks=163
    11/07/28 17:20:29 INFO mapred.JobClient:     SLOTS_MILLIS_REDUCES=0
    11/07/28 17:20:29 INFO mapred.JobClient:     Failed map tasks=1
    11/07/28 17:20:29 INFO mapred.JobClient: Job Failed: NA
    Exception in thread "main" java.io.IOException: Job failed!
    at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:1246)
    at LoadToHBase.run(LoadToHBase.java:106)
    at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:65)
    at LoadToHBase.main(LoadToHBase.java:91)


    why it cannot find XMap, it's a inner class, it should be found ...  I am
    really confused with this situation... thank you for you help
    --
    Knowledge Mangement .




    --
    Knowledge Mangement .

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommon-user @
categorieshadoop
postedJul 28, '11 at 11:05a
activeJul 28, '11 at 3:48p
posts2
users2
websitehadoop.apache.org...
irc#hadoop

2 users in discussion

Stack: 1 post Air: 1 post

People

Translate

site design / logo © 2022 Grokbase