0

当我尝试使用 oozie 在 mapreduce 中运行字数统计 prg 时。它只是读取输入记录并显示它。我猜它甚至没有调用我的映射器和减速器类。因为我使用的是新的 API，所以在 workflow.xml 中也包含了 new-api 属性标签。

Map-reduce 片段：

public class WordCount {

  public static class Map extends Mapper<LongWritable, Text, Text, IntWritable> {
     private final static IntWritable one = new IntWritable(1);
     private Text word = new Text();

     public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
         String line = value.toString();
         StringTokenizer tokenizer = new StringTokenizer(line);
         while (tokenizer.hasMoreTokens()) {
             word.set(tokenizer.nextToken());
             context.write(word, one);
         }
 }

}

  public static class Reduce extends Reducer<Text,IntWritable,Text,IntWritable> {

     public void reduce(Text key, Iterable<IntWritable> values, Context context) 
       throws IOException, InterruptedException {
         int sum = 0;
         for (IntWritable val : values) {
             sum += val.get();
         }
             context.write(key, new IntWritable(sum));
     }

}

我的工作流.xml：

    <?xml version="1.0" encoding="UTF-8"?>
    <workflow-app xmlns='uri:oozie:workflow:0.1' name="wordcount">
    <start to="wc-node" />
    <action name="wc-node">
    <map-reduce>
        <job-tracker>${jobTracker}</job-tracker>
        <name-node>${nameNode}</name-node>
        <prepare>
            <delete path="${nameNode}/user/${wf:user()}/${wordcountRoot}/output-    data/${outputDir}"/>
        </prepare>

        <configuration>

            <property>
                <name>mapred.mapper.new-api</name>
                <value>true</value>
            </property>

            <property>
                <name>mapred.reducer.new-api</name>
                <value>true</value>
                 </property>

            <property>
                <name>mapreduce.map.class</name>
                <value>WordCount.Map</value>
            </property>

            <property>
                <name>mapreduce.reduce.class</name>
                <value>WordCount.Reduce</value>
            </property>

            <property>
              <name>mapred.output.key.class</name>
              <value>org.apache.hadoop.io.Text</value>
            </property>

            <property>
               <name>mapred.output.value.class</name>
               <value>org.apache.hadoop.io.IntWritable</value>
            </property>

            <property>
                <name>mapred.map.tasks</name>
                <value>1</value>
            </property>

            <property>
                <name>mapred.input.dir</name>
                <value>/user/${wf:user()}/${wordcountRoot}/input-data</value>
            </property>
            <property>
                <name>mapred.output.dir</name>
                <value>/user/${wf:user()}/${wordcountRoot}/output-data/${outputDir}</value>
            </property>

            <property>
                <name>mapred.job.queue.name</name>
                <value>${queueName}</value>
            </property>

            <property>
             <name>mapreduce.job.acl-view-job</name>
             <value>*</value>
            </property>

            <property>
               <name>oozie.launcher.mapreduce.job.acl-view-job</name>
               <value>*</value>
            </property>

         </configuration>

    </map-reduce>

    <ok to="end" />
    <error to="fail" />
</action>

<kill name="fail">
    <message>Map/Reduce failed</message>
</kill>
<end name="end" />

我提到了这个链接https://cwiki.apache.org/OOZIE/map-reduce-cookbook.html但仍然没有运气。如果any1遇到了这个问题，请指导我哪里出错了。

提前致谢。

4

1 回答 1

0

问题已解决......在使用新的 mapreduce API 时..我们需要在映射器和减速器类名前加上“$”符号：

<property>
 <name>mapreduce.map.class</name>
 <value>oozie.WordCount$Map</value>
</property> 
<property> 
 <name>mapreduce.reduce.class</name>
 <value>oozie.WordCount$Reduce</value>
</property>
于 2013-10-04T07:42:49.563 回答