0

我使用 hadoop 0.18.3 遇到以下错误

java.lang.ClassCastException：org.apache.hadoop.io.Text 无法转换为 org.apache.hadoop.io.DoubleWritable

我将我的映射器定义为：

public class HadoopMapper extends MapReduceBase implements Mapper<Text,DoubleWritable,Text,DoubleWritable> {
// The Karmasphere Studio Workflow Log displays logging from Apache Commons Logging, for example:
// private static final Log LOG = LogFactory.getLog("HadoopMapper");

@Override
public void map(Text key, DoubleWritable value, OutputCollector<Text, DoubleWritable> output, Reporter reporter)
        throws IOException {
//        throw new UnsupportedOperationException("Not supported yet.");
    Random generator = new Random();
     int i;

     final int iter = 100000;

     for (i =0; i < iter; i++)
     {
     double x = generator.nextDouble();
     double y = generator.nextDouble();

     double z;

     z = x*x + y*y;

     if (z <= 1){
         output.collect(new Text("VALUE"), new DoubleWritable(1));
     }else{
         output.collect(new Text ("VALUE"), new DoubleWritable(0));
     }
     }


  }
}

和减速机类

public class HadoopReducer extends MapReduceBase implements    Reducer<Text,DoubleWritable,Text,DoubleWritable> {
// The Karmasphere Studio Workflow Log displays logging from Apache Commons Logging, for example:
// private static final Log LOG = LogFactory.getLog("HadoopReducer");

@Override
public void reduce(Text key, Iterator<DoubleWritable> value, OutputCollector<Text, DoubleWritable> output, Reporter reporter)
        throws IOException {
    // TODO code reducer logic here
//        throw new UnsupportedOperationException("Not supported yet.");

    double pi = 0;
     double inside = 0;
     double outside = 0;

     while (value.hasNext())
     {
     if (value.next().get() == (long)1)
     inside++;
     else
     outside++; 
     }

     pi = (4*inside)/(inside + outside);

     output.collect(new Text ("pi"), new DoubleWritable(pi));
    }
}

我将jobconf设置为：

    public static void initJobConf(JobConf conf) {
// Generating code using Karmasphere Protocol for Hadoop 0.18
// CG_GLOBAL

// CG_INPUT_HIDDEN
    conf.setInputFormat(KeyValueTextInputFormat.class);
// CG_MAPPER_HIDDEN
conf.setMapperClass(HadoopMapper.class);

// CG_MAPPER

// CG_PARTITIONER_HIDDEN
conf.setPartitionerClass(org.apache.hadoop.mapred.lib.HashPartitioner.class);

// CG_PARTITIONER

 // CG_COMPARATOR_HIDDEN
conf.setOutputKeyComparatorClass(org.apache.hadoop.io.Text.Comparator.class);

// CG_COMPARATOR

// CG_COMBINER_HIDDEN

// CG_REDUCER_HIDDEN
conf.setReducerClass(HadoopReducer.class);

// CG_REDUCER
   conf.setNumReduceTasks(1);

   // CG_OUTPUT_HIDDEN
    conf.setOutputKeyClass(Text.class);
     conf.setOutputValueClass(DoubleWritable.class);
    // CG_OUTPUT

     // Others
    }

我在 conf.setInputFormat(KeyValueTextInputFormat.class) 中找不到与 KeyValueTextInputFormat.class 匹配的 Inputformat，那么如何处理呢？我可以子类化吗？你能帮我举个例子吗？谢谢

4

1 回答 1

0

KeyValueTextInputFormat 期望输入一个文本键和一个由 SEPARATOR_CHARACTER 分隔的文本值（默认选项卡）。您正在尝试将其转换为默认情况下不可能的 DoubleWritable。

因此，将您的映射器修改为： Mapper< Text,Text,Text,DoubleWritable >

以及相应的 map 方法，然后自己将文本转换为 double 。

于 2012-07-07T23:51:14.227 回答