0

我是 apache hadoop 的新手，我已经按照教程 http://v-lad.org/Tutorials/Hadoop使用 cygwin 在我的 windows xp 上配置了 hadoop 。配置成功，但在运行示例程序时显示 IOException。

示例代码：

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;

public class TestDriver {
    public static void main(String[] args) {
        JobClient client = new JobClient();
        JobConf conf = new JobConf(TestDriver.class);
        // TODO: specify output types
        conf.setOutputKeyClass(Text.class);
        conf.setOutputValueClass(IntWritable.class);

        conf.setInputFormat(TextInputFormat.class);
        conf.setOutputFormat(TextOutputFormat.class);

        FileInputFormat.setInputPaths(conf, new Path("In"));
        FileOutputFormat.setOutputPath(conf, new Path("Out"));

        // TODO: specify a mapper
        conf.setMapperClass(org.apache.hadoop.mapred.lib.IdentityMapper.class);

        // TODO: specify a reducer
        conf.setReducerClass(org.apache.hadoop.mapred.lib.IdentityReducer.class);

        client.setConf(conf);
        try {
            JobClient.runJob(conf);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

这是我遇到的例外，

13/11/06 14:54:28 WARN mapred.JobClient: Use GenericOptionsParser for parsing the 
arguments. Applications should implement Tool for the same.

13/11/06 14:54:28 INFO mapred.FileInputFormat: Total input paths to process : 4

13/11/06 14:54:28 INFO mapred.JobClient: Running job: job_201311061219_0005

13/11/06 14:54:29 INFO mapred.JobClient:  map 0% reduce 0%

13/11/06 14:54:33 INFO mapred.JobClient: Task Id : 
attempt_201311061219_0005_m_000006_0, Status : FAILED

java.io.IOException: Task process exit with nonzero status of 1.

at org.apache.hadoop.mapred.TaskRunner.run(TaskRunner.java:425)


13/11/06 14:54:36 INFO mapred.JobClient: Task Id : 
attempt_201311061219_0005_m_000006_1, Status : FAILED

java.io.IOException: Task process exit with nonzero status of 1.

at org.apache.hadoop.mapred.TaskRunner.run(TaskRunner.java:425)

日志文件如下，

Meta VERSION="1" .
Job JOBID="job_201311151137_0001" JOBNAME="HadoopTest_TestDriver\.java-5733197832393199007\.jar" USER="akhilkc" SUBMIT_TIME="1384495706421" JOBCONF="hdfs://localhost:9100/tmp/hadoop-SYSTEM/mapred/system/job_201311151137_0001/job\.xml" .
Job JOBID="job_201311151137_0001" JOB_PRIORITY="NORMAL" .
Job JOBID="job_201311151137_0001" LAUNCH_TIME="1384495707234" TOTAL_MAPS="5" TOTAL_REDUCES="1" JOB_STATUS="PREP" .
Task TASKID="task_201311151137_0001_m_000006" TASK_TYPE="SETUP" START_TIME="1384495709875" SPLITS="" .
MapAttempt TASK_TYPE="SETUP" TASKID="task_201311151137_0001_m_000006" TASK_ATTEMPT_ID="attempt_201311151137_0001_m_000006_0" START_TIME="1384495710156" TRACKER_NAME="tracker_notee:localhost/127\.0\.0\.1:1550" HTTP_PORT="50060" .
MapAttempt TASK_TYPE="SETUP" TASKID="task_201311151137_0001_m_000006" TASK_ATTEMPT_ID="attempt_201311151137_0001_m_000006_0" TASK_STATUS="FAILED" FINISH_TIME="1384495714593" HOSTNAME="tracker_notee" ERROR="java\.io\.IOException: Task process exit with nonzero status of 1\.
    at org\.apache\.hadoop\.mapred\.TaskRunner\.run(TaskRunner\.java:425)
" .
MapAttempt TASK_TYPE="SETUP" TASKID="task_201311151137_0001_m_000006" TASK_ATTEMPT_ID="attempt_201311151137_0001_m_000006_1" START_TIME="1384495714625" TRACKER_NAME="tracker_notee:localhost/127\.0\.0\.1:1550" HTTP_PORT="50060" .
MapAttempt TASK_TYPE="SETUP" TASKID="task_201311151137_0001_m_000006" TASK_ATTEMPT_ID="attempt_201311151137_0001_m_000006_1" TASK_STATUS="FAILED" FINISH_TIME="1384495718234" HOSTNAME="tracker_notee" ERROR="java\.io\.IOException: Task process exit with nonzero status of 1\.
    at org\.apache\.hadoop\.mapred\.TaskRunner\.run(TaskRunner\.java:425)
" .
MapAttempt TASK_TYPE="SETUP" TASKID="task_201311151137_0001_m_000006" TASK_ATTEMPT_ID="attempt_201311151137_0001_m_000006_2" START_TIME="1384495718281" TRACKER_NAME="tracker_notee:localhost/127\.0\.0\.1:1550" HTTP_PORT="50060" .
MapAttempt TASK_TYPE="SETUP" TASKID="task_201311151137_0001_m_000006" TASK_ATTEMPT_ID="attempt_201311151137_0001_m_000006_2" TASK_STATUS="FAILED" FINISH_TIME="1384495721875" HOSTNAME="tracker_notee" ERROR="java\.io\.IOException: Task process exit with nonzero status of 1\.
    at org\.apache\.hadoop\.mapred\.TaskRunner\.run(TaskRunner\.java:425)
" .
MapAttempt TASK_TYPE="SETUP" TASKID="task_201311151137_0001_m_000006" TASK_ATTEMPT_ID="attempt_201311151137_0001_m_000006_3" START_TIME="1384495721906" TRACKER_NAME="tracker_notee:localhost/127\.0\.0\.1:1550" HTTP_PORT="50060" .
MapAttempt TASK_TYPE="SETUP" TASKID="task_201311151137_0001_m_000006" TASK_ATTEMPT_ID="attempt_201311151137_0001_m_000006_3" TASK_STATUS="FAILED" FINISH_TIME="1384495725375" HOSTNAME="tracker_notee" ERROR="java\.io\.IOException: Task process exit with nonzero status of 1\.
    at org\.apache\.hadoop\.mapred\.TaskRunner\.run(TaskRunner\.java:425)
" .
Task TASKID="task_201311151137_0001_m_000006" TASK_TYPE="SETUP" TASK_STATUS="FAILED" FINISH_TIME="1384495725375" ERROR="java\.io\.IOException: Task process exit with nonzero status of 1\.
    at org\.apache\.hadoop\.mapred\.TaskRunner\.run(TaskRunner\.java:425)
" TASK_ATTEMPT_ID="" .
Task TASKID="task_201311151137_0001_m_000005" TASK_TYPE="CLEANUP" START_TIME="1384495725390" SPLITS="" .
MapAttempt TASK_TYPE="CLEANUP" TASKID="task_201311151137_0001_m_000005" TASK_ATTEMPT_ID="attempt_201311151137_0001_m_000005_0" START_TIME="1384495725406" TRACKER_NAME="tracker_notee:localhost/127\.0\.0\.1:1550" HTTP_PORT="50060" .
MapAttempt TASK_TYPE="CLEANUP" TASKID="task_201311151137_0001_m_000005" TASK_ATTEMPT_ID="attempt_201311151137_0001_m_000005_0" TASK_STATUS="FAILED" FINISH_TIME="1384495728906" HOSTNAME="tracker_notee" ERROR="java\.io\.IOException: Task process exit with nonzero status of 1\.
    at org\.apache\.hadoop\.mapred\.TaskRunner\.run(TaskRunner\.java:425)
" .
MapAttempt TASK_TYPE="CLEANUP" TASKID="task_201311151137_0001_m_000005" TASK_ATTEMPT_ID="attempt_201311151137_0001_m_000005_1" START_TIME="1384495728937" TRACKER_NAME="tracker_notee:localhost/127\.0\.0\.1:1550" HTTP_PORT="50060" .
MapAttempt TASK_TYPE="CLEANUP" TASKID="task_201311151137_0001_m_000005" TASK_ATTEMPT_ID="attempt_201311151137_0001_m_000005_1" TASK_STATUS="FAILED" FINISH_TIME="1384495732531" HOSTNAME="tracker_notee" ERROR="java\.io\.IOException: Task process exit with nonzero status of 1\.
    at org\.apache\.hadoop\.mapred\.TaskRunner\.run(TaskRunner\.java:425)
" .
MapAttempt TASK_TYPE="CLEANUP" TASKID="task_201311151137_0001_m_000005" TASK_ATTEMPT_ID="attempt_201311151137_0001_m_000005_2" START_TIME="1384495732562" TRACKER_NAME="tracker_notee:localhost/127\.0\.0\.1:1550" HTTP_PORT="50060" .
MapAttempt TASK_TYPE="CLEANUP" TASKID="task_201311151137_0001_m_000005" TASK_ATTEMPT_ID="attempt_201311151137_0001_m_000005_2" TASK_STATUS="FAILED" FINISH_TIME="1384495736171" HOSTNAME="tracker_notee" ERROR="java\.io\.IOException: Task process exit with nonzero status of 1\.
    at org\.apache\.hadoop\.mapred\.TaskRunner\.run(TaskRunner\.java:425)
" .
MapAttempt TASK_TYPE="CLEANUP" TASKID="task_201311151137_0001_m_000005" TASK_ATTEMPT_ID="attempt_201311151137_0001_m_000005_3" START_TIME="1384495736187" TRACKER_NAME="tracker_notee:localhost/127\.0\.0\.1:1550" HTTP_PORT="50060" .
MapAttempt TASK_TYPE="CLEANUP" TASKID="task_201311151137_0001_m_000005" TASK_ATTEMPT_ID="attempt_201311151137_0001_m_000005_3" TASK_STATUS="FAILED" FINISH_TIME="1384495739781" HOSTNAME="tracker_notee" ERROR="java\.io\.IOException: Task process exit with nonzero status of 1\.
    at org\.apache\.hadoop\.mapred\.TaskRunner\.run(TaskRunner\.java:425)
" .
Task TASKID="task_201311151137_0001_m_000005" TASK_TYPE="CLEANUP" TASK_STATUS="FAILED" FINISH_TIME="1384495739781" ERROR="java\.io\.IOException: Task process exit with nonzero status of 1\.
    at org\.apache\.hadoop\.mapred\.TaskRunner\.run(TaskRunner\.java:425)
" TASK_ATTEMPT_ID="" .
Job JOBID="job_201311151137_0001" FINISH_TIME="1384495739781" JOB_STATUS="FAILED" FINISHED_MAPS="0" FINISHED_REDUCES="0" .

我在 localhost:50030/jobtasks.jsp?jobid=job_201311151723_0002&type=setup&pagenum=1&state=killed 找到的任务日志，

java.io.IOException: Task process exit with nonzero status of 1.
 at org.apache.hadoop.mapred.TaskRunner.run(TaskRunner.java:425)

java.io.IOException: Task process exit with nonzero status of 1.
 at org.apache.hadoop.mapred.TaskRunner.run(TaskRunner.java:425)

java.io.IOException: Task process exit with nonzero status of 1.
 at org.apache.hadoop.mapred.TaskRunner.run(TaskRunner.java:425)

java.io.IOException: Task process exit with nonzero status of 1.
 at org.apache.hadoop.mapred.TaskRunner.run(TaskRunner.java:425)
4

1 回答 1

0

因为 JVM 堆栈大小太小，请更改您的 MR JVM 堆栈大小

# cat /var/log/hadoop-0.20-mapreduce/userlogs/job_201507091122_0015/attempt_201507091122_0015_m_000005_3/stdout

Error occurred during initialization of VM
Incompatible minimum and maximum heap sizes specified

因为JVM堆栈大小太小，改变你的MR JVM堆栈大小，fg：mapred.child.java.opts -> -Xms2048M -Xmx4096M

于 2015-07-10T16:58:01.427 回答