我在 IntelliJ 中创建了一个包含三个类文件的 maven 项目：
package mavenKris;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
import java.util.StringTokenizer;
/**
* Created with IntelliJ IDEA.
* User: kshk
* Date: 11/27/13
* Time: 2:27 PM
* To change this template use File | Settings | File Templates.
*/
public class WordMapper extends Mapper<Text,Text,Text,Text> {
private Text word = new Text();
public void map(Text key, Text value, Context context) throws IOException, InterruptedException
{
StringTokenizer itr = new StringTokenizer(value.toString(),",");
while (itr.hasMoreTokens())
{
word.set(itr.nextToken());
context.write(key, word);
}
}
}
和
package mavenKris;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
/**
* Created with IntelliJ IDEA.
* User: kshk
* Date: 11/27/13
* Time: 2:29 PM
* To change this template use File | Settings | File Templates.
*/
public class AllTranslationsReducer extends Reducer<Text, Text, Text, Text> {
private Text result = new Text();
@Override
protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
String translations = "";
for (Text val : values) {
translations += "|" + val.toString();
}
result.set(translations);
context.write(key, result);
}
}
和主要...
package mavenKris;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import java.io.IOException;
/**
* Created with IntelliJ IDEA.
* User: kshk
* Date: 11/27/13
* Time: 2:30 PM
* To change this template use File | Settings | File Templates.
*/
public class Dictionary {
public static void main(String[] args) throws Exception
{
Configuration conf = new Configuration();
Job job = new Job(conf, "dictionary");
job.setJarByClass(Dictionary.class);
job.setMapperClass(WordMapper.class);
job.setReducerClass(AllTranslationsReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setInputFormatClass(KeyValueTextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
boolean result = job.waitForCompletion(true);
System.exit(result ? 0 : 1);
}
}
这就是 pom.xml 的样子
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>mavenKris</groupId>
<artifactId>mavenKris</artifactId>
<version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>mavenKris</name>
<url>http://maven.apache.org</url>
<repositories>
<repository>
<id>anorak-releases</id>
<url>http://ip-10-40-3-96.cloud.trendinglines.co.uk:8081/nexus/content/repositories/releases/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>atlassian-releases</id>
<url>http://repository.opencastproject.org/nexus/content/repositories/atlassian/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>central</id>
<name>Maven Central</name>
<url>http://repo1.maven.org/maven2/</url>
</repository>
</repositories>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>1.0.3-gphd-1.2.0.0</version>
<scope>provided</scope>
</dependency>
</dependencies>
</project>
我正在使用 hadoop 的 greenplum 发行版，现在..当我运行上述项目时，我收到此错误....
/usr/java/jdk1.7.0_40/bin/java -Didea.launcher.port=7551 -Didea.launcher.bin.path=/home/kshk/Software/idea-IC-129.1359/bin -Dfile.encoding=UTF-8 -classpath /usr/java/jdk1.7.0_40/jre/lib/jce.jar:/usr/java/jdk1.7.0_40/jre/lib/management-agent.jar:/usr/java/jdk1.7.0_40/jre/lib/resources.jar:/usr/java/jdk1.7.0_40/jre/lib/jfr.jar:/usr/java/jdk1.7.0_40/jre/lib/jsse.jar:/usr/java/jdk1.7.0_40/jre/lib/rt.jar:/usr/java/jdk1.7.0_40/jre/lib/plugin.jar:/usr/java/jdk1.7.0_40/jre/lib/deploy.jar:/usr/java/jdk1.7.0_40/jre/lib/charsets.jar:/usr/java/jdk1.7.0_40/jre/lib/javaws.jar:/usr/java/jdk1.7.0_40/jre/lib/jfxrt.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/zipfs.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/localedata.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/sunpkcs11.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/sunec.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/dnsns.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/sunjce_provider.jar:/home/kshk/IdeaProjects/mavenKris/target/classes:/home/kshk/Software/idea-IC-129.1359/lib/idea_rt.jar com.intellij.rt.execution.application.AppMain mavenKris.Dictionary
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/conf/Configuration
at mavenKris.Dictionary.main(Dictionary.java:24)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:120)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.conf.Configuration
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 6 more
Process finished with exit code 1
有任何想法吗？
更新：
如果我从命令行指定类路径的路径，它调用 Dictionary 类但给出错误，所以我的 IDE 上必须有一些设置，它没有在运行时指定 hadoop-core 的类路径.....
[kshk@localhost mavenKris]$ /usr/java/jdk1.7.0_40/bin/java -Didea.launcher.port=7537 -Didea.launcher.bin.path=/home/kshk/Software/idea-IC-129.1359/bin -Dfile.encoding=UTF-8 -classpath /usr/java/jdk1.7.0_40/jre/lib/jce.jar:/usr/java/jdk1.7.0_40/jre/lib/management-agent.jar:/usr/java/jdk1.7.0_40/jre/lib/resources.jar:/usr/java/jdk1.7.0_40/jre/lib/jfr.jar:/usr/java/jdk1.7.0_40/jre/lib/jsse.jar:/usr/java/jdk1.7.0_40/jre/lib/rt.jar:/usr/java/jdk1.7.0_40/jre/lib/plugin.jar:/usr/java/jdk1.7.0_40/jre/lib/deploy.jar:/usr/java/jdk1.7.0_40/jre/lib/charsets.jar:/usr/java/jdk1.7.0_40/jre/lib/javaws.jar:/usr/java/jdk1.7.0_40/jre/lib/jfxrt.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/zipfs.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/localedata.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/sunpkcs11.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/sunec.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/dnsns.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/sunjce_provider.jar:/home/kshk/IdeaProjects/mavenKris/target/classes:/home/kshk/Software/idea-IC-129.1359/lib/idea_rt.jar:/home/kshk/.m2/repository/org/apache/hadoop/hadoop-core/1.0.3-gphd-1.2.0.0/hadoop-core-1.0.3-gphd-1.2.0.0.jar com.intellij.rt.execution.application.AppMain mavenKris.Dictionary
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/commons/logging/LogFactory
at org.apache.hadoop.conf.Configuration.<clinit>(Configuration.java:143)
at mavenKris.Dictionary.main(Dictionary.java:25)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:120)
Caused by: java.lang.ClassNotFoundException: org.apache.commons.logging.LogFactory
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 7 more
[kshk@localhost mavenKris]$
更新2：
添加commons-logging-1.0.3.jar的路径后......
[kshk@localhost repository]$ /usr/java/jdk1.7.0_40/bin/java -Didea.launcher.port=7537 -Didea.launcher.bin.path=/home/kshk/Software/idea-IC-129.1359/bin -Dfile.encoding=UTF-8 -classpath /usr/java/jdk1.7.0_40/jre/lib/jce.jar:/usr/java/jdk1.7.0_40/jre/lib/management-agent.jar:/usr/java/jdk1.7.0_40/jre/lib/resources.jar:/usr/java/jdk1.7.0_40/jre/lib/jfr.jar:/usr/java/jdk1.7.0_40/jre/lib/jsse.jar:/usr/java/jdk1.7.0_40/jre/lib/rt.jar:/usr/java/jdk1.7.0_40/jre/lib/plugin.jar:/usr/java/jdk1.7.0_40/jre/lib/deploy.jar:/usr/java/jdk1.7.0_40/jre/lib/charsets.jar:/usr/java/jdk1.7.0_40/jre/lib/javaws.jar:/usr/java/jdk1.7.0_40/jre/lib/jfxrt.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/zipfs.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/localedata.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/sunpkcs11.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/sunec.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/dnsns.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/sunjce_provider.jar:/home/kshk/IdeaProjects/mavenKris/target/classes:/home/kshk/Software/idea-IC-129.1359/lib/idea_rt.jar:/home/kshk/.m2/repository/org/apache/hadoop/hadoop-core/1.0.3-gphd-1.2.0.0/hadoop-core-1.0.3-gphd-1.2.0.0.jar:/home/kshk/.m2/repository/commons-logging/commons-logging/1.0.3/commons-logging-1.0.3.jar com.intellij.rt.execution.application.AppMain mavenKris.Dictionary
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/commons/configuration/Configuration
at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.<init>(DefaultMetricsSystem.java:37)
at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.<clinit>(DefaultMetricsSystem.java:34)
at org.apache.hadoop.security.UgiInstrumentation.create(UgiInstrumentation.java:51)
at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:216)
at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:184)
at org.apache.hadoop.security.UserGroupInformation.isSecurityEnabled(UserGroupInformation.java:236)
at org.apache.hadoop.security.KerberosName.<clinit>(KerberosName.java:79)
at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:209)
at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:184)
at org.apache.hadoop.security.UserGroupInformation.isSecurityEnabled(UserGroupInformation.java:236)
at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:466)
at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:452)
at org.apache.hadoop.mapreduce.JobContext.<init>(JobContext.java:80)
at org.apache.hadoop.mapreduce.Job.<init>(Job.java:50)
at org.apache.hadoop.mapreduce.Job.<init>(Job.java:54)
at mavenKris.Dictionary.main(Dictionary.java:26)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:120)
Caused by: java.lang.ClassNotFoundException: org.apache.commons.configuration.Configuration
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 21 more
更新 3：
添加 :/home/kshk/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar 到路径....
[kshk@localhost repository]$ /usr/java/jdk1.7.0_40/bin/java -Didea.launcher.port=7537 -Didea.launcher.bin.path=/home/kshk/Software/idea-IC-129.1359/bin -Dfile.encoding=UTF-8 -classpath /usr/java/jdk1.7.0_40/jre/lib/jce.jar:/usr/java/jdk1.7.0_40/jre/lib/management-agent.jar:/usr/java/jdk1.7.0_40/jre/lib/resources.jar:/usr/java/jdk1.7.0_40/jre/lib/jfr.jar:/usr/java/jdk1.7.0_40/jre/lib/jsse.jar:/usr/java/jdk1.7.0_40/jre/lib/rt.jar:/usr/java/jdk1.7.0_40/jre/lib/plugin.jar:/usr/java/jdk1.7.0_40/jre/lib/deploy.jar:/usr/java/jdk1.7.0_40/jre/lib/charsets.jar:/usr/java/jdk1.7.0_40/jre/lib/javaws.jar:/usr/java/jdk1.7.0_40/jre/lib/jfxrt.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/zipfs.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/localedata.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/sunpkcs11.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/sunec.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/dnsns.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/sunjce_provider.jar:/home/kshk/IdeaProjects/mavenKris/target/classes:/home/kshk/Software/idea-IC-129.1359/lib/idea_rt.jar:/home/kshk/.m2/repository/org/apache/hadoop/hadoop-core/1.0.3-gphd-1.2.0.0/hadoop-core-1.0.3-gphd-1.2.0.0.jar:/home/kshk/.m2/repository/commons-logging/commons-logging/1.0.3/commons-logging-1.0.3.jar:/home/kshk/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar com.intellij.rt.execution.application.AppMain mavenKris.Dictionary
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/commons/lang/StringUtils
at org.apache.hadoop.metrics2.lib.MetricMutableStat.<init>(MetricMutableStat.java:59)
at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.<init>(MetricsSystemImpl.java:75)
at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.<init>(MetricsSystemImpl.java:120)
at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.<init>(DefaultMetricsSystem.java:37)
at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.<clinit>(DefaultMetricsSystem.java:34)
at org.apache.hadoop.security.UgiInstrumentation.create(UgiInstrumentation.java:51)
at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:216)
at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:184)
at org.apache.hadoop.security.UserGroupInformation.isSecurityEnabled(UserGroupInformation.java:236)
at org.apache.hadoop.security.KerberosName.<clinit>(KerberosName.java:79)
at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:209)
at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:184)
at org.apache.hadoop.security.UserGroupInformation.isSecurityEnabled(UserGroupInformation.java:236)
at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:466)
at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:452)
at org.apache.hadoop.mapreduce.JobContext.<init>(JobContext.java:80)
at org.apache.hadoop.mapreduce.Job.<init>(Job.java:50)
at org.apache.hadoop.mapreduce.Job.<init>(Job.java:54)
at mavenKris.Dictionary.main(Dictionary.java:26)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:120)
Caused by: java.lang.ClassNotFoundException: org.apache.commons.lang.StringUtils
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 24 more
更新 4：
添加：/home/kshk/.m2/repository/commons-lang/commons-lang/2.1/commons-lang-2.1.jar 到类路径
[kshk@localhost tmp]$ /usr/java/jdk1.7.0_40/bin/java -Didea.launcher.port=7537 -Didea.launcher.bin.path=/home/kshk/Software/idea-IC-129.1359/bin -Dfile.encoding=UTF-8 -classpath /usr/java/jdk1.7.0_40/jre/lib/jce.jar:/usr/java/jdk1.7.0_40/jre/lib/management-agent.jar:/usr/java/jdk1.7.0_40/jre/lib/resources.jar:/usr/java/jdk1.7.0_40/jre/lib/jfr.jar:/usr/java/jdk1.7.0_40/jre/lib/jsse.jar:/usr/java/jdk1.7.0_40/jre/lib/rt.jar:/usr/java/jdk1.7.0_40/jre/lib/plugin.jar:/usr/java/jdk1.7.0_40/jre/lib/deploy.jar:/usr/java/jdk1.7.0_40/jre/lib/charsets.jar:/usr/java/jdk1.7.0_40/jre/lib/javaws.jar:/usr/java/jdk1.7.0_40/jre/lib/jfxrt.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/zipfs.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/localedata.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/sunpkcs11.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/sunec.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/dnsns.jar:/usr/java/jdk1.7.0_40/jre/lib/ext/sunjce_provider.jar:/home/kshk/IdeaProjects/mavenKris/target/classes:/home/kshk/Software/idea-IC-129.1359/lib/idea_rt.jar:/home/kshk/.m2/repository/org/apache/hadoop/hadoop-core/1.0.3-gphd-1.2.0.0/hadoop-core-1.0.3-gphd-1.2.0.0.jar:/home/kshk/.m2/repository/commons-logging/commons-logging/1.0.3/commons-logging-1.0.3.jar:/home/kshk/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/home/kshk/.m2/repository/commons-lang/commons-lang/2.1/commons-lang-2.1.jar com.intellij.rt.execution.application.AppMain mavenKris.Dictionary
Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: 0
at mavenKris.Dictionary.main(Dictionary.java:37)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:120)
[kshk@localhost tmp]$
编辑：
Intellij 中的库 jars
