3

我正在尝试启动一个迷你 Hadoop 集群以进行单元测试。

我的 Maven 配置具有以下依赖项：

    <dependency>
        <groupId>org.apache.hbase</groupId>
        <artifactId>hbase</artifactId>
        <version>0.95.0</version>
        <type>pom</type>
    </dependency>

    <dependency>
        <groupId>org.apache.mrunit</groupId>
        <artifactId>mrunit</artifactId>
        <classifier>hadoop2</classifier>
        <version>1.0.0</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>1.2.1</version>

    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-test</artifactId>
        <version>1.2.1</version>
    </dependency>

我的代码如下所示：

class Foo {
    private final HBaseTestingUtility testUtil = new HBaseTestingUtility();
    private HTable hTable;

    @Before
    public void setUp() throws Exception {
        testUtil.getConfiguration();
        //Creates temp Hbase
        logger.info("Test setup...");
        testUtil.startMiniCluster();
        hTable = testUtil.createTable("TableName".getBytes(),"d".getBytes());
        logger.info("Hbase table created.");
    }

    @Test
    public void bar() {
        [...]
    }
}

我使用 mvn clean test -e -DskipTests=false 启动测试

现在 Hbase 无法加载 MiniDFSCluster，这是错误跟踪：

-------------------------------------------------------------------------------
Test set: com.Foo
-------------------------------------------------------------------------------
Tests run: 1, Failures: 0, Errors: 1, Skipped: 0, Time elapsed: 0.684 sec <<< FAILURE!
bar(com.Foo)  Time elapsed: 0.034 sec  <<< ERROR!
java.lang.NoSuchMethodError: org.apache.hadoop.hdfs.MiniDFSCluster.<init>(ILorg/apache/hadoop/conf/Configuration;IZZZLorg/apache/hadoop/hdfs/server/common/HdfsServerConstants$StartupOption;[Ljava/lang/String;[Ljava/lang/String;[J)V
    at org.apache.hadoop.hbase.HBaseTestingUtility.startMiniDFSCluster(HBaseTestingUtility.java:328)
    at org.apache.hadoop.hbase.HBaseTestingUtility.startMiniCluster(HBaseTestingUtility.java:496)
    at org.apache.hadoop.hbase.HBaseTestingUtility.startMiniCluster(HBaseTestingUtility.java:452)
    at org.apache.hadoop.hbase.HBaseTestingUtility.startMiniCluster(HBaseTestingUtility.java:421)
    at Foo.setUp(Foo.java:51)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:271)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50)
    at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
    at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
    at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
    at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:236)
    at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:134)
    at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:113)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.apache.maven.surefire.util.ReflectionUtils.invokeMethodWithArray(ReflectionUtils.java:189)
    at org.apache.maven.surefire.booter.ProviderFactory$ProviderProxy.invoke(ProviderFactory.java:165)
    at org.apache.maven.surefire.booter.ProviderFactory.invokeProvider(ProviderFactory.java:85)
    at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:103)
    at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:74)

关于问题根源的任何想法？

4

1 回答 1

0

较新版本的 MiniDFSCluster.java 位于 hadoop-hdfs jar 中。

利用：

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-hdfs</artifactId>
    <version>2.2.0</version>
</dependency>

代替：

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-test</artifactId>
    <version>1.2.1</version>
</dependency>
于 2014-02-25T21:31:53.113 回答