1

我尝试使用 hdfs c api 运行用 c++ 代码编写的程序。但它不起作用。此代码用于读取本地文件并写入 hdfs，如hdfs dfs -put命令。

这是我的工作环境：

linux vesrion: 2.6.32-573.el6.x86_64
hadoop version: 2.6.5
hadoop datanode number : 1 (local)
java version: 1.8.0

hdfs-site.xml：

<property>
<name>dfs.datanode.drop.cache.behind.reads</name>
<value>true</value>
</property>

<property>
<name>dfs.datanode.drop.cache.behind.writes</name>
<value>true</value>
</property>

<property>
<name>dfs.datanode.sync.behind.writes</name>
<value>true</value>
</property>

我的代码：

#include "hdfs.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

int main(int argc, char **argv) {

  int i = 0;
  const char* writePath = "/tmp_api/file_";
  const char* readPath = "data_set/file_";

  hdfsFS fs;
  hdfsFile writeFile;

  char path[32];

  char buffer[20971520 + 128];

  FILE* fp;

  tSize result;

  tSize numWrittenBytes;

  int status;

  fs = hdfsConnect("default", 0);

  for(i=1;i<101;i++){
    memset(path, 0, 32);
    sprintf(path, "%s%d", readPath, i);

    fp = fopen(path, "r");

    printf("read path : %s\n", path);

    result = fread(buffer, 1, 20971520, fp);

    fclose(fp);

    memset(path, 0, 32);
    sprintf(path, "%s%d", writePath, i);


    printf("write path : %s\nresult : %lu\n", path, result);

    writeFile = hdfsOpenFile(fs, path, O_CREAT | O_WRONLY, 0, 0, 0);
    if(!writeFile) {
          printf( "Failed to open %s for writing!\n", path);
          exit(-1);
    }

    numWrittenBytes = hdfsWrite(fs, writeFile, (void*)buffer, result);
    if(numWrittenBytes < 0)
    {
      printf("error\n");
      exit(-1);

    }

    printf("written byte : %d\n", numWrittenBytes);
    if (hdfsFlush(fs, writeFile)) {
      printf( "Failed to 'flush' %s\n", path);
      exit(-1);
    }
    hdfsCloseFile(fs, writeFile);
  }

  hdfsDisconnect(fs);

  return 0;

}

错误 ：

FSDataOutputStream#close error:
java.lang.IllegalMonitorStateException
        at java.lang.Object.wait(Native Method)
        at java.lang.Thread.join(Thread.java:1245)
        at java.lang.Thread.join(Thread.java:1319)
        at org.apache.hadoop.hdfs.DFSOutputStream.closeThreads(DFSOutputStream.java:2187)
        at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:2232)
        at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72)
        at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106)

请帮我。

4

0 回答 0