我尝试使用 Node.js 处理一个 500MB 的 Apache 日志文件,将其语法从
ip.ip.ip.ip - - [02/Aug/2012:05:01:17 -0600] "GET /path/of/access/ HTTP/1.1" 302 26
到
ip.ip.ip.ip - - 02/Aug/2012:05:01:17 GET /path/of/access/ HTTP/1.1 302 26
,然后写入另一个文本文件。
为了更好的内存控制和性能,我使用了fs.createReadStream
and fs.createWriteStream
,但只设法将第一行写入output.txt
,因为脚本以错误结尾:
{ [Error: EBADF, write] errno: 9, code: 'EBADF' }
在这里,我发布了一些可能有助于调试的信息。
负责人input.txt
:
ip.ip.ip.ip - - [02/Aug/2012:05:01:17 -0600] "GET /path/of/access/ HTTP/1.1" 302 26
ip.ip.ip.ip - - [02/Aug/2012:05:01:17 -0600] "GET /path/of/access/ HTTP/1.1" 302 26
ip.ip.ip.ip - - [02/Aug/2012:05:01:17 -0600] "GET /path/of/access/ HTTP/1.1" 302 26
ip.ip.ip.ip - - [02/Aug/2012:05:01:17 -0600] "GET /path/of/access/ HTTP/1.1" 302 26
ip.ip.ip.ip - - [02/Aug/2012:05:01:17 -0600] "GET /path/of/access/ HTTP/1.1" 302 26
ip.ip.ip.ip - - [02/Aug/2012:05:01:17 -0600] "GET /path/of/access/ HTTP/1.1" 302 26
ip.ip.ip.ip - - [02/Aug/2012:05:01:18 -0600] "GET /path/of/access/ HTTP/1.1" 302 26
内容output.txt
:
ip.ip.ip.ip - - [02/Aug/2012:05:01:17 -0600] "GET /path/of/access/ HTTP/1.1" 302 26
整个脚本:
var fs = require('fs');
var data ='';
var n=0; //For line control
var r = fs.createReadStream('./input.txt',{
encoding: 'ascii',
start:0,
// end: 100000,
});
var w = fs.createWriteStream('./output.txt',{
encoding:'ascii'
});
function put(line){ //write into w;
++n;
w.write(line+'\n');
}
function end(){
r.destroy();
w.destroy();
}
function onData(chunk){
var hasNewline = chunk.indexOf('\n')!==-1;
if(hasNewline){
var arr = chunk.split('\n');
var first = arr.shift();
var last = arr.pop();
data+=first;
put(data); //write a complete line
arr.forEach(function(line){
put(line); //write a complete line
});
data=last;
}else{
data+=chunk;
}
if(n>100){
end();
}
}
function onErr(e){
console.log(e);
}
r.addListener( "data", onData);
r.addListener( "end", end);
r.addListener('error',onErr);
w.addListener('error',onErr);