2

Update:

I believe that this could be caused by the fact that I'm using the body parser provided by express. Could this be messing with the stream that multiparty is trying to parse?

I'm basing my solution on this answer.

What I'm trying to do: Stream a file from a client browser straight to S3 with my NodeJS server acting as a proxy (for security purposes). I don't want the file to touch the server's filesystem to avoid that bottleneck.

I'm getting the following error:

events.js:72
        throw er; // Unhandled 'error' event
              ^
Error: stream ended unexpectedly
    at Form.<anonymous> (/Users/kentcdodds/Developer/bucketstreams/node_modules/multiparty/index.js:619:24)
    at Form.EventEmitter.emit (events.js:117:20)
    at finishMaybe (/Users/kentcdodds/Developer/bucketstreams/node_modules/multiparty/node_modules/readable-stream/lib/_stream_writable.js:443:14)
    at endWritable (/Users/kentcdodds/Developer/bucketstreams/node_modules/multiparty/node_modules/readable-stream/lib/_stream_writable.js:452:3)
    at Form.Writable.end (/Users/kentcdodds/Developer/bucketstreams/node_modules/multiparty/node_modules/readable-stream/lib/_stream_writable.js:419:5)
    at onend (_stream_readable.js:457:10)
    at process._tickCallback (node.js:415:13)

I've looked at the code and can't quite seem to understand what's causing the issue. I'm using angular-file-upload because I'm using angular on the front end. Here's what the request looks like:

Request URL:http://local.bucketstreams.com:3000/upload/image
Request Headers CAUTION: Provisional headers are shown.
Accept:application/json, text/plain, */*
Cache-Control:no-cache
Content-Type:multipart/form-data; boundary=----WebKitFormBoundaryNKuH2H9IUB7kvmea
Origin:http://local.bucketstreams.com:3000
Pragma:no-cache
Referer:http://local.bucketstreams.com:3000/getting-started
User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36
Request Payload
------WebKitFormBoundaryNKuH2H9IUB7kvmea
Content-Disposition: form-data; name="type"

profile
------WebKitFormBoundaryNKuH2H9IUB7kvmea
Content-Disposition: form-data; name="user"

pie
------WebKitFormBoundaryNKuH2H9IUB7kvmea
Content-Disposition: form-data; name="file0"; filename="Screen Shot 2014-02-18 at 10.54.06 PM.png"
Content-Type: image/png


------WebKitFormBoundaryNKuH2H9IUB7kvmea--

And here's what my code looks like:

var ErrorController = require('../controller/ErrorController');
var AuthenticationController = require('../controller/AuthenticationController');
var logger = require('winston');

var http = require('http');
var util = require('util');
var multiparty = require('multiparty');
var knox = require('knox');
var Batch = require('batch');

var s3Client = knox.createClient({
  secure: false,
  key: process.env.S3_KEY,
  secret: process.env.S3_SECRET,
  bucket: process.env.S3_BUCKET_IMAGES
});

var Writable = require('readable-stream').Writable;
util.inherits(ByteCounter, Writable);
function ByteCounter(options) {
  Writable.call(this, options);
  this.bytes = 0;
}

ByteCounter.prototype._write = function(chunk, encoding, cb) {
  this.bytes += chunk.length;
  cb();
};

var supportedTypes = {
  profile: true,
  post: true
};

module.exports = function(app) {
  app.post('/upload/image', AuthenticationController.checkAuthenticated, function(req, res) {
    var type = req.body.type;
    var userId = req.user._id;
    if (!supportedTypes[type]) {
      return ErrorController.sendErrorJson(res, 401, 'Unsupported image upload type: ' + type);
    }

    var headers = {
      'x-amz-acl': 'public-read'
    };
    var form = new multiparty.Form();
    var batch = new Batch();
    batch.push(function(cb) {
      form.on('field', function(name, value) {
        if (name === 'path') {
          var destPath = value;
          if (destPath[0] !== '/') destPath = '/' + destPath;
          cb(null, destPath);
        }
      });
    });

    batch.push(function(cb) {
      form.on('part', function(part) {
        if (! part.filename) return;
        cb(null, part);
      });
    });

    batch.end(function(err, results) {
      if (err) throw err;
      form.removeListener('close', onEnd);
      var destPath = '/' + userId + results[0];
      var part = results[1];

      var counter = new ByteCounter();
      part.pipe(counter); // need this until knox upgrades to streams2
      headers['Content-Length'] = part.byteCount;
      s3Client.putStream(part, destPath, headers, function(err, s3Response) {
        if (err) throw err;
        res.statusCode = s3Response.statusCode;
        s3Response.pipe(res);
        console.log('https://s3.amazonaws.com/' + process.env.S3_BUCKET_IMAGES + destPath);
      });
      part.on('end', function() {
        console.log('part end');
        console.log('size', counter.bytes);
      });
    });

    form.on('close', function(error) {
      logger.error(error);
      return ErrorController.sendErrorJson(res, 500, 'There was a problem uploading the file.');
    });
    form.parse(req);
  });
};

It looks like the part that is blowing up is multiparty and I've looked into that code a little bit to no avail. I'm not certain if I'm making the request incorrectly or if there's something wrong with my server code. I don't think it has anything to do with my S3 bucket, but I suppose it could be that as well.

Anyway, any tips are welcome.

4

2 回答 2

2

节点多方源上有一个 S3 示例。

https://github.com/andrewrk/node-multiparty/tree/master/examples

var http = require('http'),
    util = require('util'),
    multiparty = require('../'),
    knox = require('knox'),
    Batch = require('batch'),
    PORT = process.env.PORT || 27372

var s3Client = knox.createClient({
    secure: false,
    key: process.env.S3_KEY,
    secret: process.env.S3_SECRET,
    bucket: process.env.S3_BUCKET,
});

var server = http.createServer(function(req, res) {
    if (req.url === '/') {
        res.writeHead(200, {
            'content-type': 'text/html'
        });
        res.end(
            '<form action="/upload" enctype="multipart/form-data" method="post">' +
            '<input type="text" name="path"><br>' +
            '<input type="file" name="upload"><br>' +
            '<input type="submit" value="Upload">' +
            '</form>'
        );
    } else if (req.url === '/upload') {
        var headers = {
            'x-amz-acl': 'public-read',
        };
        var form = new multiparty.Form();
        var batch = new Batch();
        batch.push(function(cb) {
            form.on('field', function(name, value) {
                if (name === 'path') {
                    var destPath = value;
                    if (destPath[0] !== '/') destPath = '/' + destPath;
                    cb(null, destPath);
                }
            });
        });
        batch.push(function(cb) {
            form.on('part', function(part) {
                if (!part.filename) return;
                cb(null, part);
            });
        });
        batch.end(function(err, results) {
            if (err) throw err;
            form.removeListener('close', onEnd);
            var destPath = results[0],
                part = results[1];

            headers['Content-Length'] = part.byteCount;
            s3Client.putStream(part, destPath, headers, function(err, s3Response) {
                if (err) throw err;
                res.statusCode = s3Response.statusCode;
                s3Response.pipe(res);
                console.log("https://s3.amazonaws.com/" + process.env.S3_BUCKET + destPath);
            });
        });
        form.on('close', onEnd);
        form.parse(req);

    } else {
        res.writeHead(404, {
            'content-type': 'text/plain'
        });
        res.end('404');
    }

    function onEnd() {
        throw new Error("no uploaded file");
    }
});
server.listen(PORT, function() {
    console.info('listening on http://0.0.0.0:' + PORT + '/');
});

````

于 2014-02-20T17:48:38.640 回答
2

解决方案的关键是不使用 (显然已弃用) bodyParser()。我不确定它做了什么，但它搞砸了表单的multiparty使用部分。因此，如果您遇到与我相同的问题，请不要使用 using bodyParser()，而是明确使用您需要的东西（例如）：

app.use(express.urlencoded());
app.use(express.json());

然后对于您的多部分内容，只需使用 multiparty 自己解析正文。multiparty 的作者提供了有关该主题的更多信息。

于 2014-02-23T05:35:05.953 回答