3

我无法将视频等大文件上传到 s3。它最终超时。我尝试使用 fs 来流式传输它，但我一定不能正确使用它。

我已经尝试了所有我能想到的让 fs 流式传输此文件的方法。我不知道是否可以像我在 multerS3 中使用 fs 那样在单独的上传路径中使用它。我可以上传图片和非常小的视频，但仅此而已。

// Here is my s3 index file which exports upload

const crypto = require('crypto');
const aws = require('aws-sdk');
const multerS3 = require('multer-s3');
const fs = require('fs');

aws.config.update({
  secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
  accessKeyId: process.env.AWS_ACCESS_KEY_ID,
  region: 'us-east-1',
  ACL: 'public-read'
});

const s3 = new aws.S3({ httpOptions: { timeout: 10 * 60 * 1000 }});
var options = { partSize: 5 * 1024 * 1024, queueSize: 10 };

const fileFilter = (req, file, cb) => {
  console.log('file.mimetype is ', file.mimetype);
  if (file.mimetype === 'image/jpeg' || file.mimetype === 'image/png' || file.mimetype === 'video/mp4' || file.mimetype === 'video/avi' || file.mimetype === 'video/mov' || file.mimetype === 'video/quicktime') {
    cb(null, true);
  } else {
    cb(new Error('Invalid file type'), false);
  }
}
const filename = getFileName();

const upload = multer({
  fileFilter,
  storage: multerS3({
    acl: 'public-read',
    s3,
    options,
    body: fs.createReadStream(filename),
    bucket: 'skilljack',
    metadata: function (req, file, cb) {
      cb(null, {fieldName: 'TESTING_METADATA'})
    },
    key: function (req, file, cb) {
        let buf = crypto.randomBytes(16);
        buf = buf.toString('hex');
        let uniqFileName = file.originalname.replace(/\.jpeg|\.jpg|\.png|\.avi|\.mov|\.mp4/ig, '');
        uniqFileName += buf;
      cb(undefined, uniqFileName );
    }
  })
});

function getFileName (req, file) {
  if (file) {
    const body = fs.createReadStream(file.originalname);
    return body;
  }
}

  module.exports = {
      upload
  }

// Here is my route file
const express = require('express');
const router = express.Router({ mergeParams: true });
const multer = require('multer');
const { upload } = require('../s3');
const { asyncErrorHandler, isLoggedIn, isAuthor } = require('../middleware');


const {
    postCreate,
    postDestroy
} = require('../controllers/posts');

router.post('/', isLoggedIn, asyncErrorHandler(isAuthor), upload.single('image'), asyncErrorHandler(postCreate));
router.delete('/:post_id', isLoggedIn, asyncErrorHandler(isAuthor), asyncErrorHandler(postDestroy));
module.exports = router;
4

2 回答 2

1

除了将设置queueSize为 1之外，您可能还需要禁用超时。

const s3 = new aws.S3({
    accessKeyId: config.get('accessKeyId'),
    secretAccessKey: config.get('secretAccessKey'),
    Bucket: config.get('bucket'),
});

s3.config.httpOptions.timeout = 0

参考

于 2020-09-17T14:18:15.700 回答
0

这几天都遇到同样的问题。避免它的一种方法（但需要很长时间）是将 queueSize 减少到 1。

当您的网络不是很好时，就会发生这种情况，最终，一些队列将保持空闲并触发超时。

QueueSize 为 1 将允许为正在上传的部分分配更多带宽，避免超时。

于 2020-07-23T09:46:02.683 回答