2

我正在使用 AzureblockBlobURL.download()下载图像，但只收到图像的顶部。我可以从 Azure blob 下载到可读流的数量是否有限制？内容长度为 172628 并且存在一个属性highWaterMark: 16384。这些有关系吗？

async function compareToBaseline(imageData, blobName, metadata){

  const baselineBlobName = "MacOSX10.12/chrome/initial"

  const containerURL = ContainerURL.fromServiceURL(serviceURL, "baselines")
  const blockBlobURL = BlockBlobURL.fromContainerURL(containerURL, baselineBlobName );
  let baseLineImage = await blockBlobURL.download(aborter, 0)

  baseLineImage = baseLineImage.originalResponse.readableStreamBody.read()
  console.log(baseLineImage.length);

  baseLineImage = new Buffer(baseLineImage, 'base64');
  await fs.writeFile('./newest.png', baseLineImage, 'binary', function(err){
    console.log('written');
  })
}

结果只是图像的顶部。

4

3 回答 3

1

对 Azure 存储服务的每次调用都有 4 MB 的限制。如果您的文件大于 4 MB，则必须将其分块。有关详细信息，请参阅Azure 存储可伸缩性和性能目标。

这是以 1MB 块下载非常大文件的示例 c# 代码。它也以性能为导向。

 private static void DownloadLargeFile()
        {
            string connectionString = "connString"; //ConfigurationSettings.AppSettings["StorageConnectionString"]; //blob connection string
#pragma warning restore CS0618 // Type or member is obsolete
#pragma warning disable CS0618 // Type or member is obsolete
            string sourceContainerName = "quickstartblob"; //ConfigurationSettings.AppSettings["sourcecontainerName"]; //source blob container name            
#pragma warning restore CS0618 // Type or member is obsolete
            string sourceBlobFileName = "QuickStart1.txt"; //source blob name
            CloudStorageAccount account = CloudStorageAccount.Parse(connectionString);
            var blobClient = account.CreateCloudBlobClient();
            var container = blobClient.GetContainerReference(sourceContainerName);
            var file = sourceBlobFileName;
            var blob = container.GetBlockBlobReference(file);
            //First fetch the size of the blob. We use this to create an empty file with size = blob's size
            blob.FetchAttributes();
            var blobSize = blob.Properties.Length;
            long blockSize = (1 * 1024 * 1024);//1 MB chunk;
            blockSize = Math.Min(blobSize, blockSize);
            //Create an empty file of blob size
            using (FileStream fs = new FileStream(file, FileMode.Create))//Create empty file.
            {
                fs.SetLength(blobSize);//Set its size
            }
            var blobRequestOptions = new BlobRequestOptions
            {
                RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(5), 3),
                MaximumExecutionTime = TimeSpan.FromMinutes(60),
                ServerTimeout = TimeSpan.FromMinutes(60)
            };
            long currentPointer = 0;
            long bytesRemaining = blobSize;
            do
            {
                var bytesToFetch = Math.Min(blockSize, bytesRemaining);
                using (MemoryStream ms = new MemoryStream())
                {
                    //Download range (by default 1 MB)
                    blob.DownloadRangeToStream(ms, currentPointer, bytesToFetch, null, blobRequestOptions);
                    ms.Position = 0;
                    var contents = ms.ToArray();
                    using (var fs = new FileStream(file, FileMode.Open))//Open that file
                    {
                        fs.Position = currentPointer;//Move the cursor to the end of file.
                        fs.Write(contents, 0, contents.Length);//Write the contents to the end of file.
                    }
                    currentPointer += contents.Length;//Update pointer
                    bytesRemaining -= contents.Length;//Update bytes to fetch
                }
            }
            while (bytesRemaining > 0);
        }

在节点 js 中如下所示

var azure = require('azure-storage');
var fs = require('fs');

module.exports = function (context, input) {

context.done();

var accessKey = 'myaccesskey';
var storageAccount = 'mystorageaccount';
var containerName = 'mycontainer';

var blobService = azure.createBlobService(storageAccount, accessKey);

var recordName = "a_large_movie.mov";
var blobName = "standard/mov/" + recordName;

var blobSize;
var chunkSize = (1024 * 512) * 8; // I'm experimenting with this variable
var startPos = 0;
var fullPath = "D:/home/site/wwwroot/myAzureFunction/input/";
var blobProperties = blobService.getBlobProperties(containerName, blobName,   null, function (error, blob) {
    if (error) {
        throw error;
    }
    else    {
        blobSize = blob.contentLength;
        context.log('Registered length: ' + blobSize);
        fullPath = fullPath + recordName;
        console.log(fullPath);
        doDownload();
    }
}
);

function doDownload() {
var stream = fs.createWriteStream(fullPath, {flags: 'a'});
var endPos = startPos + chunkSize;
if (endPos > blobSize) {
    endPos = blobSize;
    context.log('Reached end of file endPos: ' + endPos);
}

context.log("Downloading " + (endPos - startPos) + " bytes starting from " + startPos + " marker.");

blobService.getBlobToStream(
    containerName, 
    blobName, 
    stream, 
    { 
        "rangeStart": startPos, 
        "rangeEnd": endPos-1 
    }, 
    function(error) {
        if (error) {
            throw error;
        }
        else if (!error) {
            startPos = endPos;
            if (startPos <= blobSize - 1) {
                doDownload();
            }
        }
    }
);
}

};

希望能帮助到你。

于 2019-06-06T05:22:55.743 回答
0

blockBlobURL.download() doesn't have a limit to file size. But read() returns null doesn't mean no more data in the stream. You need to follow Node.js practices to get all data by listening to data or readable event.

For example, the data event posted by Peter Pan. Or the readable event posted by Node.js official documents:

readable.on('readable', () => {
  let chunk;
  while (null !== (chunk = readable.read())) {
    console.log(`Received ${chunk.length} bytes of data.`);
  }
});

Please always call read() inside readable event callback.

于 2019-06-13T08:46:24.173 回答
-1

看来这个问题与您的其他线程Unable to read readableStreamBody 从下载的 blob类似。

这是我的功能，可帮助将文件保存baseLineImage.readableStreamBody到文件中，如下所示。

async function streamToFs(filename, readableStream) {
    const ws = fs.createWriteStream(filename);
    readableStream.on("data", data => {
      ws.write(data);
    }).on("end", () => {
      console.log('written');
    });
}

并如下更改您的代码。

async function compareToBaseline(imageData, blobName, metadata){

  const baselineBlobName = "MacOSX10.12/chrome/initial"

  const containerURL = ContainerURL.fromServiceURL(serviceURL, "baselines");
  const blockBlobURL = BlockBlobURL.fromContainerURL(containerURL, baselineBlobName );
  let baseLineImage = await blockBlobURL.download(aborter, 0);

  await streamToFs('./newest.png', baseLineImage.readableStreamBody)；
}

有用。希望能帮助到你。

于 2019-06-06T10:38:48.983 回答