我想将 20-200 GG 左右的大文件从 Blob 传输到 FTP。
我正在使用下面的代码来执行相同的操作，但在达到 500 MB 后使内存出现异常。
public void TriggerEfaTrasfer()
{
string blobConnectionString = ConfigurationManager.AppSettings["BlobConnectionString"];
string blobContainer = ConfigurationManager.AppSettings["FileContainer"];
CloudStorageAccount storageaccount = CloudStorageAccount.Parse(blobConnectionString);
var blobClient = storageaccount.CreateCloudBlobClient();
var container = blobClient.GetContainerReference(blobContainer);
var blobs = container.ListBlobs();
TransferFilesFromBlobToEfa(blobs);
}
private void TransferFilesFromBlobToEfa(IEnumerable<IListBlobItem> blobitem)
{
try
{
OpenConnection();
foreach (var item in blobitem.Where((blobItem, type) => blobItem is CloudBlockBlob))
{
UploadFileStream(item as CloudBlockBlob);
}
// List all additional subdirectories in the current directory, and call recursively:
foreach (var item in blobitem.Where((blobItem, type) => blobItem is CloudBlobDirectory))
{
var directory = item as CloudBlobDirectory;
Directory.CreateDirectory(directory.Prefix);
TransferFilesFromBlobToEfa(directory.ListBlobs());
}
}
catch (Exception ex)
{
// throw ex.InnerException;
log.Error(ex);
}
finally
{
ftpConnection.Close();
}
}
private void UploadFileStream(CloudBlockBlob blobFile)
{
string fileName = Path.GetFileName(blobFile.Name);
FtpWebRequest request = (FtpWebRequest)WebRequest.Create("ftp://" + ConfigurationManager.AppSettings["StagingServerHostName"] + "//" + fileName);
request.Method = WebRequestMethods.Ftp.UploadFile;
request.Credentials = new NetworkCredential(ConfigurationManager.AppSettings["FtpUserName"], ConfigurationManager.AppSettings["FtpPassword"]);
request.UsePassive = true;
request.UseBinary = true;
request.KeepAlive = true;
request.Timeout = 700000;
try
{
byte[] chunk = new byte[4000000];
long blobTotalLength = blobFile.Properties.Length;
long totalChunks = (blobTotalLength - 1) / 4000000 + 1;
using (Stream reqStream = request.GetRequestStream())
{
using (var fileStream = new MemoryStream())
{
if (totalChunks > 1)
{
for (int index = 0; index < totalChunks; index++)
{
try
{
if (index == 0)
{
blobFile.DownloadRangeToStream(fileStream, 0, chunk.Length);
reqStream.Write(fileStream.ToArray(), 0, chunk.Length);
}
//last chunk
else if (index == totalChunks - 1)
{
long remaningContent = blobTotalLength - (index * 4000000);
blobFile.DownloadRangeToStream(fileStream, index * 4000000, remaningContent);
reqStream.Write(fileStream.ToArray(), index * 4000000, (Int32)remaningContent);
}
else
{
blobFile.DownloadRangeToStream(fileStream, index * 4000000, chunk.Length);
reqStream.Write(fileStream.ToArray(), index * 4000000, chunk.Length);
}
}
catch (Exception ex)
{
}
}
}
else
{
blobFile.DownloadToStream(fileStream);
reqStream.Write(fileStream.ToArray(), 0, fileStream.ToArray().Length);
}
fileStream.Flush();
fileStream.Close();
reqStream.Flush();
reqStream.Close();
}
}
//Gets the FtpWebResponse of the uploading operation
FtpWebResponse response = (FtpWebResponse)request.GetResponse();
log.Warn("Response for " + fileName + " " + response.StatusDescription);
response.Close();
}
catch (Exception ex)
{
log.Error(ex);
}
}