0

我在 S3 存储桶中有几个数据（csv）文件排列在以下虚拟目录中：

s3://mybucketname/data/Type1/

存储桶名称：mybucketname
密钥前缀：/data/Type1/

我想在这个bucket+key中组合这些csv文件的内容并将输出作为字符串返回。代码 -

public string GetBucketContents(string bucketName)
{
    string bucketContent = "";
    ListObjectsRequest request = new ListObjectsRequest
    {
        BucketName = bucketName,
        Prefix = "data/Type1/"
    };

    using (ListObjectsResponse response = amazonS3Client.ListObjects(request))
    {
        //Loop all objects in S3 bucket
        foreach (S3Object s3Object in response.S3Objects)
        {
            //Get S3 object @ s3Object.Key
            GetObjectRequest objectRequest = new GetObjectRequest().WithBucketName(bucketName).WithKey(s3Object.Key);                   
            using (GetObjectResponse objectResponse = amazonS3Client.GetObject(objectRequest))
            {
                using (StreamReader reader = new StreamReader(objectResponse.ResponseStream))
                {
                    bucketContent += reader.ReadToEnd();
                }
            }
        }
    }

    return bucketContent;
}

我得到了预期的输出:-)
但仍然想验证是否有更好的（性能方面）方法来实现这一目标？

谢谢！

4

2 回答 2

1

如果您使用的是支持 aync 并等待 ajaxy 魔法的较新版本的 .net，那么您可以立即触发所有读取请求，然后等待所有请求返回以构造您的字符串。在大数据读取时，这可以提高性能并节省大量时间。

这是一些代码：（您需要修改字符串以适合您的目的并摆脱秒表但已包含在内，以便您查看时差）

public string GetBucketContents(string bucketname)

    string prefix = "S3/PathToCSVs/";
    StringBuilder sb = new StringBuilder();

    List<Task<string>> tasks = new List<Task<string>>();

    Stopwatch sw = new Stopwatch();

    // Begin timing
    sw.Start();

    using (client = new AmazonS3Client(AWSAccessKey, AWSSecretKey, Amazon.RegionEndpoint.USWest2))
    {
        try
        {
            ListObjectsRequest request = new ListObjectsRequest
            {
                BucketName = bucketname,
                Prefix = prefix
            };

            do
            {
                ListObjectsResponse response = client.ListObjects(request);


                // Process response.
                foreach (S3Object entry in response.S3Objects)
                {
                    GetObjectRequest gor = new GetObjectRequest
                    {
                        BucketName = bucketname,
                        Key = entry.Key
                    };


                    tasks.Add(Task<string>.Factory.StartNew(() =>
                    {
                        return ProcessResults(gor, sw);
                    }));

                }

                // If response is truncated, set the marker to get the next 
                // set of keys.
                if (response.IsTruncated)
                {
                    request.Marker = response.NextMarker;
                }
                else
                {
                    request = null;
                }
            } while (request != null);


            Task.WaitAll(tasks.ToArray());

            foreach (var s in tasks)
            {
                sb.AppendLine(sw.Elapsed.ToString() + "<br/>");
                sb.AppendLine(s.Result + "<br/>");
            }


            // Stop timing
            sw.Stop();
            sb.AppendLine(sw.Elapsed.ToString() + "<br/>");

            return sb.ToString();

        }
        catch (AmazonS3Exception amazonS3Exception)
        {
            if (amazonS3Exception.ErrorCode != null && (amazonS3Exception.ErrorCode.Equals("InvalidAccessKeyId") || amazonS3Exception.ErrorCode.Equals("InvalidSecurity")))
            {
                return "Check the provided AWS Credentials. To sign up for service, go to http://aws.amazon.com/s3";
            }
            else
            {
                return string.Format("Error occurred. Message:'{0}' when listing objects", amazonS3Exception.Message);
            }
        }
    }
}

public static string ProcessResults(GetObjectRequest gor, Stopwatch sw)
{
    string result = "";

    using (GetObjectResponse goresp = client.GetObject(gor))
    {
        using (StreamReader reader = new StreamReader(goresp.ResponseStream))
        {
            result = sw.Elapsed.ToString() + "<br/>" + reader.ReadToEnd() + "<br/>";
        }
    }

    return result;
}
于 2014-02-10T11:11:37.417 回答
-1

我来自 Bucket Explorer 的开发者团队网站。Bucket Explorer 的导出到 csv 功能为您提供了导出数据相关信息以导出为 csv 格式的选项。

http://www.bucketexplorer.com/documentation/amazon-s3--export-object-details-to-csv.html

于 2012-07-30T06:47:08.363 回答