您可以启动所需数量的处理任务并用于BlockingCollection
工作enqueue
。像这样的东西:
// my dummy async enumerable
public async IAsyncEnumerable<int> ReadAllAsync()
{
for (int i = 0; i < 3; i++)
{
yield return i*3 + 1;
yield return i*3 + 2;
yield return i*3 + 3;
await Task.Delay(200);
}
yield return 777;
}
var collection = new BlockingCollection<int>();
// start "processors"
var tasks = Enumerable.Range(0, 4)
.Select(i =>
Task.Run(() =>
{
while (!collection.IsCompleted)
{
int? data = null;
try
{
data = collection.Take();
}
catch (InvalidOperationException) { }
if (data != null)
{
// simulate processing
Thread.Sleep(400);
Console.WriteLine(data.Value);
}
}
Console.WriteLine("No more items to take.");
}))
.ToArray();
await foreach (var record in ReadAllAsync())
{
collection.Add(record);
}
collection.CompleteAdding(); // signal that enqueuing has finished
await Task.WhenAll(tasks);
这可以通过引入一些异步信号(例如使用SemaphoreSlim.WaitAsync
或AsyncManualResetEvent.WaitAsync
使用)来改进,以便消费者线程在等待新项目时不会消耗 CPU。例如:
var collection = new ConcurrentQueue<int>();
var semaphore = new SemaphoreSlim(0, 4);
var cts = new CancellationTokenSource(); // to signal that queueing is completed
var tasks = Enumerable.Range(0, 4)
.Select(i =>
Task.Run(async () =>
{
while (true)
{
if (cts.Token.IsCancellationRequested && !collection.Any())
{
Console.WriteLine("No more items to take.");
break;
}
else if (!cts.Token.IsCancellationRequested)
{
try
{
await semaphore.WaitAsync(cts.Token);
}
catch (OperationCanceledException)
{
//ignore
}
}
if(collection.TryDequeue(out var data))
{
//simulate work
Thread.Sleep(400);
Console.WriteLine(data);
}
}
}))
.ToArray();
await foreach (var record in ReadAllAsync())
{
collection.Enqueue(record);
semaphore.Release();
}
cts.Cancel(); // addition completed.
await Task.WhenAll(tasks);
Console.WriteLine("end");