15 minlesson

Throttling Patterns

Throttling Patterns

Throttling limits the rate of operations to prevent overwhelming resources. This is essential for API calls, database queries, and any operation with rate limits.

Why Throttle?

Without throttling:

  • APIs return rate limit errors (HTTP 429)
  • Databases become overloaded
  • Network connections exhaust
  • Memory usage spikes
  • Systems become unresponsive

SemaphoreSlim for Concurrency Limiting

The simplest throttling approach - limit concurrent operations:

csharp
1class ThrottledClient
2{
3    private readonly SemaphoreSlim _throttle;
4    private readonly HttpClient _client = new();
5

6    public ThrottledClient(int maxConcurrent)
7    {
8        _throttle = new SemaphoreSlim(maxConcurrent);
9    }
10

11    public async Task<string> GetAsync(string url)
12    {
13        await _throttle.WaitAsync();
14        try
15        {
16            return await _client.GetStringAsync(url);
17        }
18        finally
19        {
20            _throttle.Release();
21        }
22    }
23}
24

25// Usage: Max 5 concurrent requests
26var client = new ThrottledClient(5);
27var tasks = urls.Select(url => client.GetAsync(url));
28var results = await Task.WhenAll(tasks);

Parallel.ForEachAsync with MaxDegreeOfParallelism

.NET 6+ provides built-in parallel throttling:

csharp
1var options = new ParallelOptions
2{
3    MaxDegreeOfParallelism = 5  // Max 5 concurrent
4};
5

6await Parallel.ForEachAsync(urls, options, async (url, ct) =>
7{
8    var data = await httpClient.GetStringAsync(url, ct);
9    Console.WriteLine($"Downloaded: {url}");
10});

Rate Limiting (Requests per Second)

Limit the rate, not just concurrency:

csharp
1class RateLimiter
2{
3    private readonly SemaphoreSlim _semaphore;
4    private readonly TimeSpan _interval;
5

6    public RateLimiter(int maxRequestsPerSecond)
7    {
8        _semaphore = new SemaphoreSlim(maxRequestsPerSecond);
9        _interval = TimeSpan.FromSeconds(1.0 / maxRequestsPerSecond);
10    }
11

12    public async Task<T> ExecuteAsync<T>(Func<Task<T>> operation)
13    {
14        await _semaphore.WaitAsync();
15

16        try
17        {
18            return await operation();
19        }
20        finally
21        {
22            // Release after interval, not immediately
23            _ = ReleaseAfterDelay();
24        }
25    }
26

27    private async Task ReleaseAfterDelay()
28    {
29        await Task.Delay(_interval);
30        _semaphore.Release();
31    }
32}
33

34// Usage: Max 10 requests per second
35var limiter = new RateLimiter(10);
36foreach (var url in urls)
37{
38    var result = await limiter.ExecuteAsync(() => httpClient.GetStringAsync(url));
39}

Token Bucket Pattern

Classic rate limiting algorithm:

csharp
1class TokenBucket
2{
3    private readonly int _maxTokens;
4    private readonly double _tokensPerSecond;
5    private double _tokens;
6    private DateTime _lastRefill;
7    private readonly object _lock = new();
8

9    public TokenBucket(int maxTokens, double tokensPerSecond)
10    {
11        _maxTokens = maxTokens;
12        _tokensPerSecond = tokensPerSecond;
13        _tokens = maxTokens;
14        _lastRefill = DateTime.UtcNow;
15    }
16

17    public bool TryConsume(int tokens = 1)
18    {
19        lock (_lock)
20        {
21            Refill();
22

23            if (_tokens >= tokens)
24            {
25                _tokens -= tokens;
26                return true;
27            }
28

29            return false;
30        }
31    }
32

33    private void Refill()
34    {
35        var now = DateTime.UtcNow;
36        var elapsed = (now - _lastRefill).TotalSeconds;
37        var newTokens = elapsed * _tokensPerSecond;
38        _tokens = Math.Min(_maxTokens, _tokens + newTokens);
39        _lastRefill = now;
40    }
41}
42

43// Usage
44var bucket = new TokenBucket(maxTokens: 10, tokensPerSecond: 2);
45

46while (hasWork)
47{
48    if (bucket.TryConsume())
49    {
50        await DoWorkAsync();
51    }
52    else
53    {
54        await Task.Delay(100);  // Wait for tokens
55    }
56}

Sliding Window Pattern

Track requests in a time window:

csharp
1class SlidingWindowLimiter
2{
3    private readonly int _maxRequests;
4    private readonly TimeSpan _window;
5    private readonly Queue<DateTime> _timestamps = new();
6    private readonly object _lock = new();
7

8    public SlidingWindowLimiter(int maxRequests, TimeSpan window)
9    {
10        _maxRequests = maxRequests;
11        _window = window;
12    }
13

14    public bool TryAcquire()
15    {
16        lock (_lock)
17        {
18            var now = DateTime.UtcNow;
19            var windowStart = now - _window;
20

21            // Remove expired timestamps
22            while (_timestamps.Count > 0 && _timestamps.Peek() < windowStart)
23            {
24                _timestamps.Dequeue();
25            }
26

27            if (_timestamps.Count < _maxRequests)
28            {
29                _timestamps.Enqueue(now);
30                return true;
31            }
32

33            return false;
34        }
35    }
36}
37

38// Usage: Max 100 requests per minute
39var limiter = new SlidingWindowLimiter(100, TimeSpan.FromMinutes(1));

Batch Processing

Process items in batches to reduce overhead:

csharp
1async Task ProcessInBatchesAsync<T>(
2    IEnumerable<T> items,
3    int batchSize,
4    Func<T[], Task> processBatch)
5{
6    var batch = new List<T>(batchSize);
7

8    foreach (var item in items)
9    {
10        batch.Add(item);
11

12        if (batch.Count >= batchSize)
13        {
14            await processBatch(batch.ToArray());
15            batch.Clear();
16        }
17    }
18

19    // Process remaining items
20    if (batch.Count > 0)
21    {
22        await processBatch(batch.ToArray());
23    }
24}
25

26// Usage
27await ProcessInBatchesAsync(
28    records,
29    batchSize: 100,
30    async batch => await database.BulkInsertAsync(batch)
31);

Combining Throttling Strategies

csharp
1class SmartThrottler
2{
3    private readonly SemaphoreSlim _concurrencyLimiter;
4    private readonly SlidingWindowLimiter _rateLimiter;
5

6    public SmartThrottler(int maxConcurrent, int maxPerMinute)
7    {
8        _concurrencyLimiter = new SemaphoreSlim(maxConcurrent);
9        _rateLimiter = new SlidingWindowLimiter(maxPerMinute, TimeSpan.FromMinutes(1));
10    }
11

12    public async Task<T> ExecuteAsync<T>(Func<Task<T>> operation)
13    {
14        // Wait for rate limit
15        while (!_rateLimiter.TryAcquire())
16        {
17            await Task.Delay(100);
18        }
19

20        // Wait for concurrency slot
21        await _concurrencyLimiter.WaitAsync();
22        try
23        {
24            return await operation();
25        }
26        finally
27        {
28            _concurrencyLimiter.Release();
29        }
30    }
31}

Built-in Rate Limiting (.NET 7+)

.NET 7 introduces System.Threading.RateLimiting:

csharp
1using System.Threading.RateLimiting;
2

3// Token bucket limiter
4var limiter = new TokenBucketRateLimiter(new TokenBucketRateLimiterOptions
5{
6    TokenLimit = 10,
7    QueueLimit = 100,
8    ReplenishmentPeriod = TimeSpan.FromSeconds(1),
9    TokensPerPeriod = 2
10});
11

12// Usage
13using var lease = await limiter.AcquireAsync(permitCount: 1);
14if (lease.IsAcquired)
15{
16    await DoWorkAsync();
17}

Key Takeaways

  • Use SemaphoreSlim for simple concurrency limiting
  • Use Parallel.ForEachAsync with MaxDegreeOfParallelism for easy throttling
  • Token bucket allows bursts up to max, then steady rate
  • Sliding window enforces strict rate over time period
  • Combine concurrency + rate limiting for robust control
  • .NET 7+ has built-in rate limiting APIs
  • Batch processing reduces overhead for bulk operations