using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Orleans.Hosting;
using Orleans.Internal;
using Orleans.Runtime;
using Orleans.Runtime.Internal;
using Orleans.Runtime.Messaging;
using Orleans.Runtime.Scheduler;
namespace Orleans.DurableJobs;
///
internal partial class LocalDurableJobManager : SystemTarget, ILocalDurableJobManager, ILocalDurableJobManagerSystemTarget, ILifecycleParticipant
{
internal static readonly GrainType JobManagerGrainType = SystemTargetGrainId.CreateGrainType("job-manager");
private readonly JobShardManager _shardManager;
private readonly ShardExecutor _shardExecutor;
private readonly IInternalGrainFactory _grainFactory;
private readonly IAsyncEnumerable _clusterMembershipUpdates;
private readonly IOverloadDetector _overloadDetector;
private readonly TimeProvider _timeProvider;
private readonly DurableJobsInstruments _durableJobsInstruments;
private readonly ILogger _logger;
private readonly DurableJobsOptions _options;
private readonly CancellationTokenSource _cts = new();
private readonly CancellationTokenSource _requestCts = new();
private readonly CancellationTokenSource _cleanupCts = new();
private Task? _listenForClusterChangesTask;
private Task? _periodicCheckTask;
// Shard tracking state
private readonly ConcurrentDictionary _shardCache = new();
private readonly ConcurrentDictionary _writeableShards = new();
private readonly ConcurrentDictionary _writeableShardKeys = new();
private readonly ConcurrentDictionary _runningShards = new();
private readonly AdmissionGate _admission = new();
private readonly SemaphoreSlim _shardCreationLock = new(1, 1);
private readonly SemaphoreSlim _shardCheckSignal = new(0);
// Slow-start state
private long _startTimestamp;
private int _totalClaimedShards;
private int _stripeCounter;
private const string ShardStripeMetadataKey = "stripe";
public LocalDurableJobManager(
JobShardManager shardManager,
ShardExecutor shardExecutor,
IInternalGrainFactory grainFactory,
IClusterMembershipService clusterMembership,
IOverloadDetector overloadDetector,
[FromKeyedServices(DurableJobTimeProviderNames.DurableJobs)] TimeProvider timeProvider,
IOptions options,
SystemTargetShared shared,
ILogger logger,
DurableJobsInstruments? durableJobsInstruments = null)
: base(JobManagerGrainType, shared)
{
_shardManager = shardManager;
_shardExecutor = shardExecutor;
_grainFactory = grainFactory;
_clusterMembershipUpdates = clusterMembership.MembershipUpdates;
_overloadDetector = overloadDetector;
_timeProvider = timeProvider;
_durableJobsInstruments = durableJobsInstruments ?? DurableJobsInstruments.CreateForDirectConstruction();
_logger = logger;
_options = options.Value;
}
///
public async Task ScheduleJobAsync(ScheduleJobRequest request, CancellationToken cancellationToken)
{
var startTimestamp = _timeProvider.GetTimestamp();
using var activity = DurableJobsDiagnostics.StartScheduleActivity(in request);
request = EnsureScheduleRequestHasTraceContext(request, activity);
using var admission = _admission.TryEnter();
try
{
request.Validate();
if (!admission.Entered)
{
throw new OperationCanceledException("The durable job manager is stopping.", _requestCts.Token);
}
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _requestCts.Token);
var schedulingToken = linkedCts.Token;
LogSchedulingJob(_logger, request.JobName, request.Target, request.DueTime);
var shardKey = GetWritableShardKey(request);
_durableJobsInstruments.OnStripeAssigned(shardKey.Stripe);
while (true)
{
schedulingToken.ThrowIfCancellationRequested();
// Fast path: shard already exists
if (_writeableShards.TryGetValue(shardKey, out var existingShard))
{
DurableJob? job;
try
{
job = await existingShard.TryScheduleJobAsync(request, schedulingToken);
}
catch (ObjectDisposedException ex) when (TryRemoveWritableShard(shardKey, existingShard) || !IsWritableShard(shardKey, existingShard))
{
LogWritableShardDisposedDuringScheduling(_logger, ex, existingShard.Id, shardKey.StartTime, shardKey.Stripe);
continue;
}
if (job is not null)
{
var elapsed = _timeProvider.GetElapsedTime(startTimestamp);
LogJobScheduled(_logger, request.JobName, job.Id, existingShard.Id, request.Target);
_durableJobsInstruments.OnJobScheduled(elapsed);
_durableJobsInstruments.OnScheduleJobCallSucceeded(elapsed);
DurableJobsDiagnostics.SetScheduledJobTags(activity, job);
return job;
}
// Shard is full or no longer writable, remove from writable shards and try again
TryRemoveWritableShard(shardKey, existingShard);
continue;
}
// Slow path: need to create shard
await _shardCreationLock.WaitAsync(schedulingToken);
try
{
schedulingToken.ThrowIfCancellationRequested();
// Double-check after acquiring lock
if (_writeableShards.TryGetValue(shardKey, out existingShard))
{
continue;
}
// Create new shard
var endTime = shardKey.StartTime.Add(_options.ShardDuration);
var newShard = await _shardManager.CreateShardAsync(shardKey.StartTime, endTime, CreateShardMetadata(shardKey), schedulingToken);
LogCreatingNewShard(_logger, shardKey.StartTime, shardKey.Stripe);
// Own successful creations even if cancellation raced with the provider returning.
// Shutdown waits for this scheduling call before draining the cache.
_writeableShards[shardKey] = newShard;
_writeableShardKeys[newShard.Id] = shardKey;
_shardCache.TryAdd(newShard.Id, newShard);
TryActivateShard(newShard);
}
finally
{
_shardCreationLock.Release();
}
}
}
catch (OperationCanceledException)
{
_durableJobsInstruments.OnScheduleJobCallCanceled(_timeProvider.GetElapsedTime(startTimestamp));
throw;
}
catch (Exception ex)
{
_durableJobsInstruments.OnScheduleJobCallFailed(_timeProvider.GetElapsedTime(startTimestamp));
DurableJobsDiagnostics.SetError(activity, ex);
throw;
}
}
private static ScheduleJobRequest EnsureScheduleRequestHasTraceContext(ScheduleJobRequest request, Activity? scheduleActivity)
{
if (!string.IsNullOrEmpty(request.TraceParent))
{
return request;
}
var source = scheduleActivity ?? Activity.Current;
var (traceParent, traceState) = DurableJobsDiagnostics.CaptureTraceContext(source);
if (traceParent is null)
{
return request;
}
return new ScheduleJobRequest
{
Target = request.Target,
JobName = request.JobName,
DueTime = request.DueTime,
Metadata = request.Metadata,
TraceParent = traceParent,
TraceState = traceState,
};
}
public void Participate(ISiloLifecycle lifecycle)
{
lifecycle.Subscribe(
nameof(LocalDurableJobManager),
ServiceLifecycleStage.Active,
ct => Start(ct),
ct => Stop(ct));
}
private Task Start(CancellationToken ct)
{
LogStarting(_logger);
_startTimestamp = _timeProvider.GetTimestamp();
using (var _ = new ExecutionContextSuppressor())
{
_listenForClusterChangesTask = Task.Factory.StartNew(
state => ((LocalDurableJobManager)state!).ProcessMembershipUpdates(),
this,
CancellationToken.None,
TaskCreationOptions.None,
WorkItemGroup.TaskScheduler).Unwrap();
_listenForClusterChangesTask.Ignore();
_periodicCheckTask = Task.Factory.StartNew(
state => ((LocalDurableJobManager)state!).PeriodicShardCheck(),
this,
CancellationToken.None,
TaskCreationOptions.None,
WorkItemGroup.TaskScheduler).Unwrap();
_periodicCheckTask.Ignore();
}
LogStarted(_logger);
return Task.CompletedTask;
}
private async Task Stop(CancellationToken ct)
{
var admissionDrained = _admission.CloseAsync();
using var cleanupCancellation = ct.Register(() => CancelForShutdown(_cleanupCts, "cleanup"));
LogStopping(_logger, _runningShards.Count);
CancelForShutdown(_requestCts, "requests");
// Drain requests and activation publication before taking the execution snapshot.
await admissionDrained;
var runningShards = _runningShards.Values.ToArray();
CancelForShutdown(_cts, "execution");
try
{
if (_listenForClusterChangesTask is not null)
{
await _listenForClusterChangesTask.SuppressThrowing();
}
if (_periodicCheckTask is not null)
{
await _periodicCheckTask.SuppressThrowing();
}
}
finally
{
try
{
// Admission is closed; every accepted activation already has its execution task in this snapshot.
await Task.WhenAll(runningShards);
}
finally
{
// Running tasks remove their shards before completing; the remaining shards never activated.
foreach (var shard in _shardCache.Values)
{
TryRemoveWritableShard(shard);
_shardCache.TryRemove(shard.Id, out _);
try
{
await _shardManager.UnregisterShardAsync(shard, ct);
LogUnregisteredShard(_logger, shard.Id);
}
catch (Exception ex)
{
LogErrorUnregisteringShard(_logger, ex, shard.Id);
}
await DisposeShardAsync(shard);
}
}
}
LogStopped(_logger);
}
private void CancelForShutdown(CancellationTokenSource cancellation, string phase)
{
try
{
cancellation.Cancel(throwOnFirstException: false);
}
catch (AggregateException exception)
{
LogErrorCancelingOperations(_logger, exception, phase);
}
}
///
public async Task CancelAsync(DurableJob job, CancellationToken requestCancellationToken)
{
ArgumentNullException.ThrowIfNull(job);
var startTimestamp = _timeProvider.GetTimestamp();
using var admission = _admission.TryEnter();
try
{
if (!admission.Entered)
{
throw new OperationCanceledException("The durable job manager is stopping.", _requestCts.Token);
}
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(requestCancellationToken, _requestCts.Token);
var cancellationToken = linkedCts.Token;
LogRequestingJobCancellation(_logger, job.Id, job.Name, job.ShardId);
for (var routingAttempt = 0; routingAttempt < 2; routingAttempt++)
{
cancellationToken.ThrowIfCancellationRequested();
if (_shardCache.TryGetValue(job.ShardId, out var shard))
{
if (!await _shardManager.IsShardOwnedByLocalSiloAsync(job.ShardId, cancellationToken))
{
var entry = new KeyValuePair(job.ShardId, shard);
((ICollection>)_shardCache).Remove(entry);
}
else
{
var removeResult = await shard.RemoveJobAsync(job.Id, cancellationToken);
var cancellationRequested = removeResult == DurableJobMutationResult.Applied;
if (cancellationRequested)
{
LogJobCancellationRequested(_logger, job.Id, job.Name, job.ShardId);
_durableJobsInstruments.OnJobCancellationRequested();
}
else
{
LogJobCancellationRequestNotRecorded(_logger, job.Id, job.Name, job.ShardId);
}
_durableJobsInstruments.OnCancelJobCall(_timeProvider.GetElapsedTime(startTimestamp), cancellationRequested);
return cancellationRequested;
}
}
var owner = await _shardManager.GetShardOwnerAsync(job.ShardId, cancellationToken);
if (owner is null)
{
break;
}
if (owner.Equals(Silo))
{
continue;
}
var remote = _grainFactory.GetSystemTarget(JobManagerGrainType, owner);
var routed = await remote.CancelAsync(job, cancellationToken);
if (!routed)
{
LogJobCancellationRequestNotRecorded(_logger, job.Id, job.Name, job.ShardId);
}
_durableJobsInstruments.OnCancelJobCall(_timeProvider.GetElapsedTime(startTimestamp), routed);
return routed;
}
LogJobCancellationRequestNotRecorded(_logger, job.Id, job.Name, job.ShardId);
_durableJobsInstruments.OnCancelJobCall(_timeProvider.GetElapsedTime(startTimestamp), cancellationRequested: false);
return false;
}
catch (OperationCanceledException)
{
_durableJobsInstruments.OnCancelJobCallCanceled(_timeProvider.GetElapsedTime(startTimestamp));
throw;
}
catch
{
_durableJobsInstruments.OnCancelJobCallFailed(_timeProvider.GetElapsedTime(startTimestamp));
throw;
}
}
private async Task ProcessMembershipUpdates()
{
await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding | ConfigureAwaitOptions.ContinueOnCapturedContext);
var current = new HashSet();
try
{
await foreach (var membershipSnapshot in _clusterMembershipUpdates.WithCancellation(_cts.Token))
{
try
{
// Get active members
var update = new HashSet(membershipSnapshot.Members.Values
.Where(member => member.Status == SiloStatus.Active)
.Select(member => member.SiloAddress));
// If active list has changed, trigger immediate shard check
if (!current.SetEquals(update))
{
current = update;
_shardCheckSignal.Release();
}
}
catch (Exception exception)
{
LogErrorProcessingClusterMembership(_logger, exception);
}
}
}
catch (OperationCanceledException)
{
if (!_cts.Token.IsCancellationRequested)
{
throw;
}
}
}
private async Task PeriodicShardCheck()
{
await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding | ConfigureAwaitOptions.ContinueOnCapturedContext);
using var timer = new PeriodicTimer(_options.ShardCheckInterval, _timeProvider);
var timerTask = timer.WaitForNextTickAsync(_cts.Token).AsTask();
var signalTask = _shardCheckSignal.WaitAsync(_cts.Token);
while (!_cts.Token.IsCancellationRequested)
{
try
{
var completedTask = await Task.WhenAny(timerTask, signalTask);
await completedTask;
// Retain the other pending wait: it can complete during discovery.
if (ReferenceEquals(completedTask, timerTask))
{
timerTask = timer.WaitForNextTickAsync(_cts.Token).AsTask();
}
else
{
signalTask = _shardCheckSignal.WaitAsync(_cts.Token);
}
await ProcessShardCheckCycleAsync(_cts.Token);
}
catch (OperationCanceledException) when (_cts.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
LogErrorInPeriodicCheck(_logger, ex);
await Task.Delay(TimeSpan.FromSeconds(5), _timeProvider, _cts.Token).SuppressThrowing();
}
}
}
internal async Task ProcessShardCheckCycleAsync(CancellationToken cancellationToken)
{
LogCheckingPendingShards(_logger);
// Clean up old writable shards that have passed their time window.
var now = _timeProvider.GetUtcNow();
foreach (var key in _writeableShards.Keys.ToArray())
{
var shardEndTime = key.StartTime.Add(_options.ShardDuration);
if (shardEndTime < now && _writeableShards.TryRemove(key, out var expiredShard))
{
_writeableShardKeys.TryRemove(expiredShard.Id, out _);
try
{
await expiredShard.MarkAsCompleteAsync(cancellationToken);
}
catch (ObjectDisposedException ex)
{
LogExpiredWritableShardAlreadyDisposed(_logger, ex, expiredShard.Id, key.StartTime, key.Stripe);
}
}
}
await ProcessShardDiscoveryAsync(cancellationToken);
}
private async Task ProcessShardDiscoveryAsync(CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var now = _timeProvider.GetUtcNow();
var maxDueTime = new DateTimeOffset(now.UtcDateTime.AddClamped(_options.ShardLoadLookaheadPeriod));
// Compute the slow-start budget for this sweep.
var budget = ComputeClaimBudget();
var newClaimsThisCycle = 0;
var assignedCount = 0;
await foreach (var shard in _shardManager.DiscoverJobShardsAsync(maxDueTime, budget, cancellationToken))
{
// Take responsibility for yielded resources before observing cancellation.
if (_shardCache.TryAdd(shard.Id, shard))
{
newClaimsThisCycle++;
_totalClaimedShards++;
}
cancellationToken.ThrowIfCancellationRequested();
assignedCount++;
if (!_runningShards.ContainsKey(shard.Id))
{
TryActivateShard(shard);
}
}
if (assignedCount > 0)
{
LogAssignedShards(_logger, assignedCount);
}
else
{
LogNoShardsToAssign(_logger);
}
if (newClaimsThisCycle > 0)
{
LogOrphanedShardsClaimed(_logger, newClaimsThisCycle, _totalClaimedShards);
}
}
///
/// Computes the maximum number of orphaned shards this silo may claim in the current check cycle.
/// Returns when unlimited (ramp-up complete or disabled).
/// Returns 0 when the silo is overloaded, pausing all new claims.
///
private int ComputeClaimBudget()
{
// If overloaded, claim nothing new regardless of ramp-up phase
if (_overloadDetector.IsOverloaded)
{
LogOverloadPausingShardClaims(_logger);
return 0;
}
var elapsed = _timeProvider.GetElapsedTime(_startTimestamp);
var budget = ComputeClaimBudget(
_options.ShardClaimRampUpDuration,
_options.ShardClaimInitialBudget,
_options.ShardClaimMaxBudget,
elapsed,
_totalClaimedShards);
if (budget < int.MaxValue)
{
LogShardClaimBudget(_logger, budget, _options.ShardClaimInitialBudget + (int)(elapsed.TotalMilliseconds / _options.ShardClaimRampUpDuration.TotalMilliseconds * (_options.ShardClaimMaxBudget - _options.ShardClaimInitialBudget)), _totalClaimedShards, elapsed, _options.ShardClaimRampUpDuration);
}
return budget;
}
///
/// Pure computation of shard-claim budget based on ramp-up parameters and elapsed time.
/// Returns when unlimited (ramp-up complete or disabled).
///
internal static int ComputeClaimBudget(
TimeSpan rampUpDuration,
int initialBudget,
int maxBudget,
TimeSpan elapsed,
int totalClaimedShards)
{
// Shard-claim ramp-up disabled
if (rampUpDuration <= TimeSpan.Zero)
{
return int.MaxValue;
}
// After ramp-up period, no limit
if (elapsed >= rampUpDuration)
{
return int.MaxValue;
}
// Linear interpolation: initialBudget at t=0, maxBudget at t=rampUpDuration
var progress = elapsed.TotalMilliseconds / rampUpDuration.TotalMilliseconds;
var totalBudget = initialBudget + (int)(progress * (maxBudget - initialBudget));
var remaining = totalBudget - totalClaimedShards;
return Math.Max(0, remaining);
}
private void TryActivateShard(IJobShard shard)
{
var shardId = shard.Id;
// Only start if not already running
if (_runningShards.ContainsKey(shardId))
{
return;
}
// Only start if it's time to start (within buffer period)
if (!ShouldStartShardNow(shard))
{
LogShardNotReadyYet(_logger, shard.Id, shard.StartTime);
return;
}
using var admission = _admission.TryEnter();
if (!admission.Entered)
{
return;
}
var workItem = new AsyncClosureWorkItem(() => RunShardWithCleanupAsync(shard), this);
if (!_runningShards.TryAdd(shardId, workItem.Task))
{
return;
}
LogStartingShard(_logger, shardId, shard.StartTime, shard.EndTime);
using var _ = new ExecutionContextSuppressor();
WorkItemGroup.QueueWorkItem(workItem);
}
private async Task RunShardWithCleanupAsync(IJobShard shard)
{
try
{
try
{
await _shardExecutor.RunShardAsync(shard, _cts.Token);
}
catch (OperationCanceledException) when (_cts.IsCancellationRequested)
{
// Execution cancellation leaves ownership cleanup with this runner.
}
// Cleanup is canceled by the shutdown deadline, independently of execution.
try
{
await _shardManager.UnregisterShardAsync(shard, _cleanupCts.Token);
LogUnregisteredShard(_logger, shard.Id);
}
catch (Exception ex)
{
LogErrorUnregisteringShard(_logger, ex, shard.Id);
}
}
finally
{
// Clean up tracking and dispose the shard
TryRemoveWritableShard(shard);
_shardCache.TryRemove(shard.Id, out _);
await DisposeShardAsync(shard);
_runningShards.TryRemove(shard.Id, out _);
}
}
private async ValueTask DisposeShardAsync(IJobShard shard)
{
try
{
await shard.DisposeAsync();
}
catch (Exception ex)
{
LogErrorDisposingShard(_logger, ex, shard.Id);
}
}
private bool ShouldStartShardNow(IJobShard shard)
{
var activationTime = shard.StartTime.Subtract(_options.ShardActivationBufferPeriod);
return _timeProvider.GetUtcNow() >= activationTime;
}
private bool IsWritableShard(WritableShardKey shardKey, IJobShard shard)
=> _writeableShards.TryGetValue(shardKey, out var existingShard) && ReferenceEquals(existingShard, shard);
private bool TryRemoveWritableShard(WritableShardKey shardKey, IJobShard shard)
{
var entry = new KeyValuePair(shardKey, shard);
var removed = ((ICollection>)_writeableShards).Remove(entry);
if (removed)
{
var keyEntry = new KeyValuePair(shard.Id, shardKey);
((ICollection>)_writeableShardKeys).Remove(keyEntry);
}
return removed;
}
private bool TryRemoveWritableShard(IJobShard shard)
{
if (_writeableShardKeys.TryGetValue(shard.Id, out var shardKey))
{
return TryRemoveWritableShard(shardKey, shard);
}
return false;
}
internal sealed class TestAccessor(LocalDurableJobManager manager)
{
public void SignalShardCheck() => manager._shardCheckSignal.Release();
public Task ProcessShardCheckCycleAsync(CancellationToken cancellationToken) => manager.ProcessShardCheckCycleAsync(cancellationToken);
public void AddWritableShard(DateTimeOffset shardKey, IJobShard shard, int stripe = 0)
{
var key = new WritableShardKey(shardKey, stripe);
manager._writeableShards[key] = shard;
manager._writeableShardKeys[shard.Id] = key;
manager._shardCache.TryAdd(shard.Id, shard);
}
public bool HasWritableShard(DateTimeOffset shardKey, int stripe = 0) => manager._writeableShards.ContainsKey(new WritableShardKey(shardKey, stripe));
public bool TryGetWritableShard(DateTimeOffset shardKey, out IJobShard? shard, int stripe = 0) => manager._writeableShards.TryGetValue(new WritableShardKey(shardKey, stripe), out shard);
public int GetWritableShardStripe(ScheduleJobRequest request) => manager.GetWritableShardKey(request).Stripe;
public int WritableShardCount => manager._writeableShards.Count;
public int WritableShardKeyCount => manager._writeableShardKeys.Count;
public int CachedShardCount => manager._shardCache.Count;
public void TryActivateShard(IJobShard shard) => manager.TryActivateShard(shard);
public bool TryGetRunningShardTask(string shardId, out Task? task) => manager._runningShards.TryGetValue(shardId, out task);
public bool HasCachedShard(string shardId) => manager._shardCache.ContainsKey(shardId);
}
private WritableShardKey GetWritableShardKey(ScheduleJobRequest request)
=> new(GetShardStartTime(request.DueTime), GetShardStripe());
private IDictionary CreateShardMetadata(WritableShardKey shardKey)
{
var result = new Dictionary(StringComparer.Ordinal);
if (_options.ShardStripeCount > 1)
{
result[ShardStripeMetadataKey] = shardKey.Stripe.ToString(CultureInfo.InvariantCulture);
}
return result;
}
private DateTimeOffset GetShardStartTime(DateTimeOffset scheduledTime)
{
var shardDurationTicks = _options.ShardDuration.Ticks;
var epochTicks = scheduledTime.UtcTicks;
var bucketTicks = (epochTicks / shardDurationTicks) * shardDurationTicks;
return new DateTimeOffset(bucketTicks, TimeSpan.Zero);
}
private int GetShardStripe()
{
if (_options.ShardStripeCount <= 1)
{
return 0;
}
// Round-robin assignment. Stripe selection is a write-side fan-out knob only:
// the persisted job location is the shard id, not (StartTime, Stripe), so consistency
// across calls/silos is not required and round-robin distributes evenly under any input skew.
var next = (uint)Interlocked.Increment(ref _stripeCounter);
return (int)(next % (uint)_options.ShardStripeCount);
}
private readonly record struct WritableShardKey(DateTimeOffset StartTime, int Stripe);
}