The Timeout Background Job

Job Implementation

// HILTimeoutJob.cs
public class HILTimeoutJob : IHangfireJob
{
    private readonly ISuspendedExecutionRepository _repo;
    private readonly IHILTimeoutDispatcher         _dispatcher;
    private readonly HILTimeoutOptions             _options;

    public async Task ExecuteAsync(CancellationToken ct)
    {
        var expiredBatch = await _repo.GetExpiredAsync(
            limit    : _options.BatchSize,
            asOfUtc  : DateTimeOffset.UtcNow,
            ct       : ct);

        foreach (var suspension in expiredBatch)
        {
            try
            {
                await _dispatcher.DispatchAsync(suspension, ct);
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Failed to process timeout for {ExecutionResId}",
                    suspension.ExecutionResId);
                // Continue processing remaining items — one failure doesn't stop the batch
            }
        }
    }
}

Expired Task Query

-- Query run by the job
SELECT TOP (@BatchSize) *
FROM   Process_SuspendedExecutions
WHERE  ExpiresAt < GETUTCDATE()
  AND  Status = 0  -- Pending
  AND  ResumedAt IS NULL
ORDER BY ExpiresAt ASC;  -- oldest first

IHILTimeoutDispatcher

public interface IHILTimeoutDispatcher
{
    Task DispatchAsync(SuspendedExecution suspension, CancellationToken ct);
}

// Implementation routes to the appropriate service based on TimeoutBehavior
public class HILTimeoutDispatcher : IHILTimeoutDispatcher
{
    public async Task DispatchAsync(SuspendedExecution s, CancellationToken ct)
    {
        switch (s.TimeoutBehavior)
        {
            case "Escalate":
                await _escalationService.EscalateAsync(s, ct);
                break;
            case "AutoApprove":
            case "AutoReject":
                await _autoDecisionService.DecideAsync(s, ct);
                break;
            default: // Fail
                await _failureService.FailAsync(s, ct);
                break;
        }

        // Publish HILExpired event regardless of behavior
        await _eventBus.PublishAsync(new HILExpiredEvent
        {
            ExecutionResId = s.ExecutionResId,
            ExecutionId    = s.ExecutionId,
            TenantId       = s.TenantId,
            NodeId         = s.SuspendedNodeId,
            ExpiredAt      = DateTimeOffset.UtcNow
        }, ct);
    }
}

Idempotency

The job is idempotent — if the same expired suspension is processed twice (e.g., due to a crash mid-batch), the second processing attempt detects the Status is no longer Pending and skips it. This prevents double-timeout from triggering two escalations.