Data Retention

Episodic Memory Retention

Episodic memory is the only memory type with automatic time-based expiry. The retention lifecycle has three stages:

Retention Background Job

public class EpisodicRetentionJob : BackgroundService
{
    protected override async Task ExecuteAsync(CancellationToken ct)
    {
        while (!ct.IsCancellationRequested)
        {
            await RunRetentionPassAsync(ct);
            await Task.Delay(TimeSpan.FromHours(1), ct);  // Run hourly
        }
    }

    private async Task RunRetentionPassAsync(CancellationToken ct)
    {
        using var scope = _services.CreateScope();
        var db = scope.ServiceProvider.GetRequiredService<OctopusDbContext>();

        // Stage 1: Soft-delete expired episodes
        // (uses IgnoreQueryFilters to operate across all tenants)
        var expiredEpisodes = await db.Episodes
            .IgnoreQueryFilters()
            .Where(e => !e.IsDeleted
                && e.EndedAt < DateTime.UtcNow.AddDays(-e.RetentionDays))
            .ToListAsync(ct);

        foreach (var ep in expiredEpisodes)
        {
            ep.IsDeleted = true;
            ep.DeletedAt = DateTime.UtcNow;
        }

        // Stage 2: Hard-purge grace-period-elapsed soft-deleted episodes
        var toPurge = await db.Episodes
            .IgnoreQueryFilters()
            .Where(e => e.IsDeleted
                && e.DeletedAt < DateTime.UtcNow.AddDays(-7))  // 7-day grace
            .ToListAsync(ct);

        db.Episodes.RemoveRange(toPurge);
        await db.SaveChangesAsync(ct);
    }
}

User Data Erasure (GDPR Right to Erasure)

The erasure endpoint immediately hard-deletes all episodic data for a specific user — bypassing TTL and the soft-delete grace period:

// DELETE /api/octopus/users/{userId}/memory?agentId={agentId}
public async Task EraseUserDataAsync(string userId, Guid agentId, Guid tenantId, CancellationToken ct)
{
    using var tx = await _db.Database.BeginTransactionAsync(ct);

    // Hard delete episode messages
    var messageIds = await _db.EpisodeMessages
        .IgnoreQueryFilters()
        .Where(m => m.Episode.UserId == userId
                 && m.Episode.AgentId == agentId
                 && m.TenantId == tenantId)
        .Select(m => m.MessageId)
        .ToListAsync(ct);

    await _db.Database.ExecuteSqlRawAsync(
        "DELETE FROM Octopus_EpisodeMessages WHERE MessageId IN ({0})",
        string.Join(",", messageIds.Select(id => $"'{id}'")));

    // Hard delete episodes
    await _db.Database.ExecuteSqlRawAsync(
        "DELETE FROM Octopus_Episodes WHERE UserId = {0} AND AgentId = {1} AND TenantId = {2}",
        userId, agentId, tenantId);

    await tx.CommitAsync(ct);
}

Retention Policy Reference