diff --git a/Iceshrimp.Backend/Core/Configuration/Config.cs b/Iceshrimp.Backend/Core/Configuration/Config.cs index f53b2eac..d7cef60f 100644 --- a/Iceshrimp.Backend/Core/Configuration/Config.cs +++ b/Iceshrimp.Backend/Core/Configuration/Config.cs @@ -20,6 +20,7 @@ public sealed class Config public required StorageSection Storage { get; init; } = new(); public required PerformanceSection Performance { get; init; } = new(); public required QueueSection Queue { get; init; } = new(); + public required BackfillSection Backfill { get; init; } = new(); public sealed class InstanceSection { @@ -97,34 +98,7 @@ public sealed class Config public string? MediaRetention { get => MediaRetentionTimeSpan?.ToString(); - init - { - if (value == null || string.IsNullOrWhiteSpace(value) || value.Trim() == "0") - { - MediaRetentionTimeSpan = null; - return; - } - - if (value.Trim() == "-1") - { - MediaRetentionTimeSpan = TimeSpan.MaxValue; - return; - } - - if (value.Length < 2 || !int.TryParse(value[..^1].Trim(), out var num)) - throw new Exception("Invalid media retention time"); - - var suffix = value[^1]; - - MediaRetentionTimeSpan = suffix switch - { - 'd' => TimeSpan.FromDays(num), - 'w' => TimeSpan.FromDays(num * 7), - 'm' => TimeSpan.FromDays(num * 30), - 'y' => TimeSpan.FromDays(num * 365), - _ => throw new Exception("Unsupported suffix, use one of: [d]ays, [w]eeks, [m]onths, [y]ears") - }; - } + init => MediaRetentionTimeSpan = ParseNaturalDuration(value, "media retention time"); } public string? MaxUploadSize @@ -369,4 +343,63 @@ public sealed class Config [Range(0, int.MaxValue)] public int Completed { get; init; } = 100; [Range(0, int.MaxValue)] public int Failed { get; init; } = 10; } + + public sealed class BackfillSection + { + public BackfillRepliesSection Replies { get; init; } = new(); + } + + public sealed class BackfillRepliesSection + { + public bool Enabled { get; init; } = false; + + public string? NewNoteThreshold + { + get => NewNoteThresholdTimeSpan.ToString(); + init => NewNoteThresholdTimeSpan = + ParseNaturalDuration(value, "new note threshold") ?? TimeSpan.FromHours(3); + } + + public string? NewNoteDelay + { + get => NewNoteDelayTimeSpan.ToString(); + init => NewNoteDelayTimeSpan = + ParseNaturalDuration(value, "new note delay") ?? TimeSpan.FromHours(3); + } + + public string? RefreshOnRenoteAfter + { + get => RefreshOnRenoteAfterTimeSpan.ToString(); + init => RefreshOnRenoteAfterTimeSpan = + ParseNaturalDuration(value, "refresh renote after duration") ?? TimeSpan.FromDays(7); + } + + public TimeSpan NewNoteThresholdTimeSpan = TimeSpan.FromHours(3); + public TimeSpan NewNoteDelayTimeSpan = TimeSpan.FromHours(3); + public TimeSpan RefreshOnRenoteAfterTimeSpan = TimeSpan.FromDays(7); + } + + private static TimeSpan? ParseNaturalDuration(string? value, string name) + { + if (value == null || string.IsNullOrWhiteSpace(value) || value.Trim() == "0") + return null; + + if (value.Trim() == "-1") + return TimeSpan.MaxValue; + + if (value.Length < 2 || !int.TryParse(value[..^1].Trim(), out var num)) + throw new Exception($"Invalid {name}"); + + var suffix = value[^1]; + + return suffix switch + { + 'h' => TimeSpan.FromHours(num), + 'd' => TimeSpan.FromDays(num), + 'w' => TimeSpan.FromDays(num * 7), + 'm' => TimeSpan.FromDays(num * 30), + 'y' => TimeSpan.FromDays(num * 365), + _ => throw new Exception("Unsupported suffix, use one of: [h]ours, [d]ays, [w]eeks, [m]onths, [y]ears") + }; + } } \ No newline at end of file diff --git a/Iceshrimp.Backend/Core/Extensions/ServiceExtensions.cs b/Iceshrimp.Backend/Core/Extensions/ServiceExtensions.cs index 0b38cf72..eeaf7b6f 100644 --- a/Iceshrimp.Backend/Core/Extensions/ServiceExtensions.cs +++ b/Iceshrimp.Backend/Core/Extensions/ServiceExtensions.cs @@ -135,6 +135,8 @@ public static class ServiceExtensions .ConfigureWithValidation(configuration, "Security") .ConfigureWithValidation(configuration, "Performance") .ConfigureWithValidation(configuration, "Performance:QueueConcurrency") + .ConfigureWithValidation(configuration, "Backfill") + .ConfigureWithValidation(configuration, "Backfill:Replies") .ConfigureWithValidation(configuration, "Queue") .ConfigureWithValidation(configuration, "Queue:JobRetention") .ConfigureWithValidation(configuration, "Database") diff --git a/Iceshrimp.Backend/Core/Services/NoteService.cs b/Iceshrimp.Backend/Core/Services/NoteService.cs index 5c2bb6c6..ff6c5945 100644 --- a/Iceshrimp.Backend/Core/Services/NoteService.cs +++ b/Iceshrimp.Backend/Core/Services/NoteService.cs @@ -33,6 +33,7 @@ public class NoteService( DatabaseContext db, ActivityPub.UserResolver userResolver, IOptionsSnapshot config, + IOptionsSnapshot backfillConfig, ActivityPub.ActivityFetcherService fetchSvc, ActivityPub.ActivityDeliverService deliverSvc, ActivityPub.NoteRenderer noteRenderer, @@ -314,14 +315,18 @@ public class NoteService( }); } + var replyBackfillConfig = backfillConfig.Value.Replies; + // If we're renoting a note we backfilled replies to some time ago (and know how to backfill), enqueue a backfill. - if (renote != null && renote.RepliesCollection != null && renote.RepliesFetchedAt != null && renote.RepliesFetchedAt?.AddDays(7) <= DateTime.UtcNow) + if (replyBackfillConfig.Enabled && + renote?.RepliesCollection != null && + renote.RepliesFetchedAt?.Add(replyBackfillConfig.RefreshOnRenoteAfterTimeSpan) <= DateTime.UtcNow) { logger.LogDebug("Enqueueing reply collection fetch for renote {renoteId}", renote.Id); await queueSvc.BackfillQueue.EnqueueAsync(new BackfillJobData { - NoteId = renote.Id, - RecursionLimit = _recursionLimit, + NoteId = renote.Id, + RecursionLimit = _recursionLimit, AuthenticatedUserId = null, // TODO: for private replies }); } @@ -337,23 +342,27 @@ public class NoteService( .ExecuteUpdateAsync(p => p.SetProperty(i => i.NotesCount, i => i.NotesCount + 1)); }); - if (note.RepliesCollection != null) + if (replyBackfillConfig.Enabled && note.RepliesCollection != null) { var jobData = new BackfillJobData { - NoteId = note.Id, - RecursionLimit = _recursionLimit, + NoteId = note.Id, + RecursionLimit = _recursionLimit, AuthenticatedUserId = null, // TODO: for private replies - Collection = JsonConvert.SerializeObject(asNote?.Replies as ASObject, LdHelpers.JsonSerializerSettings) + Collection = asNote?.Replies?.IsUnresolved == false + ? JsonConvert.SerializeObject(asNote.Replies, LdHelpers.JsonSerializerSettings) + : null }; logger.LogDebug("Enqueueing reply collection fetch for note {noteId}", note.Id); // Delay reply backfilling for brand new notes to allow them time to collect replies. - if (note.CreatedAt.AddHours(3) <= DateTime.UtcNow) + if (note.CreatedAt + replyBackfillConfig.NewNoteThresholdTimeSpan <= DateTime.UtcNow) await queueSvc.BackfillQueue.EnqueueAsync(jobData); else - await queueSvc.BackfillQueue.ScheduleAsync(jobData, DateTime.UtcNow.AddHours(3)); + await queueSvc.BackfillQueue.ScheduleAsync(jobData, + DateTime.UtcNow + + replyBackfillConfig.NewNoteDelayTimeSpan); } return note; @@ -1170,10 +1179,13 @@ public class NoteService( return await ResolveNoteAsync(note.Id, note); } - public async Task BackfillRepliesAsync(Note note, User? fetchUser, ASCollection? repliesCollection, int recursionLimit) + public async Task BackfillRepliesAsync( + Note note, User? fetchUser, ASCollection? repliesCollection, int recursionLimit + ) { if (note.RepliesCollection == null) return; - note.RepliesFetchedAt = DateTime.UtcNow; // should get committed alongside the resolved reply objects + await db.Notes.Where(p => p == note) + .ExecuteUpdateAsync(p => p.SetProperty(i => i.RepliesFetchedAt, _ => DateTime.UtcNow)); repliesCollection ??= new ASCollection(note.RepliesCollection); @@ -1367,10 +1379,11 @@ public class NoteService( // ReSharper disable once EntityFramework.UnsupportedServerSideFunctionCall var followingUser = await db.Users.FirstOrDefaultAsync(p => p.IsFollowing(user)); - var notes = await objectResolver.IterateCollection(collection).Take(10) - .Where(p => p.Id != null) - .Select(p => ResolveNoteAsync(p.Id!, null, followingUser, true)) - .AwaitAllNoConcurrencyAsync(); + var notes = await objectResolver.IterateCollection(collection) + .Take(10) + .Where(p => p.Id != null) + .Select(p => ResolveNoteAsync(p.Id!, null, followingUser, true)) + .AwaitAllNoConcurrencyAsync(); var previousPins = await db.Users.Where(p => p.Id == user.Id) .Select(p => p.PinnedNotes.Select(i => i.Id)) diff --git a/Iceshrimp.Backend/Core/Services/QueueService.cs b/Iceshrimp.Backend/Core/Services/QueueService.cs index 1a59bd27..ed58158d 100644 --- a/Iceshrimp.Backend/Core/Services/QueueService.cs +++ b/Iceshrimp.Backend/Core/Services/QueueService.cs @@ -16,6 +16,7 @@ public class QueueService( IServiceScopeFactory scopeFactory, ILogger logger, IOptions queueConcurrency, + IOptions backfill, IHostApplicationLifetime lifetime ) : BackgroundService { @@ -30,7 +31,10 @@ public class QueueService( protected override async Task ExecuteAsync(CancellationToken token) { - _queues.AddRange([InboxQueue, PreDeliverQueue, DeliverQueue, BackgroundTaskQueue, BackfillQueue]); + _queues.AddRange([InboxQueue, PreDeliverQueue, DeliverQueue, BackgroundTaskQueue]); + + if (backfill.Value.Replies.Enabled) + _queues.Add(BackfillQueue); var tokenSource = new CancellationTokenSource(); var queueTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token, lifetime.ApplicationStopping); diff --git a/Iceshrimp.Backend/configuration.ini b/Iceshrimp.Backend/configuration.ini index 04b15a1e..a8827fd5 100644 --- a/Iceshrimp.Backend/configuration.ini +++ b/Iceshrimp.Backend/configuration.ini @@ -81,6 +81,22 @@ Inbox = 4 Deliver = 20 PreDeliver = 4 BackgroundTask = 4 +Backfill = 4 + +[Backfill:Replies] +;; Enables backfilling of replies. This is disabled by default as it may have a significant performance impact. +;; This is an experimental feature that hasn't had too much time to bake, so only enable if you're open for instability. +;; Note that replies can only be fetched from remote instances that expose a replies collection. +Enabled = false + +;; Notes newer than this threshold will have reply backfilling delayed, to allow them time to accumulate replies. +NewNoteThreshold = 3h + +;; The duration backfilling of new notes will be delayed by. +NewNoteDelay = 3h + +;; Renoting a note that was backfilled before this threshold will attempt to fetch any new replies that may have been created since the last backfill. +RefreshOnRenoteAfter = 7d ;; How many completed & failed jobs to keep around, per queue. ;; Excess is trimmed every 15 minutes, oldest jobs first.