notfire/Iceshrimp.NET
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[backend/configuration] Add configuration for backfill of note replies

Browse source
This commit is contained in:
Kopper 2024-09-14 22:08:09 +03:00 • committed by Laura Hausmann
parent 4081aeb036
commit 83e830b5df
No known key found for this signature in database
GPG key ID: D044E84C5BE01605
5 changed files with 112 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

89
Iceshrimp.Backend/Core/Configuration/Config.cs
View file

@ -20,6 +20,7 @@ public sealed class Config
public required StorageSection Storage { get; init; } = new();
public required PerformanceSection Performance { get; init; } = new();
public required QueueSection Queue { get; init; } = new();
public required BackfillSection Backfill { get; init; } = new();
public sealed class InstanceSection
{
@ -97,34 +98,7 @@ public sealed class Config
public string? MediaRetention
{
get => MediaRetentionTimeSpan?.ToString();
init
{
if (value == null || string.IsNullOrWhiteSpace(value) || value.Trim() == "0")
{
MediaRetentionTimeSpan = null;
return;
}
if (value.Trim() == "-1")
{
MediaRetentionTimeSpan = TimeSpan.MaxValue;
return;
}
if (value.Length < 2 || !int.TryParse(value[..^1].Trim(), out var num))
throw new Exception("Invalid media retention time");
var suffix = value[^1];
MediaRetentionTimeSpan = suffix switch
{
'd' => TimeSpan.FromDays(num),
'w' => TimeSpan.FromDays(num * 7),
'm' => TimeSpan.FromDays(num * 30),
'y' => TimeSpan.FromDays(num * 365),
_ => throw new Exception("Unsupported suffix, use one of: [d]ays, [w]eeks, [m]onths, [y]ears")
};
}
init => MediaRetentionTimeSpan = ParseNaturalDuration(value, "media retention time");
}
public string? MaxUploadSize
@ -369,4 +343,63 @@ public sealed class Config
[Range(0, int.MaxValue)] public int Completed { get; init; } = 100;
[Range(0, int.MaxValue)] public int Failed { get; init; } = 10;
}
public sealed class BackfillSection
{
public BackfillRepliesSection Replies { get; init; } = new();
}
public sealed class BackfillRepliesSection
{
public bool Enabled { get; init; } = false;
public string? NewNoteThreshold
{
get => NewNoteThresholdTimeSpan.ToString();
init => NewNoteThresholdTimeSpan =
ParseNaturalDuration(value, "new note threshold") ?? TimeSpan.FromHours(3);
}
public string? NewNoteDelay
{
get => NewNoteDelayTimeSpan.ToString();
init => NewNoteDelayTimeSpan =
ParseNaturalDuration(value, "new note delay") ?? TimeSpan.FromHours(3);
}
public string? RefreshOnRenoteAfter
{
get => RefreshOnRenoteAfterTimeSpan.ToString();
init => RefreshOnRenoteAfterTimeSpan =
ParseNaturalDuration(value, "refresh renote after duration") ?? TimeSpan.FromDays(7);
}
public TimeSpan NewNoteThresholdTimeSpan = TimeSpan.FromHours(3);
public TimeSpan NewNoteDelayTimeSpan = TimeSpan.FromHours(3);
public TimeSpan RefreshOnRenoteAfterTimeSpan = TimeSpan.FromDays(7);
}
private static TimeSpan? ParseNaturalDuration(string? value, string name)
{
if (value == null || string.IsNullOrWhiteSpace(value) || value.Trim() == "0")
return null;
if (value.Trim() == "-1")
return TimeSpan.MaxValue;
if (value.Length < 2 || !int.TryParse(value[..^1].Trim(), out var num))
throw new Exception($"Invalid {name}");
var suffix = value[^1];
return suffix switch
{
'h' => TimeSpan.FromHours(num),
'd' => TimeSpan.FromDays(num),
'w' => TimeSpan.FromDays(num * 7),
'm' => TimeSpan.FromDays(num * 30),
'y' => TimeSpan.FromDays(num * 365),
_ => throw new Exception("Unsupported suffix, use one of: [h]ours, [d]ays, [w]eeks, [m]onths, [y]ears")
};
}
}

2
Iceshrimp.Backend/Core/Extensions/ServiceExtensions.cs
View file

@ -135,6 +135,8 @@ public static class ServiceExtensions
.ConfigureWithValidation<Config.SecuritySection>(configuration, "Security")
.ConfigureWithValidation<Config.PerformanceSection>(configuration, "Performance")
.ConfigureWithValidation<Config.QueueConcurrencySection>(configuration, "Performance:QueueConcurrency")
.ConfigureWithValidation<Config.BackfillSection>(configuration, "Backfill")
.ConfigureWithValidation<Config.BackfillRepliesSection>(configuration, "Backfill:Replies")
.ConfigureWithValidation<Config.QueueSection>(configuration, "Queue")
.ConfigureWithValidation<Config.JobRetentionSection>(configuration, "Queue:JobRetention")
.ConfigureWithValidation<Config.DatabaseSection>(configuration, "Database")

29
Iceshrimp.Backend/Core/Services/NoteService.cs
View file

@ -33,6 +33,7 @@ public class NoteService(
DatabaseContext db,
ActivityPub.UserResolver userResolver,
IOptionsSnapshot<Config.InstanceSection> config,
IOptionsSnapshot<Config.BackfillSection> backfillConfig,
ActivityPub.ActivityFetcherService fetchSvc,
ActivityPub.ActivityDeliverService deliverSvc,
ActivityPub.NoteRenderer noteRenderer,
@ -314,8 +315,12 @@ public class NoteService(
});
}
var replyBackfillConfig = backfillConfig.Value.Replies;
// If we're renoting a note we backfilled replies to some time ago (and know how to backfill), enqueue a backfill.
if (renote != null && renote.RepliesCollection != null && renote.RepliesFetchedAt != null && renote.RepliesFetchedAt?.AddDays(7) <= DateTime.UtcNow)
if (replyBackfillConfig.Enabled &&
renote?.RepliesCollection != null &&
renote.RepliesFetchedAt?.Add(replyBackfillConfig.RefreshOnRenoteAfterTimeSpan) <= DateTime.UtcNow)
{
logger.LogDebug("Enqueueing reply collection fetch for renote {renoteId}", renote.Id);
await queueSvc.BackfillQueue.EnqueueAsync(new BackfillJobData
@ -337,23 +342,27 @@ public class NoteService(
.ExecuteUpdateAsync(p => p.SetProperty(i => i.NotesCount, i => i.NotesCount + 1));
});
if (note.RepliesCollection != null)
if (replyBackfillConfig.Enabled && note.RepliesCollection != null)
{
var jobData = new BackfillJobData
{
NoteId = note.Id,
RecursionLimit = _recursionLimit,
AuthenticatedUserId = null, // TODO: for private replies
Collection = JsonConvert.SerializeObject(asNote?.Replies as ASObject, LdHelpers.JsonSerializerSettings)
Collection = asNote?.Replies?.IsUnresolved == false
? JsonConvert.SerializeObject(asNote.Replies, LdHelpers.JsonSerializerSettings)
: null
};
logger.LogDebug("Enqueueing reply collection fetch for note {noteId}", note.Id);
// Delay reply backfilling for brand new notes to allow them time to collect replies.
if (note.CreatedAt.AddHours(3) <= DateTime.UtcNow)
if (note.CreatedAt + replyBackfillConfig.NewNoteThresholdTimeSpan <= DateTime.UtcNow)
await queueSvc.BackfillQueue.EnqueueAsync(jobData);
else
await queueSvc.BackfillQueue.ScheduleAsync(jobData, DateTime.UtcNow.AddHours(3));
await queueSvc.BackfillQueue.ScheduleAsync(jobData,
DateTime.UtcNow +
replyBackfillConfig.NewNoteDelayTimeSpan);
}
return note;
@ -1170,10 +1179,13 @@ public class NoteService(
return await ResolveNoteAsync(note.Id, note);
}
public async Task BackfillRepliesAsync(Note note, User? fetchUser, ASCollection? repliesCollection, int recursionLimit)
public async Task BackfillRepliesAsync(
Note note, User? fetchUser, ASCollection? repliesCollection, int recursionLimit
)
{
if (note.RepliesCollection == null) return;
note.RepliesFetchedAt = DateTime.UtcNow; // should get committed alongside the resolved reply objects
await db.Notes.Where(p => p == note)
.ExecuteUpdateAsync(p => p.SetProperty(i => i.RepliesFetchedAt, _ => DateTime.UtcNow));
repliesCollection ??= new ASCollection(note.RepliesCollection);
@ -1367,7 +1379,8 @@ public class NoteService(
// ReSharper disable once EntityFramework.UnsupportedServerSideFunctionCall
var followingUser = await db.Users.FirstOrDefaultAsync(p => p.IsFollowing(user));
var notes = await objectResolver.IterateCollection(collection).Take(10)
var notes = await objectResolver.IterateCollection(collection)
.Take(10)
.Where(p => p.Id != null)
.Select(p => ResolveNoteAsync(p.Id!, null, followingUser, true))
.AwaitAllNoConcurrencyAsync();

6
Iceshrimp.Backend/Core/Services/QueueService.cs
View file

@ -16,6 +16,7 @@ public class QueueService(
IServiceScopeFactory scopeFactory,
ILogger<QueueService> logger,
IOptions<Config.QueueConcurrencySection> queueConcurrency,
IOptions<Config.BackfillSection> backfill,
IHostApplicationLifetime lifetime
) : BackgroundService
{
@ -30,7 +31,10 @@ public class QueueService(
protected override async Task ExecuteAsync(CancellationToken token)
{
_queues.AddRange([InboxQueue, PreDeliverQueue, DeliverQueue, BackgroundTaskQueue, BackfillQueue]);
_queues.AddRange([InboxQueue, PreDeliverQueue, DeliverQueue, BackgroundTaskQueue]);
if (backfill.Value.Replies.Enabled)
_queues.Add(BackfillQueue);
var tokenSource = new CancellationTokenSource();
var queueTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token, lifetime.ApplicationStopping);

16
Iceshrimp.Backend/configuration.ini
View file

@ -81,6 +81,22 @@ Inbox = 4
Deliver = 20
PreDeliver = 4
BackgroundTask = 4
Backfill = 4
[Backfill:Replies]
;; Enables backfilling of replies. This is disabled by default as it may have a significant performance impact.
;; This is an experimental feature that hasn't had too much time to bake, so only enable if you're open for instability.
;; Note that replies can only be fetched from remote instances that expose a replies collection.
Enabled = false
;; Notes newer than this threshold will have reply backfilling delayed, to allow them time to accumulate replies.
NewNoteThreshold = 3h
;; The duration backfilling of new notes will be delayed by.
NewNoteDelay = 3h
;; Renoting a note that was backfilled before this threshold will attempt to fetch any new replies that may have been created since the last backfill.
RefreshOnRenoteAfter = 7d
;; How many completed & failed jobs to keep around, per queue.
;; Excess is trimmed every 15 minutes, oldest jobs first.