[backend/federation] Limit the maximum number of replies to backfill per post

This commit is contained in:
Kopper 2024-09-16 23:15:14 +03:00 committed by Iceshrimp development
parent 7fd1f81d21
commit 91aab7c8d6
2 changed files with 4 additions and 6 deletions

View file

@ -78,7 +78,7 @@ public class ObjectResolver(
} }
} }
public async IAsyncEnumerable<ASObject> IterateCollection(ASCollection? collection) public async IAsyncEnumerable<ASObject> IterateCollection(ASCollection? collection, int pageLimit = 10)
{ {
if (collection == null) yield break; if (collection == null) yield break;
@ -91,10 +91,6 @@ public class ObjectResolver(
foreach (var item in collection.Items) foreach (var item in collection.Items)
yield return item; yield return item;
// we only limit based on pages here. the consumer of this iterator may
// additionally limit per-item via System.Linq.Async Take()
var pageLimit = 50;
// some remote software (e.g. fedibird) can get in a state where page.next == page.id // some remote software (e.g. fedibird) can get in a state where page.next == page.id
var visitedPages = new HashSet<string>(); var visitedPages = new HashSet<string>();
@ -118,6 +114,8 @@ public class ObjectResolver(
visitedPages.Add(page.Next.Id); visitedPages.Add(page.Next.Id);
} }
// we only limit based on pages here. the consumer of this iterator may
// additionally limit per-item via System.Linq.Async Take()
if (--pageLimit <= 0) if (--pageLimit <= 0)
break; break;

View file

@ -1195,7 +1195,7 @@ public class NoteService(
_recursionLimit = recursionLimit; _recursionLimit = recursionLimit;
await objectResolver.IterateCollection(repliesCollection) await objectResolver.IterateCollection(repliesCollection)
.Take(100) // does this limit make sense? .Take(50)
.Where(p => p.Id != null && (replyBackfillConfig.BackfillEverything || new Uri(p.Id).Authority == collectionId.Authority)) .Where(p => p.Id != null && (replyBackfillConfig.BackfillEverything || new Uri(p.Id).Authority == collectionId.Authority))
.Select(p => ResolveNoteAsync(p.Id!, null, fetchUser, forceRefresh: false)) .Select(p => ResolveNoteAsync(p.Id!, null, fetchUser, forceRefresh: false))
.AwaitAllNoConcurrencyAsync(); .AwaitAllNoConcurrencyAsync();