notfire/Iceshrimp.NET
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[backend/core] Improve note table query performance by aggregating block/mute checks (ISH-206)

Browse source 
This also implements checking for blocks & mutes users in the mentions field, implementing ISH-225.
This commit is contained in:
Laura Hausmann 2024-04-23 20:57:18 +02:00
parent 1a4dd75301
commit 5fca0620cf
No known key found for this signature in database
GPG key ID: D044E84C5BE01605
12 changed files with 146 additions and 123 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Iceshrimp.Backend/Controllers/Mastodon/AccountController.cs
View file

@ -362,7 +362,7 @@ public class AccountController(
.FilterByUser(account)
.FilterByAccountStatusesRequest(request)
.EnsureVisibleFor(user)
.FilterIncomingBlocks(user)
.FilterHidden(user, db, except: id)
.Paginate(query, ControllerContext)
.PrecomputeVisibilities(user)
.RenderAllForMastodonAsync(noteRenderer, user, Filter.FilterContext.Accounts);

3
Iceshrimp.Backend/Controllers/Mastodon/ConversationsController.cs
View file

@ -40,8 +40,7 @@ public class ConversationsController(
var conversations = await db.Conversations(user)
.IncludeCommonProperties()
.FilterMutedConversations(user, db)
.FilterBlockedConversations(user, db)
.FilterHiddenConversations(user, db)
.Paginate(p => p.ThreadId ?? p.Id, pq, ControllerContext)
.Select(p => new Conversation
{

3
Iceshrimp.Backend/Controllers/Mastodon/NotificationController.cs
View file

@ -48,8 +48,7 @@ public class NotificationController(DatabaseContext db, NotificationRenderer not
p.Type == NotificationType.Edit)
.FilterByGetNotificationsRequest(request)
.EnsureNoteVisibilityFor(p => p.Note, user)
.FilterBlocked(p => p.Notifier, user)
.FilterBlocked(p => p.Note, user)
.FilterHiddenNotifications(user, db)
.Paginate(p => p.MastoId, query, ControllerContext)
.PrecomputeNoteVisibilities(user)
.RenderAllForMastodonAsync(notificationRenderer, user);

4
Iceshrimp.Backend/Controllers/Mastodon/SearchController.cs
View file

@ -184,9 +184,7 @@ public class SearchController(
.Where(p => !search.Following || p.User.IsFollowedBy(user))
.FilterByUser(search.UserId)
.EnsureVisibleFor(user)
.FilterHiddenListMembers(user)
.FilterBlocked(user)
.FilterMuted(user)
.FilterHidden(user, db)
.Paginate(pagination, ControllerContext)
.Skip(pagination.Offset ?? 0)
.PrecomputeVisibilities(user)

40
Iceshrimp.Backend/Controllers/Mastodon/StatusController.cs
View file

@ -46,7 +46,8 @@ public class StatusController(
var note = await db.Notes
.Where(p => p.Id == id)
.IncludeCommonProperties()
.FilterIncomingBlocks(user)
.FilterHidden(user, db, filterOutgoingBlocks: false, filterMutes: false,
filterMentions: false)
.EnsureVisibleFor(user)
.PrecomputeVisibilities(user)
.FirstOrDefaultAsync() ??
@ -69,14 +70,13 @@ public class StatusController(
_ = await db.Notes
.Where(p => p.Id == id)
.EnsureVisibleFor(user)
.FilterIncomingBlocks(user)
.FilterHidden(user, db, filterOutgoingBlocks: false, filterMutes: false)
.FirstOrDefaultAsync() ??
throw GracefulException.RecordNotFound();
var shouldShowContext = await db.Notes
.Where(p => p.Id == id)
.FilterBlocked(user)
.FilterMuted(user)
.FilterHidden(user, db)
.AnyAsync();
if (!shouldShowContext)
@ -85,8 +85,7 @@ public class StatusController(
var ancestors = await db.NoteAncestors(id, maxAncestors)
.IncludeCommonProperties()
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterMuted(user)
.FilterHidden(user, db)
.PrecomputeVisibilities(user)
.RenderAllForMastodonAsync(noteRenderer, user, Filter.FilterContext.Threads);
@ -94,8 +93,7 @@ public class StatusController(
.Where(p => !p.IsQuote || p.RenoteId != id)
.IncludeCommonProperties()
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterMuted(user)
.FilterHidden(user, db)
.PrecomputeVisibilities(user)
.RenderAllForMastodonAsync(noteRenderer, user, Filter.FilterContext.Threads);
@ -114,7 +112,7 @@ public class StatusController(
var note = await db.Notes.Where(p => p.Id == id)
.IncludeCommonProperties()
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterHidden(user, db, filterMutes: false)
.FirstOrDefaultAsync() ??
throw GracefulException.RecordNotFound();
@ -135,7 +133,7 @@ public class StatusController(
var note = await db.Notes.Where(p => p.Id == id)
.IncludeCommonProperties()
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterHidden(user, db, filterMutes: false)
.FirstOrDefaultAsync() ??
throw GracefulException.RecordNotFound();
@ -156,7 +154,7 @@ public class StatusController(
var note = await db.Notes.Where(p => p.Id == id)
.IncludeCommonProperties()
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterHidden(user, db, filterMutes: false)
.FirstOrDefaultAsync() ??
throw GracefulException.RecordNotFound();
@ -177,7 +175,7 @@ public class StatusController(
var note = await db.Notes.Where(p => p.Id == id)
.IncludeCommonProperties()
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterHidden(user, db, filterMutes: false)
.FirstOrDefaultAsync() ??
throw GracefulException.RecordNotFound();
@ -198,7 +196,7 @@ public class StatusController(
var note = await db.Notes.Where(p => p.Id == id)
.IncludeCommonProperties()
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterHidden(user, db, filterMutes: false)
.FirstOrDefaultAsync() ??
throw GracefulException.RecordNotFound();
@ -216,7 +214,7 @@ public class StatusController(
var note = await db.Notes.Where(p => p.Id == id)
.IncludeCommonProperties()
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterHidden(user, db, filterMutes: false)
.FirstOrDefaultAsync() ??
throw GracefulException.RecordNotFound();
@ -269,7 +267,7 @@ public class StatusController(
var note = await db.Notes.Where(p => p.Id == id)
.IncludeCommonProperties()
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterHidden(user, db, filterMutes: false)
.FirstOrDefaultAsync() ??
throw GracefulException.RecordNotFound();
@ -358,7 +356,7 @@ public class StatusController(
? await db.Notes.Where(p => p.Id == request.ReplyId)
.IncludeCommonProperties()
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterHidden(user, db, filterMutes: false)
.FirstOrDefaultAsync() ??
throw GracefulException.BadRequest("Reply target is nonexistent or inaccessible")
: null;
@ -389,7 +387,7 @@ public class StatusController(
? await db.Notes
.IncludeCommonProperties()
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterHidden(user, db, filterMutes: false)
.FirstOrDefaultAsync(p => p.Id == request.QuoteId) ??
throw GracefulException.BadRequest("Quote target is nonexistent or inaccessible")
: null;
@ -400,13 +398,13 @@ public class StatusController(
.IncludeCommonProperties()
.Where(p => p.Id == quoteUri.Substring($"https://{config.Value.WebDomain}/notes/".Length))
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterHidden(user, db, filterMutes: false)
.FirstOrDefaultAsync()
: await db.Notes
.IncludeCommonProperties()
.Where(p => p.Uri == quoteUri || p.Url == quoteUri)
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterHidden(user, db, filterMutes: false)
.FirstOrDefaultAsync()
: null;
@ -519,7 +517,7 @@ public class StatusController(
var user = HttpContext.GetUser();
var note = await db.Notes.Where(p => p.Id == id)
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterHidden(user, db, filterMutes: false)
.FirstOrDefaultAsync() ??
throw GracefulException.RecordNotFound();
@ -541,7 +539,7 @@ public class StatusController(
var note = await db.Notes
.Where(p => p.Id == id)
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterHidden(user, db, filterMutes: false)
.FirstOrDefaultAsync() ??
throw GracefulException.RecordNotFound();

13
Iceshrimp.Backend/Controllers/Mastodon/TimelineController.cs
View file

@ -37,9 +37,7 @@ public class TimelineController(DatabaseContext db, NoteRenderer noteRenderer, C
.IncludeCommonProperties()
.FilterByFollowingAndOwn(user, db, heuristic)
.EnsureVisibleFor(user)
.FilterHiddenListMembers(user)
.FilterBlocked(user)
.FilterMuted(user)
.FilterHidden(user, db, filterHiddenListMembers: true)
.Paginate(query, ControllerContext)
.PrecomputeVisibilities(user)
.RenderAllForMastodonAsync(noteRenderer, user, Filter.FilterContext.Home);
@ -60,8 +58,7 @@ public class TimelineController(DatabaseContext db, NoteRenderer noteRenderer, C
.IncludeCommonProperties()
.HasVisibility(Note.NoteVisibility.Public)
.FilterByPublicTimelineRequest(request)
.FilterBlocked(user)
.FilterMuted(user)
.FilterHidden(user, db)
.Paginate(query, ControllerContext)
.PrecomputeVisibilities(user)
.RenderAllForMastodonAsync(noteRenderer, user, Filter.FilterContext.Public);
@ -82,8 +79,7 @@ public class TimelineController(DatabaseContext db, NoteRenderer noteRenderer, C
.IncludeCommonProperties()
.Where(p => p.Tags.Contains(hashtag.ToLowerInvariant()))
.FilterByHashtagTimelineRequest(request)
.FilterBlocked(user)
.FilterMuted(user)
.FilterHidden(user, db)
.Paginate(query, ControllerContext)
.PrecomputeVisibilities(user)
.RenderAllForMastodonAsync(noteRenderer, user, Filter.FilterContext.Public);
@ -104,8 +100,7 @@ public class TimelineController(DatabaseContext db, NoteRenderer noteRenderer, C
.IncludeCommonProperties()
.Where(p => db.UserListMembers.Any(l => l.UserListId == id && l.UserId == p.UserId))
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterMuted(user)
.FilterHidden(user, db)
.Paginate(query, ControllerContext)
.PrecomputeVisibilities(user)
.RenderAllForMastodonAsync(noteRenderer, user, Filter.FilterContext.Lists);

12
Iceshrimp.Backend/Controllers/NoteController.cs
View file

@ -37,7 +37,7 @@ public class NoteController(
var note = await db.Notes.Where(p => p.Id == id)
.IncludeCommonProperties()
.EnsureVisibleFor(user)
.FilterIncomingBlocks(user)
.FilterHidden(user, db, filterOutgoingBlocks: false, filterMutes: false)
.PrecomputeVisibilities(user)
.FirstOrDefaultAsync() ??
throw GracefulException.NotFound("Note not found");
@ -57,7 +57,7 @@ public class NoteController(
var note = await db.Notes.Where(p => p.Id == id)
.EnsureVisibleFor(user)
.FilterIncomingBlocks(user)
.FilterHidden(user, db, filterOutgoingBlocks: false, filterMutes: false)
.FirstOrDefaultAsync() ??
throw GracefulException.NotFound("Note not found");
@ -65,8 +65,7 @@ public class NoteController(
.Include(p => p.User.UserProfile)
.Include(p => p.Renote!.User.UserProfile)
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterMuted(user)
.FilterHidden(user, db)
.PrecomputeNoteContextVisibilities(user)
.ToListAsync();
@ -86,7 +85,7 @@ public class NoteController(
var note = await db.Notes.Where(p => p.Id == id)
.EnsureVisibleFor(user)
.FilterIncomingBlocks(user)
.FilterHidden(user, db, filterOutgoingBlocks: false, filterMutes: false)
.FirstOrDefaultAsync() ??
throw GracefulException.NotFound("Note not found");
@ -94,8 +93,7 @@ public class NoteController(
.Include(p => p.User.UserProfile)
.Include(p => p.Renote!.User.UserProfile)
.EnsureVisibleFor(user)
.FilterBlocked(user)
.FilterMuted(user)
.FilterHidden(user, db)
.PrecomputeNoteContextVisibilities(user)
.ToListAsync();

3
Iceshrimp.Backend/Controllers/NotificationController.cs
View file

@ -30,8 +30,7 @@ public class NotificationController(DatabaseContext db, NotificationRenderer not
.Where(p => p.Notifiee == user)
.IncludeCommonProperties()
.EnsureNoteVisibilityFor(p => p.Note, user)
.FilterBlocked(p => p.Notifier, user)
.FilterBlocked(p => p.Note, user)
.FilterHiddenNotifications(user, db)
.Paginate(query, ControllerContext)
.PrecomputeNoteVisibilities(user)
.ToListAsync();

4
Iceshrimp.Backend/Controllers/TimelineController.cs
View file

@ -33,9 +33,7 @@ public class TimelineController(DatabaseContext db, CacheService cache, NoteRend
var notes = await db.Notes.IncludeCommonProperties()
.FilterByFollowingAndOwn(user, db, heuristic)
.EnsureVisibleFor(user)
.FilterHiddenListMembers(user)
.FilterBlocked(user)
.FilterMuted(user)
.FilterHidden(user, db, filterHiddenListMembers: true)
.Paginate(pq, ControllerContext)
.PrecomputeVisibilities(user)
.ToListAsync();

2
Iceshrimp.Backend/Controllers/UserController.cs
View file

@ -65,7 +65,7 @@ public class UserController(
.IncludeCommonProperties()
.Where(p => p.User == user)
.EnsureVisibleFor(localUser)
.FilterBlocked(localUser)
.FilterHidden(localUser, db, filterMutes: false)
.PrecomputeVisibilities(localUser)
.Paginate(pq, ControllerContext)
.ToListAsync();

181
Iceshrimp.Backend/Core/Extensions/QueryableExtensions.cs
View file

@ -1,5 +1,6 @@
using System.Diagnostics.CodeAnalysis;
using System.Linq.Expressions;
using EntityFrameworkCore.Projectables;
using Iceshrimp.Backend.Controllers.Attributes;
using Iceshrimp.Backend.Controllers.Mastodon.Renderers;
using Iceshrimp.Backend.Controllers.Mastodon.Schemas;
@ -335,92 +336,130 @@ public static class QueryableExtensions
p.IsRequested(user), p.IsRequestedBy(user)));
}
public static IQueryable<Note> FilterBlocked(this IQueryable<Note> query, User? user)
{
if (user == null) return query;
return query.Where(note => !note.User.IsBlocking(user) && !note.User.IsBlockedBy(user))
.Where(note => note.Renote == null ||
(!note.Renote.User.IsBlockedBy(user) && !note.Renote.User.IsBlocking(user)))
.Where(note => note.Renote == null ||
note.Renote.Renote == null ||
(!note.Renote.Renote.User.IsBlockedBy(user) &&
!note.Renote.Renote.User.IsBlocking(user)))
.Where(note => note.Reply == null ||
(!note.Reply.User.IsBlockedBy(user) && !note.Reply.User.IsBlocking(user)));
}
public static IQueryable<Note> FilterIncomingBlocks(this IQueryable<Note> query, User? user)
{
if (user == null) return query;
return query.Where(note => !note.User.IsBlocking(user))
.Where(note => note.Renote == null || !note.Renote.User.IsBlocking(user))
.Where(note => note.Renote == null ||
note.Renote.Renote == null ||
!note.Renote.Renote.User.IsBlocking(user))
.Where(note => note.Reply == null || !note.Reply.User.IsBlocking(user));
}
public static IQueryable<TSource> FilterBlocked<TSource>(
this IQueryable<TSource> query, Expression<Func<TSource, User?>> predicate, User? user
public static IQueryable<Notification> FilterHiddenNotifications(
this IQueryable<Notification> query, User user, DatabaseContext db
)
{
return user == null ? query : query.Where(predicate.Compose(p => p == null || !p.IsBlocking(user)));
var blocks = db.Blockings.Where(i => i.Blocker == user).Select(p => p.BlockeeId);
var mutes = db.Mutings.Where(i => i.Muter == user).Select(p => p.MuteeId);
var hidden = blocks.Concat(mutes);
return query.Where(p => !hidden.Contains(p.NotifierId) && (p.Note == null || !hidden.Contains(p.Note.Id)));
}
public static IQueryable<TSource> FilterBlocked<TSource>(
this IQueryable<TSource> query, Expression<Func<TSource, Note?>> predicate, User? user
public static IQueryable<Note> FilterHiddenConversations(this IQueryable<Note> query, User user, DatabaseContext db)
{
//TODO: handle muted instances
var blocks = db.Blockings.Where(i => i.Blocker == user).Select(p => p.BlockeeId);
var mutes = db.Mutings.Where(i => i.Muter == user).Select(p => p.MuteeId);
var hidden = blocks.Concat(mutes);
return query.Where(p => p.VisibleUserIds.IsDisjoint(hidden));
}
private static (IQueryable<string> hidden, IQueryable<string>? mentionsHidden) FilterHiddenInternal(
User? user,
DatabaseContext db,
bool filterOutgoingBlocks = true, bool filterMutes = true,
bool filterHiddenListMembers = false,
string? except = null
)
{
//TODO: handle muted instances
var hidden = db.Blockings.Where(p => p.Blockee == user).Select(p => p.BlockerId);
IQueryable<string>? mentionsHidden = null;
if (filterOutgoingBlocks)
{
var blockOut = db.Blockings.Where(p => p.Blocker == user).Select(p => p.BlockeeId);
hidden = hidden.Concat(blockOut);
mentionsHidden = mentionsHidden == null ? blockOut : mentionsHidden.Concat(blockOut);
}
if (filterMutes)
{
var mute = db.Mutings.Where(p => p.Muter == user).Select(p => p.MuteeId);
hidden = hidden.Concat(mute);
mentionsHidden = mentionsHidden == null ? mute : mentionsHidden.Concat(mute);
}
if (filterHiddenListMembers)
{
var list = db.UserListMembers.Where(p => p.UserList.User == user && p.UserList.HideFromHomeTl)
.Select(p => p.UserId);
hidden = hidden.Concat(list);
mentionsHidden = mentionsHidden == null ? list : mentionsHidden.Concat(list);
}
if (except != null)
{
hidden = hidden.Except(new[] { except });
mentionsHidden = mentionsHidden?.Except(new[] { except });
}
return (hidden, mentionsHidden);
}
private static Expression<Func<Note, bool>> FilterHiddenExpr(
IQueryable<string> hidden, IQueryable<string>? mentionsHidden, bool filterMentions
)
{
if (filterMentions && mentionsHidden != null)
{
return note => !hidden.Contains(note.UserId) &&
!hidden.Contains(note.RenoteUserId) &&
!hidden.Contains(note.ReplyUserId) &&
(note.Renote == null ||
!hidden.Contains(note.Renote.RenoteUserId)) &&
note.Mentions.IsDisjoint(mentionsHidden);
}
return note => !hidden.Contains(note.UserId) &&
!hidden.Contains(note.RenoteUserId) &&
!hidden.Contains(note.ReplyUserId) &&
(note.Renote == null ||
!hidden.Contains(note.Renote.RenoteUserId));
}
public static IQueryable<TSource> FilterHidden<TSource>(
this IQueryable<TSource> query, Expression<Func<TSource, Note>> pred, User? user,
DatabaseContext db,
bool filterOutgoingBlocks = true, bool filterMutes = true,
bool filterHiddenListMembers = false, bool filterMentions = true,
string? except = null
)
{
if (user == null)
return query;
return query.Where(predicate.Compose(note => note == null ||
(!note.User.IsBlocking(user) &&
!note.User.IsBlockedBy(user) &&
(note.Renote == null ||
(!note.Renote.User.IsBlockedBy(user) &&
!note.Renote.User.IsBlocking(user))) &&
(note.Renote == null ||
note.Renote.Renote == null ||
(!note.Renote.Renote.User.IsBlockedBy(user) &&
!note.Renote.Renote.User.IsBlocking(user))) &&
(note.Reply == null ||
(!note.Reply.User.IsBlockedBy(user) &&
!note.Reply.User.IsBlocking(user))))));
var (hidden, mentionsHidden) = FilterHiddenInternal(user, db, filterOutgoingBlocks, filterMutes,
filterHiddenListMembers, except);
return query.Where(pred.Compose(FilterHiddenExpr(hidden, mentionsHidden, filterMentions)));
}
public static IQueryable<Note> FilterMuted(this IQueryable<Note> query, User? user)
{
//TODO: handle muted instances
if (user == null) return query;
return query.Where(note => !note.User.IsMutedBy(user))
.Where(note => note.Renote == null || !note.Renote.User.IsMutedBy(user))
.Where(note => note.Renote == null ||
note.Renote.Renote == null ||
!note.Renote.Renote.User.IsMutedBy(user))
.Where(note => note.Reply == null || !note.Reply.User.IsMutedBy(user));
}
public static IQueryable<Note> FilterBlockedConversations(
this IQueryable<Note> query, User user, DatabaseContext db
public static IQueryable<Note> FilterHidden(
this IQueryable<Note> query, User? user, DatabaseContext db,
bool filterOutgoingBlocks = true, bool filterMutes = true,
bool filterHiddenListMembers = false, bool filterMentions = true,
string? except = null
)
{
return query.Where(p => !db.Blockings.Any(i => i.Blocker == user && p.VisibleUserIds.Contains(i.BlockeeId)));
if (user == null)
return query;
var (hidden, mentionsHidden) = FilterHiddenInternal(user, db, filterOutgoingBlocks, filterMutes,
filterHiddenListMembers, except);
return query.Where(FilterHiddenExpr(hidden, mentionsHidden, filterMentions));
}
public static IQueryable<Note> FilterMutedConversations(this IQueryable<Note> query, User user, DatabaseContext db)
{
//TODO: handle muted instances
return query.Where(p => !db.Mutings.Any(i => i.Muter == user && p.VisibleUserIds.Contains(i.MuteeId)));
}
public static IQueryable<Note> FilterHiddenListMembers(this IQueryable<Note> query, User user)
{
return query.Where(note => !note.User.UserListMembers.Any(p => p.UserList.User == user &&
p.UserList.HideFromHomeTl));
}
[Projectable]
[SuppressMessage("ReSharper", "MemberCanBePrivate.Global")]
[SuppressMessage("ReSharper", "ParameterTypeCanBeEnumerable.Global")]
public static bool IsDisjoint<T>(this List<T> x, IQueryable<T> y) => x.All(item => !y.Contains(item));
public static Note EnforceRenoteReplyVisibility(this Note note)
{