From 7a5786204807ff360f8ff76b8dc3617e64052c38 Mon Sep 17 00:00:00 2001 From: Laura Hausmann Date: Wed, 26 Jun 2024 17:14:46 +0200 Subject: [PATCH] [backend/core] Improve heuristics query performance, move timeline-related extensions into its own file --- .../Mastodon/TimelineController.cs | 2 +- .../Controllers/TimelineController.cs | 2 +- .../Core/Extensions/QueryableExtensions.cs | 39 -------------- .../Extensions/QueryableTimelineExtensions.cs | 52 +++++++++++++++++++ 4 files changed, 54 insertions(+), 41 deletions(-) create mode 100644 Iceshrimp.Backend/Core/Extensions/QueryableTimelineExtensions.cs diff --git a/Iceshrimp.Backend/Controllers/Mastodon/TimelineController.cs b/Iceshrimp.Backend/Controllers/Mastodon/TimelineController.cs index 9bf3f7ff..3c190c6b 100644 --- a/Iceshrimp.Backend/Controllers/Mastodon/TimelineController.cs +++ b/Iceshrimp.Backend/Controllers/Mastodon/TimelineController.cs @@ -31,7 +31,7 @@ public class TimelineController(DatabaseContext db, NoteRenderer noteRenderer, C public async Task GetHomeTimeline(MastodonPaginationQuery query) { var user = HttpContext.GetUserOrFail(); - var heuristic = await QueryableExtensions.GetHeuristic(user, db, cache); + var heuristic = await QueryableTimelineExtensions.GetHeuristic(user, db, cache); var res = await db.Notes .IncludeCommonProperties() diff --git a/Iceshrimp.Backend/Controllers/TimelineController.cs b/Iceshrimp.Backend/Controllers/TimelineController.cs index 6aba5ebf..3eebfa5b 100644 --- a/Iceshrimp.Backend/Controllers/TimelineController.cs +++ b/Iceshrimp.Backend/Controllers/TimelineController.cs @@ -29,7 +29,7 @@ public class TimelineController(DatabaseContext db, CacheService cache, NoteRend public async Task GetHomeTimeline(PaginationQuery pq) { var user = HttpContext.GetUserOrFail(); - var heuristic = await QueryableExtensions.GetHeuristic(user, db, cache); + var heuristic = await QueryableTimelineExtensions.GetHeuristic(user, db, cache); var notes = await db.Notes.IncludeCommonProperties() .FilterByFollowingAndOwn(user, db, heuristic) .EnsureVisibleFor(user) diff --git a/Iceshrimp.Backend/Core/Extensions/QueryableExtensions.cs b/Iceshrimp.Backend/Core/Extensions/QueryableExtensions.cs index ac125f2e..bfc09464 100644 --- a/Iceshrimp.Backend/Core/Extensions/QueryableExtensions.cs +++ b/Iceshrimp.Backend/Core/Extensions/QueryableExtensions.cs @@ -259,45 +259,6 @@ public static class QueryableExtensions return query.Where(note => note.Visibility == visibility); } - /// - /// Runs the most efficient following query for the user in question. - /// The different queries are identical but nudge the query planner towards the smartest query plan available. - /// - public static IQueryable FilterByFollowingAndOwn( - this IQueryable query, User user, DatabaseContext db, int heuristic - ) - { - // Determined empirically in 2023. Ask zotan for the spreadsheet if you're curious. - const int cutoff = 250; - - return heuristic < cutoff - ? query.Where(note => db.Users - .First(p => p == user) - .Following - .Select(p => p.Id) - .Concat(new[] { user.Id }) - .Contains(note.UserId)) - : query.Where(note => note.User == user || note.User.IsFollowedBy(user)); - } - - //TODO: move this into another class where it makes more sense - public static async Task GetHeuristic(User user, DatabaseContext db, CacheService cache) - { - return await cache.FetchValueAsync($"following-query-heuristic:{user.Id}", - TimeSpan.FromHours(24), FetchHeuristic); - - [SuppressMessage("ReSharper", "EntityFramework.UnsupportedServerSideFunctionCall")] - async Task FetchHeuristic() - { - var lastDate = await db.Notes.AnyAsync() - ? await db.Notes.OrderByDescending(p => p.Id).Select(p => p.CreatedAt).FirstOrDefaultAsync() - : DateTime.UtcNow; - - return await db.Notes.CountAsync(p => p.CreatedAt > lastDate - TimeSpan.FromDays(7) && - (p.User.IsFollowedBy(user) || p.User == user)); - } - } - public static IQueryable FilterByUser(this IQueryable query, User user) { return query.Where(note => note.User == user); diff --git a/Iceshrimp.Backend/Core/Extensions/QueryableTimelineExtensions.cs b/Iceshrimp.Backend/Core/Extensions/QueryableTimelineExtensions.cs new file mode 100644 index 00000000..f7aa77e8 --- /dev/null +++ b/Iceshrimp.Backend/Core/Extensions/QueryableTimelineExtensions.cs @@ -0,0 +1,52 @@ +using System.Diagnostics.CodeAnalysis; +using Iceshrimp.Backend.Core.Database; +using Iceshrimp.Backend.Core.Database.Tables; +using Iceshrimp.Backend.Core.Services; +using Microsoft.EntityFrameworkCore; + +namespace Iceshrimp.Backend.Core.Extensions; + +public static class QueryableTimelineExtensions +{ + // Determined empirically in 2023. Ask zotan for the spreadsheet if you're curious. + private const int Cutoff = 250; + + public static IQueryable FilterByFollowingAndOwn( + this IQueryable query, User user, DatabaseContext db, int heuristic + ) + { + return heuristic < Cutoff + ? query.FollowingAndOwnLowFreq(user, db) + : query.Where(note => note.User == user || note.User.IsFollowedBy(user)); + } + + private static IQueryable FollowingAndOwnLowFreq(this IQueryable query, User user, DatabaseContext db) + => query.Where(note => db.Followings + .Where(p => p.Follower == user) + .Select(p => p.FolloweeId) + .Concat(new[] { user.Id }) + .Contains(note.UserId)); + + public static async Task GetHeuristic(User user, DatabaseContext db, CacheService cache) + { + return await cache.FetchValueAsync($"following-query-heuristic:{user.Id}", + TimeSpan.FromHours(24), FetchHeuristic); + + [SuppressMessage("ReSharper", "EntityFramework.UnsupportedServerSideFunctionCall")] + async Task FetchHeuristic() + { + var latestNote = await db.Notes.OrderByDescending(p => p.Id) + .Select(p => new { p.CreatedAt }) + .FirstOrDefaultAsync() ?? + new { CreatedAt = DateTime.UtcNow }; + + //TODO: maybe we should express this as a ratio between matching and non-matching posts + return await db.Notes + .Where(p => p.CreatedAt > latestNote.CreatedAt - TimeSpan.FromDays(7)) + .FollowingAndOwnLowFreq(user, db) + //.Select(p => new { }) + .Take(Cutoff + 1) + .CountAsync(); + } + } +} \ No newline at end of file