notfire/Iceshrimp.NET
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[backend/masto-client] Use the F# SearchQuery parser for FTS queries (ISH-11)

Browse source
This commit is contained in:
Laura Hausmann 2024-03-03 23:12:17 +01:00
parent 27649e8e1e
commit 6a10d408f9
No known key found for this signature in database
GPG key ID: D044E84C5BE01605
8 changed files with 6487 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Iceshrimp.Backend/Controllers/Mastodon/SearchController.cs
View file

@ -176,8 +176,8 @@ public class SearchController(
return await db.Notes
.IncludeCommonProperties()
.Where(p => p.TextContainsCaseInsensitive(search.Query!) &&
(!search.Following || p.User.IsFollowedBy(user)))
.FilterByFtsQuery(search.Query!, user, db)
.Where(p => !search.Following || p.User.IsFollowedBy(user))
.FilterByUser(search.UserId)
.EnsureVisibleFor(user)
.FilterHiddenListMembers(user)

4
Iceshrimp.Backend/Core/Database/DatabaseContext.cs
View file

@ -146,6 +146,10 @@ public class DatabaseContext(DbContextOptions<DatabaseContext> options)
.HasDbFunction(typeof(DatabaseContext).GetMethod(nameof(Conversations),
[typeof(string)])!)
.HasName("conversations");
modelBuilder
.HasDbFunction(typeof(Note).GetMethod(nameof(Note.InternalRawAttachments),
[typeof(string)])!)
.HasName("note_attachments_raw");
modelBuilder.Entity<AbuseUserReport>(entity =>
{

6165
Iceshrimp.Backend/Core/Database/Migrations/20240303224431_AddAttachmentsRawFunction.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

29
Iceshrimp.Backend/Core/Database/Migrations/20240303224431_AddAttachmentsRawFunction.cs Normal file
View file

@ -0,0 +1,29 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Iceshrimp.Backend.Core.Database.Migrations
{
/// <inheritdoc />
public partial class AddAttachmentsRawFunction : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.Sql("""
CREATE OR REPLACE FUNCTION public.note_attachments_raw(note_id character varying)
RETURNS varchar
LANGUAGE sql
AS $function$
SELECT "attachedFileTypes"::varchar FROM "note" WHERE "id" = note_id
$function$;
""");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.Sql("DROP FUNCTION public.note_attachments_raw;");
}
}
}

7
Iceshrimp.Backend/Core/Database/Tables/Note.cs
View file

@ -235,6 +235,13 @@ public class Note : IEntity
[InverseProperty(nameof(InverseReply))]
public virtual Note? Reply { get; set; }
[Projectable]
public string RawAttachments
=> InternalRawAttachments(Id);
public static string InternalRawAttachments(string id)
=> throw new NotSupportedException();
[NotMapped] [Projectable] public bool IsPureRenote => (RenoteId != null || Renote != null) && !IsQuote;
[NotMapped]

271
Iceshrimp.Backend/Core/Extensions/QueryableFtsExtensions.cs Normal file
View file

@ -0,0 +1,271 @@
using System.Diagnostics.CodeAnalysis;
using System.Text.RegularExpressions;
using EntityFrameworkCore.Projectables;
using Iceshrimp.Backend.Core.Database;
using Iceshrimp.Backend.Core.Database.Tables;
using Iceshrimp.Backend.Core.Helpers;
using Iceshrimp.Parsing;
using Microsoft.EntityFrameworkCore;
using static Iceshrimp.Parsing.SearchQueryFilters;
namespace Iceshrimp.Backend.Core.Extensions;
public static class QueryableFtsExtensions
{
public static IQueryable<Note> FilterByFtsQuery(
this IQueryable<Note> query, string input, User user, DatabaseContext db
)
{
var parsed = SearchQuery.parse(input);
var caseSensitivity = parsed.OfType<CaseFilter>().LastOrDefault()?.Value ?? CaseFilterType.Insensitive;
var matchType = parsed.OfType<MatchFilter>().LastOrDefault()?.Value ?? MatchFilterType.Substring;
return parsed.Aggregate(query, (current, filter) => filter switch
{
CaseFilter => current,
MatchFilter => current,
AfterFilter afterFilter => current.ApplyAfterFilter(afterFilter),
AttachmentFilter attachmentFilter => current.ApplyAttachmentFilter(attachmentFilter),
BeforeFilter beforeFilter => current.ApplyBeforeFilter(beforeFilter),
FromFilter fromFilter => current.ApplyFromFilter(fromFilter, db),
InFilter inFilter => current.ApplyInFilter(inFilter, user, db),
InstanceFilter instanceFilter => current.ApplyInstanceFilter(instanceFilter),
MentionFilter mentionFilter => current.ApplyMentionFilter(mentionFilter, db),
MiscFilter miscFilter => current.ApplyMiscFilter(miscFilter, user),
ReplyFilter replyFilter => current.ApplyReplyFilter(replyFilter, db),
WordFilter wordFilter => current.ApplyWordFilter(wordFilter, caseSensitivity, matchType),
MultiWordFilter multiWordFilter =>
current.ApplyMultiWordFilter(multiWordFilter, caseSensitivity, matchType),
_ => throw new ArgumentOutOfRangeException(nameof(filter))
});
}
internal static (string username, string? host) UserToTuple(string filter)
{
filter = filter.TrimStart('@');
var split = filter.Split('@');
var username = split[0].ToLowerInvariant();
var host = split.Length > 1 ? split[1] : null;
return (username, host);
}
[Projectable]
private static IQueryable<Note> ApplyAfterFilter(this IQueryable<Note> query, AfterFilter filter)
=> query.Where(p => p.CreatedAt >= filter.Value.ToDateTime(TimeOnly.MinValue).ToUniversalTime());
[Projectable]
private static IQueryable<Note> ApplyBeforeFilter(this IQueryable<Note> query, BeforeFilter filter)
=> query.Where(p => p.CreatedAt < filter.Value.ToDateTime(TimeOnly.MinValue).ToUniversalTime());
[Projectable]
private static IQueryable<Note> ApplyWordFilter(
this IQueryable<Note> query, WordFilter filter, CaseFilterType caseSensitivity, MatchFilterType matchType
) => query.Where(p => p.FtsQueryPreEscaped(PreEscapeFtsQuery(filter.Value, matchType), filter.Negated,
caseSensitivity, matchType));
[Projectable]
private static IQueryable<Note> ApplyMultiWordFilter(
this IQueryable<Note> query, MultiWordFilter filter, CaseFilterType caseSensitivity, MatchFilterType matchType
) => query.Where(p => p.FtsQueryOneOf(filter.Values, caseSensitivity, matchType));
[Projectable]
private static IQueryable<Note> ApplyFromFilter(this IQueryable<Note> query, FromFilter filter, DatabaseContext db)
=> query.Where(p => p.User.UserSubqueryMatches(filter.Value, filter.Negated, db));
[Projectable]
private static IQueryable<Note> ApplyInstanceFilter(this IQueryable<Note> query, InstanceFilter filter)
=> query.Where(p => filter.Negated ? p.UserHost != filter.Value : p.UserHost == filter.Value);
[Projectable]
private static IQueryable<Note> ApplyMentionFilter(
this IQueryable<Note> query, MentionFilter filter, DatabaseContext db
)
=> query.Where(p => p.Mentions.UserSubqueryContains(filter.Value, filter.Negated, db));
[Projectable]
private static IQueryable<Note> ApplyReplyFilter(
this IQueryable<Note> query, ReplyFilter filter, DatabaseContext db
)
=> query.Where(p => p.Reply != null && p.Reply.User.UserSubqueryMatches(filter.Value, filter.Negated, db));
[Projectable]
private static IQueryable<Note> ApplyInFilter(
this IQueryable<Note> query, InFilter filter, User user, DatabaseContext db
)
=> filter.Value.Equals(InFilterType.Bookmarks)
? query.ApplyInBookmarksFilter(user, filter.Negated, db)
: filter.Value.Equals(InFilterType.Likes)
? query.ApplyInLikesFilter(user, filter.Negated, db)
: query.ApplyInReactionsFilter(user, filter.Negated, db);
[Projectable]
[SuppressMessage("ReSharper", "EntityFramework.UnsupportedServerSideFunctionCall", Justification = "Projectables")]
internal static IQueryable<Note> ApplyInBookmarksFilter(
this IQueryable<Note> query, User user, bool negated, DatabaseContext db
) => query.Where(p => negated
? !db.Users.First(u => u == user).HasBookmarked(p)
: db.Users.First(u => u == user).HasBookmarked(p));
[Projectable]
[SuppressMessage("ReSharper", "EntityFramework.UnsupportedServerSideFunctionCall", Justification = "Projectables")]
internal static IQueryable<Note> ApplyInLikesFilter(
this IQueryable<Note> query, User user, bool negated, DatabaseContext db
) => query.Where(p => negated
? !db.Users.First(u => u == user).HasLiked(p)
: db.Users.First(u => u == user).HasLiked(p));
[Projectable]
[SuppressMessage("ReSharper", "EntityFramework.UnsupportedServerSideFunctionCall", Justification = "Projectables")]
internal static IQueryable<Note> ApplyInReactionsFilter(
this IQueryable<Note> query, User user, bool negated, DatabaseContext db
) => query.Where(p => negated
? !db.Users.First(u => u == user).HasReacted(p)
: db.Users.First(u => u == user).HasReacted(p));
[Projectable]
private static IQueryable<Note> ApplyMiscFilter(
this IQueryable<Note> query, MiscFilter filter, User user
)
=> filter.Value.Equals(MiscFilterType.Followers)
? query.ApplyFollowersFilter(user, filter.Negated)
: filter.Value.Equals(MiscFilterType.Following)
? query.ApplyFollowingFilter(user, filter.Negated)
: filter.Value.Equals(MiscFilterType.Replies)
? query.ApplyRepliesFilter(filter.Negated)
: query.ApplyBoostsFilter(filter.Negated);
[Projectable]
[SuppressMessage("ReSharper", "EntityFramework.UnsupportedServerSideFunctionCall", Justification = "Projectables")]
internal static IQueryable<Note> ApplyFollowersFilter(
this IQueryable<Note> query, User user, bool negated
) => query.Where(p => negated
? !p.User.IsFollowing(user)
: p.User.IsFollowing(user));
[Projectable]
[SuppressMessage("ReSharper", "EntityFramework.UnsupportedServerSideFunctionCall", Justification = "Projectables")]
internal static IQueryable<Note> ApplyFollowingFilter(
this IQueryable<Note> query, User user, bool negated
) => query.Where(p => negated
? !p.User.IsFollowedBy(user)
: p.User.IsFollowedBy(user));
[Projectable]
[SuppressMessage("ReSharper", "EntityFramework.UnsupportedServerSideFunctionCall", Justification = "Projectables")]
internal static IQueryable<Note> ApplyRepliesFilter(
this IQueryable<Note> query, bool negated
) => query.Where(p => negated
? p.Reply == null
: p.Reply != null);
[Projectable]
[SuppressMessage("ReSharper", "EntityFramework.UnsupportedServerSideFunctionCall", Justification = "Projectables")]
internal static IQueryable<Note> ApplyBoostsFilter(
this IQueryable<Note> query, bool negated
) => query.Where(p => negated
? !p.IsPureRenote
: p.IsPureRenote);
[Projectable]
private static IQueryable<Note> ApplyAttachmentFilter(this IQueryable<Note> query, AttachmentFilter filter)
=> filter.Negated ? query.ApplyNegatedAttachmentFilter(filter) : query.ApplyRegularAttachmentFilter(filter);
[Projectable]
internal static IQueryable<Note> ApplyRegularAttachmentFilter(this IQueryable<Note> query, AttachmentFilter filter)
=> AttachmentQuery(filter.Value)
? query.Where(p => EF.Functions.ILike(p.RawAttachments, GetAttachmentILikeQuery(filter.Value)))
: filter.Value.Equals(AttachmentFilterType.Any)
? query.Where(p => p.AttachedFileTypes.Count != 0)
: query.Where(p => p.AttachedFileTypes.Count != 0 &&
(!EF.Functions.ILike(p.RawAttachments,
GetAttachmentILikeQuery(AttachmentFilterType.Image)) ||
!EF.Functions.ILike(p.RawAttachments,
GetAttachmentILikeQuery(AttachmentFilterType.Video)) ||
!EF.Functions.ILike(p.RawAttachments,
GetAttachmentILikeQuery(AttachmentFilterType.Audio))));
[Projectable]
internal static IQueryable<Note> ApplyNegatedAttachmentFilter(this IQueryable<Note> query, AttachmentFilter filter)
=> AttachmentQuery(filter.Value)
? query.Where(p => !EF.Functions.ILike(p.RawAttachments, GetAttachmentILikeQuery(filter.Value)))
: filter.Value.Equals(AttachmentFilterType.Any)
? query.Where(p => p.AttachedFileTypes.Count == 0)
: query.Where(p => EF.Functions
.ILike(p.RawAttachments, GetAttachmentILikeQuery(AttachmentFilterType.Image)) ||
EF.Functions
.ILike(p.RawAttachments, GetAttachmentILikeQuery(AttachmentFilterType.Video)) ||
EF.Functions
.ILike(p.RawAttachments, GetAttachmentILikeQuery(AttachmentFilterType.Audio)));
internal static bool AttachmentQuery(AttachmentFilterType filter)
{
if (filter.Equals(AttachmentFilterType.Image))
return true;
if (filter.Equals(AttachmentFilterType.Video))
return true;
if (filter.Equals(AttachmentFilterType.Audio))
return true;
return false;
}
internal static string GetAttachmentILikeQuery(AttachmentFilterType filter)
{
if (filter.Equals(AttachmentFilterType.Image))
return "%image/%";
if (filter.Equals(AttachmentFilterType.Video))
return "%video/%";
if (filter.Equals(AttachmentFilterType.Audio))
return "%audio/%";
throw new ArgumentOutOfRangeException(nameof(filter));
}
[Projectable]
internal static bool UserSubqueryMatches(
this User user, string filter, bool negated, DatabaseContext db
) => negated
? !UserSubquery(UserToTuple(filter), db).Contains(user)
: UserSubquery(UserToTuple(filter), db).Contains(user);
[Projectable]
internal static bool UserSubqueryContains(
this IEnumerable<string> userIds, string filter, bool negated, DatabaseContext db
) => negated
? userIds.All(p => p != UserSubquery(UserToTuple(filter), db).Select(i => i.Id).FirstOrDefault())
: userIds.Any(p => p == UserSubquery(UserToTuple(filter), db).Select(i => i.Id).FirstOrDefault());
[Projectable]
internal static IQueryable<User> UserSubquery((string username, string? host) filter, DatabaseContext db) =>
db.Users.Where(p => p.UsernameLower == filter.username && p.Host == filter.host);
[Projectable]
internal static bool FtsQueryPreEscaped(
this Note note, string query, bool negated, CaseFilterType caseSensitivity, MatchFilterType matchType
)
=> matchType.Equals(MatchFilterType.Substring)
? caseSensitivity.Equals(CaseFilterType.Sensitive)
? negated
? !EF.Functions.Like(note.Text!, "%" + query + "%", @"\")
: EF.Functions.Like(note.Text!, "%" + query + "%", @"\")
: negated
? !EF.Functions.ILike(note.Text!, "%" + query + "%", @"\")
: EF.Functions.ILike(note.Text!, "%" + query + "%", @"\")
: caseSensitivity.Equals(CaseFilterType.Sensitive)
? negated
? !Regex.IsMatch(note.Text!, "\\y" + query + "\\y")
: Regex.IsMatch(note.Text!, "\\y" + query + "\\y")
: negated
? !Regex.IsMatch(note.Text!, "\\y" + query + "\\y", RegexOptions.IgnoreCase)
: Regex.IsMatch(note.Text!, "\\y" + query + "\\y", RegexOptions.IgnoreCase);
internal static string PreEscapeFtsQuery(string query, MatchFilterType matchType) =>
matchType.Equals(MatchFilterType.Substring)
? EfHelpers.EscapeLikeQuery(query)
: EfHelpers.EscapeRegexQuery(query);
[Projectable]
internal static bool FtsQueryOneOf(
this Note note, IEnumerable<string> words, CaseFilterType caseSensitivity, MatchFilterType matchType
)
=> words.Select(p => PreEscapeFtsQuery(p, matchType))
.Any(p => note.FtsQueryPreEscaped(p, false, caseSensitivity, matchType));
}

5
Iceshrimp.Backend/Core/Helpers/EfHelpers.cs
View file

@ -1,3 +1,5 @@
using System.Text.RegularExpressions;
namespace Iceshrimp.Backend.Core.Helpers;
public static class EfHelpers
@ -8,4 +10,7 @@ public static class EfHelpers
.Replace("^", @"\^")
.Replace("[", @"\[")
.Replace("]", @"\]");
public static string EscapeRegexQuery(string input) =>
new Regex(@"([!$()*+.:<=>?[\\\]^{|}-])").Replace(input, "\\$1");
}

4
Iceshrimp.Backend/Iceshrimp.Backend.csproj
View file

@ -60,4 +60,8 @@
<None Remove="migrate.sql"/>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Iceshrimp.Parsing\Iceshrimp.Parsing.fsproj" />
</ItemGroup>
</Project>