notfire/Iceshrimp.NET
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[backend/database] Also index & query note.cw for full text search (ISH-129)

Browse source
This commit is contained in:
Laura Hausmann 2024-03-13 00:30:26 +01:00
parent 921f6a19b3
commit 4f8460914d
No known key found for this signature in database
GPG key ID: D044E84C5BE01605
5 changed files with 5840 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
Iceshrimp.Backend/Core/Database/DatabaseContext.cs
View file

@ -603,6 +603,9 @@ public class DatabaseContext(DbContextOptions<DatabaseContext> options)
entity.HasIndex(e => e.Text, "GIN_TRGM_note_text") entity.HasIndex(e => e.Text, "GIN_TRGM_note_text")
.HasMethod("gin") .HasMethod("gin")
.HasOperators("gin_trgm_ops"); .HasOperators("gin_trgm_ops");
entity.HasIndex(e => e.Cw, "GIN_TRGM_note_cw")
.HasMethod("gin")
.HasOperators("gin_trgm_ops");
entity.Property(e => e.AttachedFileTypes).HasDefaultValueSql("'{}'::character varying[]"); entity.Property(e => e.AttachedFileTypes).HasDefaultValueSql("'{}'::character varying[]");
entity.Property(e => e.ChannelId).HasComment("The ID of source channel."); entity.Property(e => e.ChannelId).HasComment("The ID of source channel.");

5784
Iceshrimp.Backend/Core/Database/Migrations/20240312232809_AddNoteCwIndex.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

32
Iceshrimp.Backend/Core/Database/Migrations/20240312232809_AddNoteCwIndex.cs Normal file
View file

@ -0,0 +1,32 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Iceshrimp.Backend.Core.Database.Migrations
{
/// <inheritdoc />
public partial class AddNoteCwIndex : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
Console.WriteLine("Indexing note content warnings, please hang tight!");
Console.WriteLine("This may take a long time (15-30 minutes), especially if your database is unusually large or you're running low end hardware.");
migrationBuilder.CreateIndex(
name: "GIN_TRGM_note_cw",
table: "note",
column: "cw")
.Annotation("Npgsql:IndexMethod", "gin")
.Annotation("Npgsql:IndexOperators", new[] { "gin_trgm_ops" });
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "GIN_TRGM_note_cw",
table: "note");
}
}
}

5
Iceshrimp.Backend/Core/Database/Migrations/DatabaseContextModelSnapshot.cs
View file

@ -2376,6 +2376,11 @@ namespace Iceshrimp.Backend.Core.Database.Migrations
b.HasIndex("UserId", "Id"); b.HasIndex("UserId", "Id");
b.HasIndex(new[] { "Cw" }, "GIN_TRGM_note_cw");
NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex(new[] { "Cw" }, "GIN_TRGM_note_cw"), "gin");
NpgsqlIndexBuilderExtensions.HasOperators(b.HasIndex(new[] { "Cw" }, "GIN_TRGM_note_cw"), new[] { "gin_trgm_ops" });
b.HasIndex(new[] { "Text" }, "GIN_TRGM_note_text"); b.HasIndex(new[] { "Text" }, "GIN_TRGM_note_text");
NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex(new[] { "Text" }, "GIN_TRGM_note_text"), "gin"); NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex(new[] { "Text" }, "GIN_TRGM_note_text"), "gin");

24
Iceshrimp.Backend/Core/Extensions/QueryableFtsExtensions.cs
View file

@ -244,18 +244,26 @@ public static class QueryableFtsExtensions
=> matchType.Equals(MatchFilterType.Substring) => matchType.Equals(MatchFilterType.Substring)
? caseSensitivity.Equals(CaseFilterType.Sensitive) ? caseSensitivity.Equals(CaseFilterType.Sensitive)
? negated ? negated
? !EF.Functions.Like(note.Text!, "%" + query + "%", @"\") ? !EF.Functions.Like(note.Text!, "%" + query + "%", @"\") &&
: EF.Functions.Like(note.Text!, "%" + query + "%", @"\") !EF.Functions.Like(note.Cw!, "%" + query + "%", @"\")
: EF.Functions.Like(note.Text!, "%" + query + "%", @"\") ||
EF.Functions.Like(note.Cw!, "%" + query + "%", @"\")
: negated : negated
? !EF.Functions.ILike(note.Text!, "%" + query + "%", @"\") ? !EF.Functions.ILike(note.Text!, "%" + query + "%", @"\") &&
: EF.Functions.ILike(note.Text!, "%" + query + "%", @"\") !EF.Functions.ILike(note.Cw!, "%" + query + "%", @"\")
: EF.Functions.ILike(note.Text!, "%" + query + "%", @"\") ||
EF.Functions.ILike(note.Cw!, "%" + query + "%", @"\")
: caseSensitivity.Equals(CaseFilterType.Sensitive) : caseSensitivity.Equals(CaseFilterType.Sensitive)
? negated ? negated
? !Regex.IsMatch(note.Text!, "\\y" + query + "\\y") ? !Regex.IsMatch(note.Text!, "\\y" + query + "\\y") &&
: Regex.IsMatch(note.Text!, "\\y" + query + "\\y") !Regex.IsMatch(note.Cw!, "\\y" + query + "\\y")
: Regex.IsMatch(note.Text!, "\\y" + query + "\\y") ||
Regex.IsMatch(note.Cw!, "\\y" + query + "\\y")
: negated : negated
? !Regex.IsMatch(note.Text!, "\\y" + query + "\\y", RegexOptions.IgnoreCase) ? !Regex.IsMatch(note.Text!, "\\y" + query + "\\y", RegexOptions.IgnoreCase) &&
: Regex.IsMatch(note.Text!, "\\y" + query + "\\y", RegexOptions.IgnoreCase); !Regex.IsMatch(note.Cw!, "\\y" + query + "\\y", RegexOptions.IgnoreCase)
: Regex.IsMatch(note.Text!, "\\y" + query + "\\y", RegexOptions.IgnoreCase) ||
Regex.IsMatch(note.Cw!, "\\y" + query + "\\y", RegexOptions.IgnoreCase);
internal static string PreEscapeFtsQuery(string query, MatchFilterType matchType) => internal static string PreEscapeFtsQuery(string query, MatchFilterType matchType) =>
matchType.Equals(MatchFilterType.Substring) matchType.Equals(MatchFilterType.Substring)