From 36d9a8cc492425df982b4704f1f0ae802a359a3b Mon Sep 17 00:00:00 2001 From: Kopper Date: Wed, 6 Nov 2024 18:27:24 +0300 Subject: [PATCH] [backend/federation] Context collection --- FEDERATION.md | 4 + .../Federation/ActivityPubController.cs | 32 ++++++++ .../DatabaseContextModelSnapshot.cs | 29 ++++++++ .../20241106151513_AddNoteThreadContext.cs | 74 +++++++++++++++++++ .../20241106163544_IndexNoteThreadContext.cs | 31 ++++++++ .../Core/Database/Tables/NoteThread.cs | 47 +++++++++++- .../Federation/ActivityPub/NoteRenderer.cs | 10 ++- .../ActivityStreams/Types/ASCollection.cs | 5 +- .../ActivityStreams/Types/ASNote.cs | 6 +- .../Core/Services/NoteService.cs | 47 +++++++++++- 10 files changed, 277 insertions(+), 8 deletions(-) create mode 100644 Iceshrimp.Backend/Core/Database/Migrations/v2024.1-beta5/20241106151513_AddNoteThreadContext.cs create mode 100644 Iceshrimp.Backend/Core/Database/Migrations/v2024.1-beta5/20241106163544_IndexNoteThreadContext.cs diff --git a/FEDERATION.md b/FEDERATION.md index 20bf90f4..e72b569a 100644 --- a/FEDERATION.md +++ b/FEDERATION.md @@ -66,6 +66,10 @@ This document **MAY** alias JSON-LD namespace IRIs to their well-known aliases. - [FEP-67ff: FEDERATION.md](https://codeberg.org/fediverse/fep/src/branch/main/fep/67ff/fep-67ff.md) - [FEP-2c59: Discovery of a Webfinger address from an ActivityPub actor](https://codeberg.org/fediverse/fep/src/branch/main/fep/2c59/fep-2c59.md) - [FEP-9fde: Mechanism for servers to expose supported operations](https://codeberg.org/fediverse/fep/src/branch/main/fep/9fde/fep-9fde.md) +- [FEP-7888: Demystifying the context property](https://codeberg.org/fediverse/fep/src/branch/main/fep/7888/fep-7888.md) + + Specifically, we use it in a "conversational context" sense, where each note has an attached context, which maps to an internal "thread". + + We currently do not use the context for anything other than grouping. + + Our context collections contain objects, not activities. ## FEPs we intend to support in the future - [FEP-8fcf: Followers collection synchronization across servers](https://codeberg.org/fediverse/fep/src/branch/main/fep/8fcf/fep-8fcf.md) diff --git a/Iceshrimp.Backend/Controllers/Federation/ActivityPubController.cs b/Iceshrimp.Backend/Controllers/Federation/ActivityPubController.cs index 5d91b59a..f2ce4c75 100644 --- a/Iceshrimp.Backend/Controllers/Federation/ActivityPubController.cs +++ b/Iceshrimp.Backend/Controllers/Federation/ActivityPubController.cs @@ -107,6 +107,38 @@ public class ActivityPubController( return res.Compact(); } + [HttpGet("/threads/{id}")] + [AuthorizedFetch] + [OverrideResultType] + [ProducesResults(HttpStatusCode.OK)] + [ProducesErrors(HttpStatusCode.NotFound)] + public async Task GetThread(string id) + { + var actor = HttpContext.GetActor(); + var thread = await db.NoteThreads + .Include(p => p.User) + .FirstOrDefaultAsync(p => p.Id == id && p.User != null && p.User.IsLocalUser) ?? + throw GracefulException.NotFound("Thread not found"); + + var notes = await db.Notes + .Where(p => p.ThreadId == id) + .EnsureVisibleFor(actor) + .OrderByDescending(p => p.Id) + .Select(p => new Note { Id = p.Id, Uri = p.Uri }) + .ToListAsync(); + + var rendered = notes.Select(noteRenderer.RenderLite).Cast().ToList(); + var res = new ASOrderedCollection + { + Id = thread.GetPublicUri(config.Value), + AttributedTo = [new ASObjectBase(thread.User!.GetPublicUri(config.Value))], + TotalItems = (ulong)rendered.Count, + Items = rendered + }; + + return res.Compact(); + } + [HttpGet("/users/{id}")] [AuthorizedFetch] [OutputCache(PolicyName = "federation")] diff --git a/Iceshrimp.Backend/Core/Database/Migrations/DatabaseContextModelSnapshot.cs b/Iceshrimp.Backend/Core/Database/Migrations/DatabaseContextModelSnapshot.cs index a48e4ec7..16c4a7a8 100644 --- a/Iceshrimp.Backend/Core/Database/Migrations/DatabaseContextModelSnapshot.cs +++ b/Iceshrimp.Backend/Core/Database/Migrations/DatabaseContextModelSnapshot.cs @@ -2807,8 +2807,27 @@ namespace Iceshrimp.Backend.Core.Database.Migrations .HasColumnType("timestamp with time zone") .HasColumnName("backfilledAt"); + b.Property("IsResolvable") + .HasColumnType("boolean") + .HasColumnName("isResolvable"); + + b.Property("Uri") + .HasMaxLength(512) + .HasColumnType("character varying(512)") + .HasColumnName("uri"); + + b.Property("UserId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("userId"); + b.HasKey("Id"); + b.HasIndex("Uri") + .IsUnique(); + + b.HasIndex("UserId"); + b.ToTable("note_thread"); }); @@ -5374,6 +5393,16 @@ namespace Iceshrimp.Backend.Core.Database.Migrations b.Navigation("User"); }); + modelBuilder.Entity("Iceshrimp.Backend.Core.Database.Tables.NoteThread", b => + { + b.HasOne("Iceshrimp.Backend.Core.Database.Tables.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("User"); + }); + modelBuilder.Entity("Iceshrimp.Backend.Core.Database.Tables.NoteThreadMuting", b => { b.HasOne("Iceshrimp.Backend.Core.Database.Tables.NoteThread", "Thread") diff --git a/Iceshrimp.Backend/Core/Database/Migrations/v2024.1-beta5/20241106151513_AddNoteThreadContext.cs b/Iceshrimp.Backend/Core/Database/Migrations/v2024.1-beta5/20241106151513_AddNoteThreadContext.cs new file mode 100644 index 00000000..87323723 --- /dev/null +++ b/Iceshrimp.Backend/Core/Database/Migrations/v2024.1-beta5/20241106151513_AddNoteThreadContext.cs @@ -0,0 +1,74 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Infrastructure; + +#nullable disable + +namespace Iceshrimp.Backend.Core.Database.Migrations +{ + /// + [DbContext(typeof(DatabaseContext))] + [Migration("20241106151513_AddNoteThreadContext")] + public partial class AddNoteThreadContext : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "isResolvable", + table: "note_thread", + type: "boolean", + nullable: true); + + migrationBuilder.AddColumn( + name: "uri", + table: "note_thread", + type: "character varying(512)", + maxLength: 512, + nullable: true); + + migrationBuilder.AddColumn( + name: "userId", + table: "note_thread", + type: "character varying(32)", + maxLength: 32, + nullable: true); + + migrationBuilder.CreateIndex( + name: "IX_note_thread_userId", + table: "note_thread", + column: "userId"); + + migrationBuilder.AddForeignKey( + name: "FK_note_thread_user_userId", + table: "note_thread", + column: "userId", + principalTable: "user", + principalColumn: "id", + onDelete: ReferentialAction.SetNull); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "FK_note_thread_user_userId", + table: "note_thread"); + + migrationBuilder.DropIndex( + name: "IX_note_thread_userId", + table: "note_thread"); + + migrationBuilder.DropColumn( + name: "isResolvable", + table: "note_thread"); + + migrationBuilder.DropColumn( + name: "uri", + table: "note_thread"); + + migrationBuilder.DropColumn( + name: "userId", + table: "note_thread"); + } + } +} diff --git a/Iceshrimp.Backend/Core/Database/Migrations/v2024.1-beta5/20241106163544_IndexNoteThreadContext.cs b/Iceshrimp.Backend/Core/Database/Migrations/v2024.1-beta5/20241106163544_IndexNoteThreadContext.cs new file mode 100644 index 00000000..c74fc70b --- /dev/null +++ b/Iceshrimp.Backend/Core/Database/Migrations/v2024.1-beta5/20241106163544_IndexNoteThreadContext.cs @@ -0,0 +1,31 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Infrastructure; + +#nullable disable + +namespace Iceshrimp.Backend.Core.Database.Migrations +{ + /// + [DbContext(typeof(DatabaseContext))] + [Migration("20241106163544_IndexNoteThreadContext")] + public partial class IndexNoteThreadContext : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateIndex( + name: "IX_note_thread_uri", + table: "note_thread", + column: "uri", + unique: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "IX_note_thread_uri", + table: "note_thread"); + } + } +} diff --git a/Iceshrimp.Backend/Core/Database/Tables/NoteThread.cs b/Iceshrimp.Backend/Core/Database/Tables/NoteThread.cs index 42e079db..72a81dfd 100644 --- a/Iceshrimp.Backend/Core/Database/Tables/NoteThread.cs +++ b/Iceshrimp.Backend/Core/Database/Tables/NoteThread.cs @@ -1,24 +1,67 @@ using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations.Schema; +using System.Diagnostics.CodeAnalysis; +using EntityFrameworkCore.Projectables; +using Iceshrimp.Backend.Core.Configuration; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; namespace Iceshrimp.Backend.Core.Database.Tables; [Table("note_thread")] +[Index(nameof(Uri), IsUnique = true)] public class NoteThread : IEntity { [Column("id")] [StringLength(256)] public required string Id { get; set; } - + /// /// The last time this thread has been backfilled. /// [Column("backfilledAt")] public DateTime? BackfilledAt { get; set; } - + + /// + /// The ID of the collection representing this thread. Will be null when the thread not part of a context collection. + /// + [Column("uri")] + [StringLength(512)] + public string? Uri { get; set; } + + /// + /// The User owning this thread. Will be null if unknown. Determined by the context collection's attributedTo property. + /// + [Column("userId")] + [StringLength(32)] + public string? UserId { get; set; } + + /// + /// Is the context collection associated with this thread resolvable? Null if this is a local thread that we don't need to resolve. + /// + [Column("isResolvable")] + public bool? IsResolvable { get; set; } = false; + [InverseProperty(nameof(Note.Thread))] public virtual ICollection Notes { get; set; } = new List(); [InverseProperty(nameof(NoteThreadMuting.Thread))] public virtual ICollection NoteThreadMutings { get; set; } = new List(); + + [ForeignKey(nameof(UserId))] + public virtual User? User { get; set; } + + [Projectable] + [SuppressMessage("ReSharper", "MergeIntoPattern", Justification = "Projectables does not support this")] + public string? GetPublicUri(Config.InstanceSection config) => User != null && User.IsLocalUser ? $"https://{config.WebDomain}/threads/{Id}" : null; + + private class NoteThreadConfiguration : IEntityTypeConfiguration + { + public void Configure(EntityTypeBuilder entity) + { + entity.HasOne(e => e.User) + .WithMany() + .OnDelete(DeleteBehavior.SetNull); + } + } } diff --git a/Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs b/Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs index cba926ee..906b4126 100644 --- a/Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs +++ b/Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs @@ -62,6 +62,12 @@ public class NoteRenderer( }) .ToListAsync(); + var contextId = await db.NoteThreads + .Where(p => p.Id == note.ThreadId) + .Select(p => p.Uri ?? p.GetPublicUri(config.Value)) + .FirstOrDefaultAsync(); + var context = contextId != null ? new ASCollection(contextId) : null; + var emoji = note.Emojis.Count != 0 ? await db.Emojis.Where(p => note.Emojis.Contains(p.Id) && p.Host == null).ToListAsync() : []; @@ -166,6 +172,7 @@ public class NoteRenderer( Sensitive = sensitive, InReplyTo = replyId, Replies = replies, + Context = context, Cc = cc, To = to, Tags = tags, @@ -197,6 +204,7 @@ public class NoteRenderer( Sensitive = sensitive, InReplyTo = replyId, Replies = replies, + Context = context, Cc = cc, To = to, Tags = tags, @@ -211,4 +219,4 @@ public class NoteRenderer( QuoteUrl = quoteUri }; } -} \ No newline at end of file +} diff --git a/Iceshrimp.Backend/Core/Federation/ActivityStreams/Types/ASCollection.cs b/Iceshrimp.Backend/Core/Federation/ActivityStreams/Types/ASCollection.cs index 85e42579..0ff10c88 100644 --- a/Iceshrimp.Backend/Core/Federation/ActivityStreams/Types/ASCollection.cs +++ b/Iceshrimp.Backend/Core/Federation/ActivityStreams/Types/ASCollection.cs @@ -18,6 +18,9 @@ public class ASCollection : ASObject [SetsRequiredMembers] public ASCollection(string id, bool withType = false) : this(withType) => Id = id; + [J($"{Constants.ActivityStreamsNs}#attributedTo")] + public List? AttributedTo { get; set; } + [J($"{Constants.ActivityStreamsNs}#items")] [JC(typeof(ASCollectionItemsConverter))] public List? Items { get; set; } @@ -124,4 +127,4 @@ internal class ASCollectionItemsConverter : JsonConverter { throw new NotImplementedException(); } -} \ No newline at end of file +} diff --git a/Iceshrimp.Backend/Core/Federation/ActivityStreams/Types/ASNote.cs b/Iceshrimp.Backend/Core/Federation/ActivityStreams/Types/ASNote.cs index 24fce9f2..bd6f4ced 100644 --- a/Iceshrimp.Backend/Core/Federation/ActivityStreams/Types/ASNote.cs +++ b/Iceshrimp.Backend/Core/Federation/ActivityStreams/Types/ASNote.cs @@ -91,6 +91,10 @@ public class ASNote : ASObjectWithId [JC(typeof(ASCollectionConverter))] public ASCollection? Replies { get; set; } + [J($"{Constants.ActivityStreamsNs}#context")] + [JC(typeof(ASCollectionConverter))] + public ASCollection? Context { get; set; } + public Note.NoteVisibility GetVisibility(User actor) { if (actor.IsLocalUser) throw new Exception("Can't get recipients for local actor"); @@ -124,4 +128,4 @@ public class ASNote : ASObjectWithId public const string Note = $"{Ns}#Note"; } -} \ No newline at end of file +} diff --git a/Iceshrimp.Backend/Core/Services/NoteService.cs b/Iceshrimp.Backend/Core/Services/NoteService.cs index 2e8b658d..aeea1e5c 100644 --- a/Iceshrimp.Backend/Core/Services/NoteService.cs +++ b/Iceshrimp.Backend/Core/Services/NoteService.cs @@ -252,8 +252,49 @@ public class NoteService( var noteId = IdHelpers.GenerateSnowflakeId(data.CreatedAt); var threadId = data.Reply?.ThreadId ?? noteId; - var thread = await db.NoteThreads.Where(t => t.Id == threadId).FirstOrDefaultAsync() ?? - new NoteThread { Id = threadId }; + var context = data.ASNote?.Context; + var contextId = context?.Id; + + var thread = contextId != null + ? await db.NoteThreads.Where(t => t.Uri == contextId || t.Id == threadId).FirstOrDefaultAsync() + : await db.NoteThreads.Where(t => t.Id == threadId).FirstOrDefaultAsync(); + + var contextOwner = data.User.IsLocalUser ? data.User : null; + bool? contextResolvable = data.User.IsLocalUser ? null : false; + + if (thread == null && context != null) + { + try + { + if (await objectResolver.ResolveObjectAsync(context) is ASCollection maybeContext) + { + context = maybeContext; + contextResolvable = true; + + var owner = context.AttributedTo?.FirstOrDefault(); + if (owner?.Id != null + && Uri.TryCreate(owner.Id, UriKind.Absolute, out var ownerUri) + && Uri.TryCreate(contextId, UriKind.Absolute, out var contextUri) + && ownerUri.Host == contextUri.Host) + contextOwner = await userResolver.ResolveOrNullAsync(owner.Id, ResolveFlags.Uri); + } + } + catch + { + /* + * some instance software such as the Pleroma family expose a context that isn't resolvable, which is permitted by spec. + * in that case we still use it for threading but mark it as unresolvable. + */ + } + } + + thread ??= new NoteThread + { + Id = threadId, + Uri = contextId, + User = contextOwner, + IsResolvable = contextResolvable, + }; var note = new Note { @@ -1616,4 +1657,4 @@ public class NoteService( if (dbNote == null) return; await RemoveReactionFromNoteAsync(dbNote, actor, name); } -} \ No newline at end of file +}