From 69360a8ad7fbdf83aa4b7d280d57ebfb27710b50 Mon Sep 17 00:00:00 2001 From: Laura Hausmann Date: Sat, 24 Feb 2024 22:59:13 +0100 Subject: [PATCH] [backend/federation] Handle emoji for incoming notes (ISH-89) --- .../Mastodon/Renderers/NoteRenderer.cs | 32 ++++++++++-- .../Mastodon/Schemas/Entities/EmojiEntity.cs | 14 +++++ .../Mastodon/Schemas/Entities/StatusEntity.cs | 2 +- .../Core/Configuration/Constants.cs | 1 + .../Core/Database/Tables/Emoji.cs | 2 +- .../Federation/ActivityPub/NoteRenderer.cs | 1 - .../ActivityStreams/Types/ASImage.cs | 2 +- .../Federation/ActivityStreams/Types/ASTag.cs | 40 +++++++++++---- .../Core/Services/NoteService.cs | 51 +++++++++++++++++++ 9 files changed, 127 insertions(+), 18 deletions(-) create mode 100644 Iceshrimp.Backend/Controllers/Mastodon/Schemas/Entities/EmojiEntity.cs diff --git a/Iceshrimp.Backend/Controllers/Mastodon/Renderers/NoteRenderer.cs b/Iceshrimp.Backend/Controllers/Mastodon/Renderers/NoteRenderer.cs index d70ae027..83d3d5ff 100644 --- a/Iceshrimp.Backend/Controllers/Mastodon/Renderers/NoteRenderer.cs +++ b/Iceshrimp.Backend/Controllers/Mastodon/Renderers/NoteRenderer.cs @@ -19,17 +19,17 @@ public class NoteRenderer( public async Task RenderAsync( Note note, User? user, List? accounts = null, List? mentions = null, List? attachments = null, Dictionary? likeCounts = null, - List? likedNotes = null, List? renotes = null, int recurse = 2 + List? likedNotes = null, List? renotes = null, List? emoji = null, int recurse = 2 ) { var uri = note.Uri ?? note.GetPublicUri(config.Value); var renote = note is { Renote: not null, IsQuote: false } && recurse > 0 ? await RenderAsync(note.Renote, user, accounts, mentions, attachments, likeCounts, likedNotes, - renotes, 0) + renotes, emoji, 0) : null; var quote = note is { Renote: not null, IsQuote: true } && recurse > 0 ? await RenderAsync(note.Renote, user, accounts, mentions, attachments, likeCounts, likedNotes, - renotes, --recurse) + renotes, emoji, --recurse) : null; var text = note.Text; if (note is { Renote: not null, IsQuote: true } && text != null) @@ -44,6 +44,8 @@ public class NoteRenderer( var renoted = renotes?.Contains(note.Id) ?? await db.Notes.AnyAsync(p => p.Renote == note && p.User == user && p.IsPureRenote); + var noteEmoji = emoji?.Where(p => note.Emojis.Contains(p.Id)).ToList() ?? await GetEmoji([note]); + if (mentions == null) { mentions = await db.Users.IncludeCommonProperties() @@ -119,7 +121,8 @@ public class NoteRenderer( Text = text, Mentions = mentions, IsPinned = false, - Attachments = attachments + Attachments = attachments, + Emojis = noteEmoji }; return res; @@ -184,6 +187,24 @@ public class NoteRenderer( .ToListAsync(); } + private async Task> GetEmoji(IEnumerable notes) + { + var ids = notes.SelectMany(p => p.Emojis).ToList(); + if (ids.Count == 0) return []; + + return await db.Emojis + .Where(p => ids.Contains(p.Id)) + .Select(p => new EmojiEntity + { + Id = p.Id, + Shortcode = p.Name, + Url = p.PublicUrl, + StaticUrl = p.PublicUrl, //TODO + VisibleInPicker = true + }) + .ToListAsync(); + } + public async Task> RenderManyAsync( IEnumerable notes, User? user, List? accounts = null ) @@ -200,8 +221,9 @@ public class NoteRenderer( var likeCounts = await GetLikeCounts(noteList); var likedNotes = await GetLikedNotes(noteList, user); var renotes = await GetRenotes(noteList, user); + var emoji = await GetEmoji(noteList); return await noteList.Select(p => RenderAsync(p, user, accounts, mentions, attachments, likeCounts, likedNotes, - renotes)) + renotes, emoji)) .AwaitAllAsync(); } } \ No newline at end of file diff --git a/Iceshrimp.Backend/Controllers/Mastodon/Schemas/Entities/EmojiEntity.cs b/Iceshrimp.Backend/Controllers/Mastodon/Schemas/Entities/EmojiEntity.cs new file mode 100644 index 00000000..74bf5e6c --- /dev/null +++ b/Iceshrimp.Backend/Controllers/Mastodon/Schemas/Entities/EmojiEntity.cs @@ -0,0 +1,14 @@ +using J = System.Text.Json.Serialization.JsonPropertyNameAttribute; +using JI = System.Text.Json.Serialization.JsonIgnoreAttribute; + +namespace Iceshrimp.Backend.Controllers.Mastodon.Schemas.Entities; + +public class EmojiEntity +{ + [JI] public required string Id; + [J("shortcode")] public required string Shortcode { get; set; } + [J("static_url")] public required string StaticUrl { get; set; } + [J("url")] public required string Url { get; set; } + [J("visible_in_picker")] public required bool VisibleInPicker { get; set; } + [J("category")] public string? Category { get; set; } +} \ No newline at end of file diff --git a/Iceshrimp.Backend/Controllers/Mastodon/Schemas/Entities/StatusEntity.cs b/Iceshrimp.Backend/Controllers/Mastodon/Schemas/Entities/StatusEntity.cs index cf4360b0..6f72df44 100644 --- a/Iceshrimp.Backend/Controllers/Mastodon/Schemas/Entities/StatusEntity.cs +++ b/Iceshrimp.Backend/Controllers/Mastodon/Schemas/Entities/StatusEntity.cs @@ -43,8 +43,8 @@ public class StatusEntity : IEntity [J("mentions")] public required List Mentions { get; set; } [J("media_attachments")] public required List Attachments { get; set; } + [J("emojis")] public required List Emojis { get; set; } - [J("emojis")] public object[] Emojis => []; //FIXME [J("tags")] public object[] Tags => []; //FIXME [J("reactions")] public object[] Reactions => []; //FIXME [J("filtered")] public object[] Filtered => []; //FIXME diff --git a/Iceshrimp.Backend/Core/Configuration/Constants.cs b/Iceshrimp.Backend/Core/Configuration/Constants.cs index 860e3ced..5345153b 100644 --- a/Iceshrimp.Backend/Core/Configuration/Constants.cs +++ b/Iceshrimp.Backend/Core/Configuration/Constants.cs @@ -6,6 +6,7 @@ public static class Constants public const string W3IdSecurityNs = "https://w3id.org/security"; public const string PurlDcNs = "http://purl.org/dc/terms"; public const string XsdNs = "http://www.w3.org/2001/XMLSchema"; + public const string MastodonNs = "http://joinmastodon.org/ns"; public static readonly string[] SystemUsers = ["instance.actor", "relay.actor"]; public static readonly string[] BrowserSafeMimeTypes = diff --git a/Iceshrimp.Backend/Core/Database/Tables/Emoji.cs b/Iceshrimp.Backend/Core/Database/Tables/Emoji.cs index 711a5c01..177761c4 100644 --- a/Iceshrimp.Backend/Core/Database/Tables/Emoji.cs +++ b/Iceshrimp.Backend/Core/Database/Tables/Emoji.cs @@ -30,7 +30,7 @@ public class Emoji [Column("type")] [StringLength(64)] public string? Type { get; set; } [Column("aliases", TypeName = "character varying(128)[]")] - public List Aliases { get; set; } = null!; + public List Aliases { get; set; } = []; [Column("category")] [StringLength(128)] diff --git a/Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs b/Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs index 1f991246..24d81712 100644 --- a/Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs +++ b/Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs @@ -62,7 +62,6 @@ public class NoteRenderer(IOptions config, MfmConverter var tags = mentions .Select(mention => new ASMention { - Type = $"{Constants.ActivityStreamsNs}#Mention", Name = $"@{mention.Username}@{mention.Host}", Href = new ASObjectBase(mention.Uri) }) diff --git a/Iceshrimp.Backend/Core/Federation/ActivityStreams/Types/ASImage.cs b/Iceshrimp.Backend/Core/Federation/ActivityStreams/Types/ASImage.cs index b18da58f..41ad787b 100644 --- a/Iceshrimp.Backend/Core/Federation/ActivityStreams/Types/ASImage.cs +++ b/Iceshrimp.Backend/Core/Federation/ActivityStreams/Types/ASImage.cs @@ -16,4 +16,4 @@ public class ASImage public bool? Sensitive { get; set; } } -public class ASImageConverter : ASSerializer.ListSingleObjectConverter; \ No newline at end of file +public class ASImageConverter : ASSerializer.ListSingleObjectConverter; diff --git a/Iceshrimp.Backend/Core/Federation/ActivityStreams/Types/ASTag.cs b/Iceshrimp.Backend/Core/Federation/ActivityStreams/Types/ASTag.cs index 95e9e771..30c813d2 100644 --- a/Iceshrimp.Backend/Core/Federation/ActivityStreams/Types/ASTag.cs +++ b/Iceshrimp.Backend/Core/Federation/ActivityStreams/Types/ASTag.cs @@ -3,6 +3,7 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using J = Newtonsoft.Json.JsonPropertyAttribute; using JC = Newtonsoft.Json.JsonConverterAttribute; +using VC = Iceshrimp.Backend.Core.Federation.ActivityStreams.Types.ValueObjectConverter; namespace Iceshrimp.Backend.Core.Federation.ActivityStreams.Types; @@ -20,17 +21,35 @@ public class ASTagLink : ASTag public ASObjectBase? Href { get; set; } [J($"{Constants.ActivityStreamsNs}#name")] - [JC(typeof(ValueObjectConverter))] + [JC(typeof(VC))] public string? Name { get; set; } } -public class ASMention : ASTagLink; +public class ASMention : ASTagLink +{ + public ASMention() => Type = $"{Constants.ActivityStreamsNs}#Mention"; +} -public class ASHashtag : ASTagLink; +public class ASHashtag : ASTagLink +{ + public ASHashtag() => Type = $"{Constants.ActivityStreamsNs}#Hashtag"; +} public class ASEmoji : ASTag { - //TODO + public ASEmoji() => Type = $"{Constants.MastodonNs}#Emoji"; + + [J($"{Constants.ActivityStreamsNs}#updated")] + [JC(typeof(VC))] + public DateTime? Updated { get; set; } + + [J($"{Constants.ActivityStreamsNs}#icon")] + [JC(typeof(ASImageConverter))] + public ASImage? Image { get; set; } + + [J($"{Constants.ActivityStreamsNs}#name")] + [JC(typeof(VC))] + public string? Name { get; set; } } public sealed class ASTagConverter : JsonConverter @@ -73,12 +92,15 @@ public sealed class ASTagConverter : JsonConverter private ASTag? HandleObject(JToken obj) { - var link = obj.ToObject(); - if (link is not { Href: not null }) return obj.ToObject(); + var tag = obj.ToObject(); - return link.Type == $"{Constants.ActivityStreamsNs}#Mention" - ? obj.ToObject() - : obj.ToObject(); + return tag?.Type switch + { + $"{Constants.ActivityStreamsNs}#Mention" => obj.ToObject(), + $"{Constants.ActivityStreamsNs}#Hashtag" => obj.ToObject(), + $"{Constants.MastodonNs}#Emoji" => obj.ToObject(), + _ => null + }; } public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer) diff --git a/Iceshrimp.Backend/Core/Services/NoteService.cs b/Iceshrimp.Backend/Core/Services/NoteService.cs index 7ccdb114..f3506cfa 100644 --- a/Iceshrimp.Backend/Core/Services/NoteService.cs +++ b/Iceshrimp.Backend/Core/Services/NoteService.cs @@ -1,4 +1,5 @@ using System.Diagnostics.CodeAnalysis; +using AsyncKeyedLock; using Iceshrimp.Backend.Core.Configuration; using Iceshrimp.Backend.Core.Database; using Iceshrimp.Backend.Core.Database.Tables; @@ -43,6 +44,12 @@ public class NoteService( private readonly List _resolverHistory = []; private int _recursionLimit = 100; + private static readonly AsyncKeyedLocker KeyedLocker = new(o => + { + o.PoolSize = 100; + o.PoolInitialFill = 5; + }); + public async Task CreateNoteAsync( User user, Note.NoteVisibility visibility, string? text = null, string? cw = null, Note? reply = null, Note? renote = null, IReadOnlyCollection? attachments = null @@ -313,6 +320,8 @@ public class NoteService( throw GracefulException.UnprocessableEntity("User.Uri doesn't match Note.AttributedTo"); if (actor.Uri == null) throw GracefulException.UnprocessableEntity("User.Uri is null"); + if (actor.Host == null) + throw GracefulException.UnprocessableEntity("User.Host is null"); if (new Uri(note.Id).IdnHost != new Uri(actor.Uri).IdnHost) throw GracefulException.UnprocessableEntity("User.Uri host doesn't match Note.Id host"); if (!note.Id.StartsWith("https://")) @@ -398,6 +407,9 @@ public class NoteService( dbNote.AttachedFileTypes = files.Select(p => p.Type).ToList(); } + var emoji = await ProcessEmojiAsync(note.Tags?.OfType().ToList(), actor.Host); + dbNote.Emojis = emoji.Select(p => p.Id).ToList(); + actor.NotesCount++; if (dbNote.Reply != null) dbNote.Reply.RepliesCount++; await db.Notes.AddAsync(dbNote); @@ -570,6 +582,45 @@ public class NoteService( return (userIds, localUserIds, mentions, remoteMentions, splitDomainMapping); } + private async Task> ProcessEmojiAsync(List? emoji, string host) + { + emoji?.RemoveAll(p => p.Name == null); + if (emoji is not { Count: > 0 }) return []; + + foreach (var emojo in emoji) emojo.Name = emojo.Name?.Trim(':'); + + var known = await db.Emojis.Where(p => p.Host == host && emoji.Select(e => e.Name).Contains(p.Name)) + .ToListAsync(); + + //TODO: handle updated emoji + foreach (var emojo in emoji.Where(emojo => known.All(p => p.Name != emojo.Name))) + { + using (await KeyedLocker.LockAsync($"emoji:{host}:{emojo.Name}")) + { + var dbEmojo = await db.Emojis.FirstOrDefaultAsync(p => p.Host == host && p.Name == emojo.Name); + if (dbEmojo == null) + { + dbEmojo = new Emoji + { + Id = IdHelpers.GenerateSlowflakeId(), + Host = host, + Name = emojo.Name ?? throw new Exception("emojo.Name must not be null at this stage"), + UpdatedAt = DateTime.UtcNow, + OriginalUrl = emojo.Image?.Url?.Link ?? throw new Exception("Emoji.Image has no url"), + PublicUrl = emojo.Image.Url.Link, + Uri = emojo.Id + }; + await db.AddAsync(dbEmojo); + await db.SaveChangesAsync(); + } + + known.Add(dbEmojo); + } + } + + return known; + } + private async Task> ProcessAttachmentsAsync( List? attachments, User user, bool sensitive )