[backend/federation] Handle mentions in non-misskey user bios & user fields correctly (ISH-92)

Iceshrimp.Backend/Core/Federation/ActivityPub/UserRenderer.cs

@@ -10,7 +10,7 @@ using Microsoft.Extensions.Options;
 
 namespace Iceshrimp.Backend.Core.Federation.ActivityPub;
 
-public class UserRenderer(IOptions<Config.InstanceSection> config, DatabaseContext db, MfmConverter mfmConverter)
+public class UserRenderer(IOptions<Config.InstanceSection> config, DatabaseContext db)
 {
 	/// <summary>
 	///     This function is meant for compacting an actor into the @id form as specified in ActivityStreams
@@ -58,7 +58,7 @@ public class UserRenderer(IOptions<Config.InstanceSection> config, DatabaseConte
 			Url = new ASLink(user.GetPublicUrl(config.Value)),
 			Username = user.Username,
 			DisplayName = user.DisplayName ?? user.Username,
-			Summary = profile?.Description != null ? await mfmConverter.FromHtmlAsync(profile.Description) : null,
+			Summary = profile?.Description != null ? await MfmConverter.FromHtmlAsync(profile.Description) : null,
 			MkSummary = profile?.Description,
 			IsCat = user.IsCat,
 			IsDiscoverable = user.IsExplorable,

Iceshrimp.Backend/Core/Federation/ActivityPub/UserResolver.cs

@@ -1,8 +1,10 @@
 using AsyncKeyedLock;
+using Iceshrimp.Backend.Core.Configuration;
 using Iceshrimp.Backend.Core.Database.Tables;
 using Iceshrimp.Backend.Core.Federation.WebFinger;
 using Iceshrimp.Backend.Core.Middleware;
 using Iceshrimp.Backend.Core.Services;
+using Microsoft.Extensions.Options;
 
 namespace Iceshrimp.Backend.Core.Federation.ActivityPub;
 
@@ -10,7 +12,8 @@ public class UserResolver(
 	ILogger<UserResolver> logger,
 	UserService userSvc,
 	WebFingerService webFingerSvc,
-	FollowupTaskService followupTaskSvc
+	FollowupTaskService followupTaskSvc,
+	IOptions<Config.InstanceSection> config
 )
 {
 	private static readonly AsyncKeyedLocker<string> KeyedLocker = new(o =>
@@ -162,6 +165,33 @@ public class UserResolver(
 		}
 	}
 
+	public async Task<User?> ResolveAsyncLimited(string uri, Func<bool> limitReached)
+	{
+		// First, let's see if we already know the user
+		var user = await userSvc.GetUserFromQueryAsync(uri);
+		if (user != null)
+			return await GetUpdatedUser(user);
+
+		if (uri.StartsWith($"https://{config.Value.WebDomain}/")) return null;
+
+		// We don't, so we need to run WebFinger
+		var (acct, resolvedUri) = await WebFingerAsync(uri);
+
+		// Check the database again with the new data
+		if (resolvedUri != uri) user = await userSvc.GetUserFromQueryAsync(resolvedUri);
+		if (user == null && acct != uri) await userSvc.GetUserFromQueryAsync(acct);
+		if (user != null)
+			return await GetUpdatedUser(user);
+
+		if (limitReached()) return null;
+
+		using (await KeyedLocker.LockAsync(resolvedUri))
+		{
+			// Pass the job on to userSvc, which will create the user
+			return await userSvc.CreateUserAsync(resolvedUri, acct);
+		}
+	}
+
 	private async Task<User> GetUpdatedUser(User user)
 	{
 		if (!user.NeedsUpdate) return user;

Iceshrimp.Backend/Core/Helpers/LibMfm/Conversion/MfmConverter.cs

@@ -17,7 +17,7 @@ namespace Iceshrimp.Backend.Core.Helpers.LibMfm.Conversion;
 
 public class MfmConverter(IOptions<Config.InstanceSection> config)
 {
-	public async Task<string?> FromHtmlAsync(string? html, List<Note.MentionedUser>? mentions = null)
+	public static async Task<string?> FromHtmlAsync(string? html, List<Note.MentionedUser>? mentions = null)
 	{
 		if (html == null) return null;
 
@@ -34,6 +34,24 @@ public class MfmConverter(IOptions<Config.InstanceSection> config)
 		return sb.ToString().Trim();
 	}
 
+	public static async Task<List<string>> ExtractMentionsFromHtmlAsync(string? html)
+	{
+		if (html == null) return [];
+
+		// Ensure compatibility with AP servers that send both <br> as well as newlines
+		var regex = new Regex(@"<br\s?\/?>\r?\n", RegexOptions.IgnoreCase);
+		html = regex.Replace(html, "\n");
+
+		var dom = await new HtmlParser().ParseDocumentAsync(html);
+		if (dom.Body == null) return [];
+
+		var parser = new HtmlMentionsExtractor();
+		foreach (var node in dom.Body.ChildNodes)
+			parser.ParseChildren(node);
+
+		return parser.Mentions;
+	}
+
 	public async Task<string> ToHtmlAsync(IEnumerable<MfmNode> nodes, List<Note.MentionedUser> mentions, string? host)
 	{
 		var context  = BrowsingContext.New();

Iceshrimp.Backend/Core/Helpers/LibMfm/Parsing/HtmlMentionsExtractor.cs

@@ -0,0 +1,53 @@
+using AngleSharp.Dom;
+using AngleSharp.Html.Dom;
+
+namespace Iceshrimp.Backend.Core.Helpers.LibMfm.Parsing;
+
+internal class HtmlMentionsExtractor
+{
+	internal List<string> Mentions { get; } = [];
+
+	private void ParseNode(INode node)
+	{
+		if (node.NodeType is NodeType.Text)
+			return;
+		if (node.NodeType is NodeType.Comment or NodeType.Document)
+			return;
+
+		switch (node.NodeName)
+		{
+			case "A":
+			{
+				if (node is not HtmlElement el) return;
+				var href = el.GetAttribute("href");
+				if (href == null) return;
+				if (el.ClassList.Contains("u-url") && el.ClassList.Contains("mention"))
+					Mentions.Add(href);
+				return;
+			}
+			case "PRE":
+			{
+				if (node.ChildNodes is [{ NodeName: "CODE" }])
+					return;
+				ParseChildren(node);
+				return;
+			}
+			case "BR":
+			case "BLOCKQUOTE":
+			{
+				return;
+			}
+
+			default:
+			{
+				ParseChildren(node);
+				return;
+			}
+		}
+	}
+
+	internal void ParseChildren(INode node)
+	{
+		foreach (var child in node.ChildNodes) ParseNode(child);
+	}
+}

Iceshrimp.Backend/Core/Services/NoteService.cs

@@ -33,7 +33,6 @@ public class NoteService(
 	ActivityPub.NoteRenderer noteRenderer,
 	ActivityPub.UserRenderer userRenderer,
 	ActivityPub.MentionsResolver mentionsResolver,
-	MfmConverter mfmConverter,
 	DriveService driveSvc,
 	NotificationService notificationSvc,
 	EventService eventSvc,
@@ -342,7 +341,7 @@ public class NoteService(
 			Id         = IdHelpers.GenerateSlowflakeId(createdAt),
 			Uri        = note.Id,
 			Url        = note.Url?.Id, //FIXME: this doesn't seem to work yet
-			Text       = note.MkContent ?? await mfmConverter.FromHtmlAsync(note.Content, mentions),
+			Text       = note.MkContent ?? await MfmConverter.FromHtmlAsync(note.Content, mentions),
 			Cw         = note.Summary,
 			UserId     = actor.Id,
 			CreatedAt  = createdAt,
@@ -446,7 +445,7 @@ public class NoteService(
 			await ResolveNoteMentionsAsync(note);
 
 		mentionedLocalUserIds = mentionedLocalUserIds.Except(previousMentionedLocalUserIds).ToList();
-		dbNote.Text           = note.MkContent ?? await mfmConverter.FromHtmlAsync(note.Content, mentions);
+		dbNote.Text           = note.MkContent ?? await MfmConverter.FromHtmlAsync(note.Content, mentions);
 		dbNote.Cw             = note.Summary;
 
 		if (dbNote.Cw is { Length: > 100000 })

Iceshrimp.Backend/Core/Services/UserProfileMentionsResolver.cs

@@ -2,6 +2,8 @@ using System.Diagnostics.CodeAnalysis;
 using Iceshrimp.Backend.Core.Configuration;
 using Iceshrimp.Backend.Core.Database.Tables;
 using Iceshrimp.Backend.Core.Extensions;
+using Iceshrimp.Backend.Core.Federation.ActivityStreams.Types;
+using Iceshrimp.Backend.Core.Helpers.LibMfm.Conversion;
 using Iceshrimp.Backend.Core.Helpers.LibMfm.Parsing;
 using Iceshrimp.Backend.Core.Helpers.LibMfm.Types;
 using Microsoft.Extensions.Options;
@@ -10,13 +12,82 @@ namespace Iceshrimp.Backend.Core.Services;
 
 public class UserProfileMentionsResolver(
 	ActivityPub.UserResolver userResolver,
-	IOptions<Config.InstanceSection> config,
+	IOptions<Config.InstanceSection> config
-	ILogger<UserProfileMentionsResolver> logger
 )
 {
 	private int _recursionLimit = 10;
 
-	public async Task<List<Note.MentionedUser>> ResolveMentions(UserProfile.Field[]? fields, string? bio, string? host)
+	public async Task<List<Note.MentionedUser>> ResolveMentions(
+		ASActor actor, string? host
+	)
+	{
+		var fields = actor.Attachments?.OfType<ASField>()
+		                  .Where(p => p is { Name: not null, Value: not null })
+		                  .ToList() ?? [];
+
+		if (fields is not { Count: > 0 } && (actor.MkSummary ?? actor.Summary) == null) return [];
+		var parsedFields = await fields.SelectMany<ASField, string?>(p => [p.Name, p.Value])
+		                               .Select(async p => await MfmConverter.ExtractMentionsFromHtmlAsync(p))
+		                               .AwaitAllAsync();
+
+		var parsedBio = actor.MkSummary == null ? await MfmConverter.ExtractMentionsFromHtmlAsync(actor.Summary) : [];
+
+		var userUris     = parsedFields.Prepend(parsedBio).SelectMany(p => p).ToList();
+		var mentionNodes = new List<MfmMentionNode>();
+
+		if (actor.MkSummary != null)
+		{
+			var nodes = MfmParser.Parse(actor.MkSummary);
+			mentionNodes = EnumerateMentions(nodes);
+		}
+
+		var users = await mentionNodes
+		                  .DistinctBy(p => p.Acct)
+		                  .Select(async p =>
+		                  {
+			                  try
+			                  {
+				                  return await userResolver.ResolveAsyncLimited(p.Username, p.Host ?? host,
+					                                                                () => _recursionLimit-- <= 0);
+			                  }
+			                  catch
+			                  {
+				                  return null;
+			                  }
+		                  })
+		                  .AwaitAllNoConcurrencyAsync();
+
+		users.AddRange(await userUris
+		                     .Distinct()
+		                     .Select(async p =>
+		                     {
+			                     try
+			                     {
+				                     return await userResolver.ResolveAsyncLimited(p, () => _recursionLimit-- <= 0);
+			                     }
+			                     catch
+			                     {
+				                     return null;
+			                     }
+		                     })
+		                     .AwaitAllNoConcurrencyAsync());
+
+		return users.Where(p => p != null)
+		            .Cast<User>()
+		            .DistinctBy(p => p.Id)
+		            .Select(p => new Note.MentionedUser
+		            {
+			            Host     = p.Host,
+			            Uri      = p.Uri ?? p.GetPublicUri(config.Value),
+			            Url      = p.UserProfile?.Url,
+			            Username = p.Username
+		            })
+		            .ToList();
+	}
+
+	public async Task<List<Note.MentionedUser>> ResolveMentions(
+		UserProfile.Field[]? fields, string? bio, string? host
+	)
 	{
 		if (fields is not { Length: > 0 } && bio == null) return [];
 		var input = (fields ?? [])

Iceshrimp.Backend/Core/Services/UserService.cs

@@ -25,7 +25,6 @@ public class UserService(
 	ActivityPub.ActivityRenderer activityRenderer,
 	ActivityPub.ActivityDeliverService deliverSvc,
 	DriveService driveSvc,
-	MfmConverter mfmConverter,
 	FollowupTaskService followupTaskSvc,
 	NotificationService notificationSvc,
 	EmojiService emojiSvc
@@ -108,12 +107,12 @@ public class UserService(
 			             .Where(p => p is { Name: not null, Value: not null })
 			             .Select(async p => new UserProfile.Field
 			             {
-				             Name = p.Name!, Value = await mfmConverter.FromHtmlAsync(p.Value) ?? ""
+				             Name = p.Name!, Value = await MfmConverter.FromHtmlAsync(p.Value) ?? ""
 			             })
 			             .AwaitAllAsync()
 			: null;
 
-		var bio = actor.MkSummary ?? await mfmConverter.FromHtmlAsync(actor.Summary);
+		var bio = actor.MkSummary ?? await MfmConverter.FromHtmlAsync(actor.Summary);
 
 		user = new User
 		{
@@ -165,7 +164,7 @@ public class UserService(
 			var processPendingDeletes = await ResolveAvatarAndBanner(user, actor);
 			await db.SaveChangesAsync();
 			await processPendingDeletes();
-			await UpdateProfileMentionsInBackground(user);
+			await UpdateProfileMentionsInBackground(user, actor);
 			return user;
 		}
 		catch (UniqueConstraintException)
@@ -246,7 +245,7 @@ public class UserService(
 			             .Where(p => p is { Name: not null, Value: not null })
 			             .Select(async p => new UserProfile.Field
 			             {
-				             Name = p.Name!, Value = await mfmConverter.FromHtmlAsync(p.Value) ?? ""
+				             Name = p.Name!, Value = await MfmConverter.FromHtmlAsync(p.Value) ?? ""
 			             })
 			             .AwaitAllAsync()
 			: null;
@@ -260,7 +259,7 @@ public class UserService(
 
 		var processPendingDeletes = await ResolveAvatarAndBanner(user, actor);
 
-		user.UserProfile.Description = actor.MkSummary ?? await mfmConverter.FromHtmlAsync(actor.Summary);
+		user.UserProfile.Description = actor.MkSummary ?? await MfmConverter.FromHtmlAsync(actor.Summary);
 		//user.UserProfile.Birthday = TODO;
 		//user.UserProfile.Location = TODO;
 		user.UserProfile.Fields   = fields?.ToArray() ?? [];
@@ -270,7 +269,7 @@ public class UserService(
 		db.Update(user);
 		await db.SaveChangesAsync();
 		await processPendingDeletes();
-		await UpdateProfileMentionsInBackground(user);
+		await UpdateProfileMentionsInBackground(user, actor);
 		return user;
 	}
 
@@ -605,7 +604,7 @@ public class UserService(
 	[SuppressMessage("ReSharper", "EntityFramework.NPlusOne.IncompleteDataQuery", Justification = "Projectables")]
 	[SuppressMessage("ReSharper", "EntityFramework.NPlusOne.IncompleteDataUsage", Justification = "Same as above")]
 	[SuppressMessage("ReSharper", "SuggestBaseTypeForParameter", Justification = "Method only makes sense for users")]
-	private async Task UpdateProfileMentionsInBackground(User user)
+	private async Task UpdateProfileMentionsInBackground(User user, ASActor? actor)
 	{
 		var task = followupTaskSvc.ExecuteTask("UpdateProfileMentionsInBackground", async provider =>
 		{
@@ -614,9 +613,35 @@ public class UserService(
 				.GetRequiredService<UserProfileMentionsResolver>();
 			var bgUser = await bgDbContext.Users.IncludeCommonProperties().FirstOrDefaultAsync(p => p.Id == user.Id);
 			if (bgUser?.UserProfile == null) return;
-			bgUser.UserProfile.Mentions =
+
-				await bgMentionsResolver.ResolveMentions(bgUser.UserProfile.Fields, bgUser.UserProfile.Description,
+			if (actor != null)
-				                                         bgUser.Host);
+			{
+				var mentions = await bgMentionsResolver.ResolveMentions(actor, bgUser.Host);
+				var fields = actor.Attachments != null
+					? await actor.Attachments
+					             .OfType<ASField>()
+					             .Where(p => p is { Name: not null, Value: not null })
+					             .Select(async p => new UserProfile.Field
+					             {
+						             Name  = p.Name!,
+						             Value = await MfmConverter.FromHtmlAsync(p.Value, mentions) ?? ""
+					             })
+					             .AwaitAllAsync()
+					: null;
+
+				bgUser.UserProfile.Mentions = mentions;
+				bgUser.UserProfile.Fields   = fields?.ToArray() ?? [];
+				bgUser.UserProfile.Description = actor.MkSummary ??
+				                                 await MfmConverter.FromHtmlAsync(actor.Summary,
+					                                 bgUser.UserProfile.Mentions);
+			}
+			else
+			{
+				bgUser.UserProfile.Mentions = await bgMentionsResolver.ResolveMentions(bgUser.UserProfile.Fields,
+					bgUser.UserProfile.Description,
+					bgUser.Host);
+			}
+
 			bgDbContext.Update(bgUser.UserProfile);
 			await bgDbContext.SaveChangesAsync();
 		});