From 89060599eba7dc61bd2b46d555d92f43b73391e1 Mon Sep 17 00:00:00 2001
From: Kopper <kopper@w.on-t.work>
Date: Sat, 23 Nov 2024 12:13:51 +0300
Subject: [PATCH] [backend] Implement inline media

Inline media can be created by:

1. Attach media to note as usual
2. Copy media URL (public one, for remote instances)
3. Use the new $[media url ] MFM extension to place it wherever you
   wish. (The trailing space is necessary as the parser currently
   treats the closing ] as a part of the URL)

The Iceshrimp frontend may make this easier later on (by having a
"copy inline MFM" button on attachments, maybe?)

Federates as <img>, <video>, <audio>, or <a download> HTML tags
depending on the media type for interoperability. (<a download> is
not handled for incoming media yet).

The media will also be present in the attachments field, both as a
fallback for instance software that do not support inline media,
but also for MFM federation to discover which media it is allowed to
embed (and metadata like alt text and sensitive-ness). This is not
required for remote instances sending inline media, as it will be
extracted out from the HTML.

The Iceshrimp frontend does not render inline media yet. That is
blocked on #67.
---
 .../PublicPreview/Renderers/MfmRenderer.cs    |   2 +-
 .../Mastodon/AnnouncementController.cs        |   2 +-
 .../Mastodon/Renderers/NoteRenderer.cs        |  57 +++++++--
 .../Mastodon/Renderers/UserRenderer.cs        |   4 +-
 .../Federation/ActivityPub/NoteRenderer.cs    |  40 ++++---
 .../Federation/ActivityPub/UserRenderer.cs    |   2 +-
 .../Helpers/LibMfm/Conversion/MfmConverter.cs | 109 ++++++++++++++----
 .../Core/Helpers/LibMfm/Parsing/HtmlParser.cs |  59 +++++++---
 .../Core/Services/NoteService.cs              |  49 ++++++--
 .../Core/Services/UserService.cs              |  12 +-
 10 files changed, 246 insertions(+), 90 deletions(-)

diff --git a/Iceshrimp.Backend/Components/PublicPreview/Renderers/MfmRenderer.cs b/Iceshrimp.Backend/Components/PublicPreview/Renderers/MfmRenderer.cs
index 5b0b3117..cf10105d 100644
--- a/Iceshrimp.Backend/Components/PublicPreview/Renderers/MfmRenderer.cs
+++ b/Iceshrimp.Backend/Components/PublicPreview/Renderers/MfmRenderer.cs
@@ -18,7 +18,7 @@ public class MfmRenderer(MfmConverter converter) : ISingletonService
 		// Ensure we are rendering HTML markup (AsyncLocal)
 		converter.SupportsHtmlFormatting.Value = true;
 
-		var serialized = await converter.ToHtmlAsync(parsed, mentions, host, emoji: emoji, rootElement: rootElement);
+		var serialized = (await converter.ToHtmlAsync(parsed, mentions, host, emoji: emoji, rootElement: rootElement)).Html;
 		return new MarkupString(serialized);
 	}
 }
\ No newline at end of file
diff --git a/Iceshrimp.Backend/Controllers/Mastodon/AnnouncementController.cs b/Iceshrimp.Backend/Controllers/Mastodon/AnnouncementController.cs
index e993ac12..ff12f6d1 100644
--- a/Iceshrimp.Backend/Controllers/Mastodon/AnnouncementController.cs
+++ b/Iceshrimp.Backend/Controllers/Mastodon/AnnouncementController.cs
@@ -55,7 +55,7 @@ public class AnnouncementController(DatabaseContext db, MfmConverter mfmConverte
 		                             })
 		                             .ToListAsync();
 
-		await res.Select(async p => p.Content = await mfmConverter.ToHtmlAsync(p.Content, [], null)).AwaitAllAsync();
+		await res.Select(async p => p.Content = (await mfmConverter.ToHtmlAsync(p.Content, [], null)).Html).AwaitAllAsync();
 		return res;
 	}
 
diff --git a/Iceshrimp.Backend/Controllers/Mastodon/Renderers/NoteRenderer.cs b/Iceshrimp.Backend/Controllers/Mastodon/Renderers/NoteRenderer.cs
index 767e2a59..70533f5b 100644
--- a/Iceshrimp.Backend/Controllers/Mastodon/Renderers/NoteRenderer.cs
+++ b/Iceshrimp.Backend/Controllers/Mastodon/Renderers/NoteRenderer.cs
@@ -81,6 +81,9 @@ public class NoteRenderer(
 			? await GetAttachmentsAsync([note])
 			: [..data.Attachments.Where(p => note.FileIds.Contains(p.Id))];
 
+		if (user == null && security.Value.PublicPreview == Enums.PublicPreview.RestrictedNoMedia) //TODO
+			attachments = [];
+
 		var reactions = data?.Reactions == null
 			? await GetReactionsAsync([note], user)
 			: [..data.Reactions.Where(p => p.NoteId == note.Id)];
@@ -99,13 +102,32 @@ public class NoteRenderer(
 		var quoteInaccessible =
 			note.Renote == null && ((note.RenoteId != null && recurse > 0) || note.RenoteUri != null);
 
-		var content = data?.Source != true
-			? text != null || quoteUri != null || quoteInaccessible || replyInaccessible
-				? await mfmConverter.ToHtmlAsync(text ?? "", mentionedUsers, note.UserHost, quoteUri,
-				                                 quoteInaccessible, replyInaccessible)
-				: ""
-			: null;
+		var sensitive = note.Cw != null || attachments.Any(p => p.Sensitive);
+		
+		// TODO: canDisplayInlineMedia oauth_token flag that marks all media as "other" so they end up as links
+		// (and doesn't remove the attachments)
+		var inlineMedia = attachments.Select(p => new MfmInlineMedia(p.Type switch
+		{
+			AttachmentType.Audio                       => MfmInlineMedia.MediaType.Audio,
+			AttachmentType.Video                       => MfmInlineMedia.MediaType.Video,
+			AttachmentType.Image or AttachmentType.Gif => MfmInlineMedia.MediaType.Image,
+			_                                          => MfmInlineMedia.MediaType.Other
+		}, p.RemoteUrl ?? p.Url, p.Description)).ToList();
+		
+		string? content = null;
+		if (data?.Source != true)
+			if (text != null || quoteUri != null || quoteInaccessible || replyInaccessible)
+			{
+				(content, inlineMedia) = await mfmConverter.ToHtmlAsync(text ?? "", mentionedUsers, note.UserHost, quoteUri,
+				                                          quoteInaccessible, replyInaccessible, media: inlineMedia);
 
+				attachments.RemoveAll(attachment => inlineMedia.Any(inline => inline.Src == (attachment.RemoteUrl ?? attachment.Url)));
+			}
+			else
+			{
+				content = "";
+			}
+		
 		var account = data?.Accounts?.FirstOrDefault(p => p.Id == note.UserId) ??
 		              await userRenderer.RenderAsync(note.User, user);
 
@@ -130,9 +152,6 @@ public class NoteRenderer(
 		if ((user?.UserSettings?.FilterInaccessible ?? false) && (replyInaccessible || quoteInaccessible))
 			filterResult.Insert(0, InaccessibleFilter);
 
-		if (user == null && security.Value.PublicPreview == Enums.PublicPreview.RestrictedNoMedia) //TODO
-			attachments = [];
-
 		var res = new StatusEntity
 		{
 			Id               = note.Id,
@@ -155,7 +174,7 @@ public class NoteRenderer(
 			IsRenoted        = renoted,
 			IsBookmarked     = bookmarked,
 			IsMuted          = muted,
-			IsSensitive      = note.Cw != null || attachments.Any(p => p.Sensitive),
+			IsSensitive      = sensitive,
 			ContentWarning   = note.Cw ?? "",
 			Visibility       = StatusEntity.EncodeVisibility(note.Visibility),
 			Content          = content,
@@ -195,11 +214,25 @@ public class NoteRenderer(
 		List<StatusEdit> history = [];
 		foreach (var edit in edits)
 		{
-			var files = attachments.Where(p => edit.FileIds.Contains(p.Id)).ToList();
+			var files   = attachments.Where(p => edit.FileIds.Contains(p.Id)).ToList();
+		
+			// TODO: canDisplayInlineMedia oauth_token flag that marks all media as "other" so they end up as links
+			// (and doesn't remove the attachments)
+			var inlineMedia = files.Select(p => new MfmInlineMedia(p.Type switch
+			{
+				AttachmentType.Audio                       => MfmInlineMedia.MediaType.Audio,
+				AttachmentType.Video                       => MfmInlineMedia.MediaType.Video,
+				AttachmentType.Image or AttachmentType.Gif => MfmInlineMedia.MediaType.Image,
+				_                                          => MfmInlineMedia.MediaType.Other
+			}, p.RemoteUrl ?? p.Url, p.Description)).ToList();
+
+			(var content, inlineMedia) = await mfmConverter.ToHtmlAsync(edit.Text ?? "", mentionedUsers, note.UserHost, media: inlineMedia);
+			files.RemoveAll(attachment => inlineMedia.Any(inline => inline.Src == (attachment.RemoteUrl ?? attachment.Url)));
+		
 			var entry = new StatusEdit
 			{
 				Account        = account,
-				Content        = await mfmConverter.ToHtmlAsync(edit.Text ?? "", mentionedUsers, note.UserHost),
+				Content        = content,
 				CreatedAt      = lastDate.ToStringIso8601Like(),
 				Emojis         = [],
 				IsSensitive    = files.Any(p => p.Sensitive),
diff --git a/Iceshrimp.Backend/Controllers/Mastodon/Renderers/UserRenderer.cs b/Iceshrimp.Backend/Controllers/Mastodon/Renderers/UserRenderer.cs
index 249d6d5c..9a49b7b9 100644
--- a/Iceshrimp.Backend/Controllers/Mastodon/Renderers/UserRenderer.cs
+++ b/Iceshrimp.Backend/Controllers/Mastodon/Renderers/UserRenderer.cs
@@ -33,7 +33,7 @@ public class UserRenderer(
 			               .Select(async p => new Field
 			               {
 				               Name  = p.Name,
-				               Value = await mfmConverter.ToHtmlAsync(p.Value, mentions, user.Host),
+				               Value = (await mfmConverter.ToHtmlAsync(p.Value, mentions, user.Host)).Html,
 				               VerifiedAt = p.IsVerified.HasValue && p.IsVerified.Value
 					               ? DateTime.Now.ToStringIso8601Like()
 					               : null
@@ -58,7 +58,7 @@ public class UserRenderer(
 			FollowersCount     = user.FollowersCount,
 			FollowingCount     = user.FollowingCount,
 			StatusesCount      = user.NotesCount,
-			Note               = await mfmConverter.ToHtmlAsync(profile?.Description ?? "", mentions, user.Host),
+			Note               = (await mfmConverter.ToHtmlAsync(profile?.Description ?? "", mentions, user.Host)).Html,
 			Url                = profile?.Url ?? user.Uri ?? user.GetPublicUrl(config.Value),
 			Uri                = user.Uri ?? user.GetPublicUri(config.Value),
 			AvatarStaticUrl    = user.AvatarUrl ?? user.GetIdenticonUrlPng(config.Value), //TODO
diff --git a/Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs b/Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs
index 906b4126..91f23fa3 100644
--- a/Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs
+++ b/Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs
@@ -25,7 +25,7 @@ public class NoteRenderer(
 	{
 		return new ASNote(false) { Id = note.Uri ?? note.GetPublicUri(config.Value) };
 	}
-
+	
 	public async Task<ASNote> RenderAsync(Note note, List<Note.MentionedUser>? mentions = null)
 	{
 		if (note.IsPureRenote)
@@ -109,19 +109,25 @@ public class NoteRenderer(
 		               }))
 		               .ToList();
 
-		var attachments = note.FileIds.Count > 0
-			? await db.DriveFiles
-			          .Where(p => note.FileIds.Contains(p.Id) && p.UserHost == null)
-			          .Select(p => new ASDocument
-			          {
-				          Sensitive   = p.IsSensitive,
-				          Url         = new ASLink(p.AccessUrl),
-				          MediaType   = p.Type,
-				          Description = p.Comment
-			          })
-			          .Cast<ASAttachment>()
-			          .ToListAsync()
-			: null;
+		var driveFiles = note.FileIds.Count > 0
+            			? await db.DriveFiles
+			                      .Where(p => note.FileIds.Contains(p.Id) && p.UserHost == null)
+			                      .ToListAsync()
+						: null;
+
+		var sensitive = note.Cw != null || (driveFiles?.Any(p => p.IsSensitive) ?? false);
+		var attachments = driveFiles?.Select(p => new ASDocument
+		                            {
+			                            Sensitive   = p.IsSensitive,
+			                            Url         = new ASLink(p.AccessUrl),
+			                            MediaType   = p.Type,
+			                            Description = p.Comment
+		                            })
+		                            .Cast<ASAttachment>()
+		                            .ToList();
+		
+		var inlineMedia = driveFiles?.Select(p => new MfmInlineMedia(MfmInlineMedia.GetType(p.Type), p.AccessUrl, p.Comment))
+		                            .ToList();
 
 		var quoteUri = note.IsQuote ? note.Renote?.Uri ?? note.Renote?.GetPublicUriOrNull(config.Value) : null;
 		var text     = quoteUri != null ? note.Text + $"\n\nRE: {quoteUri}" : note.Text;
@@ -137,8 +143,6 @@ public class NoteRenderer(
 			});
 		}
 
-		var sensitive = note.Cw != null || (attachments?.OfType<ASDocument>().Any(p => p.Sensitive == true) ?? false);
-
 		if (note.HasPoll)
 		{
 			var poll = await db.Polls.FirstOrDefaultAsync(p => p.Note == note);
@@ -177,7 +181,7 @@ public class NoteRenderer(
 					To           = to,
 					Tags         = tags,
 					Attachments  = attachments,
-					Content      = text != null ? await mfmConverter.ToHtmlAsync(text, mentions, note.UserHost) : null,
+					Content      = text != null ? (await mfmConverter.ToHtmlAsync(text, mentions, note.UserHost, media: inlineMedia)).Html : null,
 					Summary      = note.Cw,
 					Source = note.Text != null
 						? new ASNoteSource { Content = note.Text, MediaType = "text/x.misskeymarkdown" }
@@ -209,7 +213,7 @@ public class NoteRenderer(
 			To           = to,
 			Tags         = tags,
 			Attachments  = attachments,
-			Content      = text != null ? await mfmConverter.ToHtmlAsync(text, mentions, note.UserHost) : null,
+			Content      = text != null ? (await mfmConverter.ToHtmlAsync(text, mentions, note.UserHost, media: inlineMedia)).Html : null,
 			Summary      = note.Cw,
 			Source = note.Text != null
 				? new ASNoteSource { Content = note.Text, MediaType = "text/x.misskeymarkdown" }
diff --git a/Iceshrimp.Backend/Core/Federation/ActivityPub/UserRenderer.cs b/Iceshrimp.Backend/Core/Federation/ActivityPub/UserRenderer.cs
index d560b5dc..e909d2b9 100644
--- a/Iceshrimp.Backend/Core/Federation/ActivityPub/UserRenderer.cs
+++ b/Iceshrimp.Backend/Core/Federation/ActivityPub/UserRenderer.cs
@@ -71,7 +71,7 @@ public class UserRenderer(
 		                         .ToList();
 
 		var summary = profile?.Description != null
-			? await mfmConverter.ToHtmlAsync(profile.Description, profile.Mentions, user.Host)
+			? (await mfmConverter.ToHtmlAsync(profile.Description, profile.Mentions, user.Host)).Html
 			: null;
 
 		return new ASActor
diff --git a/Iceshrimp.Backend/Core/Helpers/LibMfm/Conversion/MfmConverter.cs b/Iceshrimp.Backend/Core/Helpers/LibMfm/Conversion/MfmConverter.cs
index 81e550fa..ea17cb3f 100644
--- a/Iceshrimp.Backend/Core/Helpers/LibMfm/Conversion/MfmConverter.cs
+++ b/Iceshrimp.Backend/Core/Helpers/LibMfm/Conversion/MfmConverter.cs
@@ -14,15 +14,36 @@ using HtmlParser = AngleSharp.Html.Parser.HtmlParser;
 
 namespace Iceshrimp.Backend.Core.Helpers.LibMfm.Conversion;
 
+public record MfmInlineMedia(MfmInlineMedia.MediaType Type, string Src, string? Alt)
+{
+	public enum MediaType
+	{
+		Other,
+		Image,
+		Video,
+		Audio
+	}
+	
+	public static MediaType GetType(string mime)
+	{
+		if (mime.StartsWith("image/")) return MediaType.Image;
+		if (mime.StartsWith("video/")) return MediaType.Video;
+		if (mime.StartsWith("audio/")) return MediaType.Audio;
+
+		return MediaType.Other;
+	}
+}
+
 public class MfmConverter(
 	IOptions<Config.InstanceSection> config
 ) : ISingletonService
 {
 	public AsyncLocal<bool> SupportsHtmlFormatting { get; } = new();
 
-	public static async Task<string?> FromHtmlAsync(string? html, List<Note.MentionedUser>? mentions = null)
+	public static async Task<(string Mfm, List<MfmInlineMedia> InlineMedia)> FromHtmlAsync(string? html, List<Note.MentionedUser>? mentions = null)
 	{
-		if (html == null) return null;
+		var media = new List<MfmInlineMedia>();
+		if (html == null) return ("", media);
 
 		// Ensure compatibility with AP servers that send both <br> as well as newlines
 		var regex = new Regex(@"<br\s?\/?>\r?\n", RegexOptions.IgnoreCase);
@@ -32,12 +53,12 @@ public class MfmConverter(
 		html = html.Replace("\u00A0", " ");
 
 		var dom = await new HtmlParser().ParseDocumentAsync(html);
-		if (dom.Body == null) return "";
+		if (dom.Body == null) return ("", media);
 
 		var sb     = new StringBuilder();
-		var parser = new MfmHtmlParser(mentions ?? []);
+		var parser = new MfmHtmlParser(mentions ?? [], media);
 		dom.Body.ChildNodes.Select(parser.ParseNode).ToList().ForEach(s => sb.Append(s));
-		return sb.ToString().Trim();
+		return (sb.ToString().Trim(), media);
 	}
 
 	public static async Task<List<string>> ExtractMentionsFromHtmlAsync(string? html)
@@ -58,10 +79,10 @@ public class MfmConverter(
 		return parser.Mentions;
 	}
 
-	public async Task<string> ToHtmlAsync(
+	public async Task<(string Html, List<MfmInlineMedia> InlineMedia)> ToHtmlAsync(
 		IEnumerable<IMfmNode> nodes, List<Note.MentionedUser> mentions, string? host, string? quoteUri = null,
 		bool quoteInaccessible = false, bool replyInaccessible = false, string rootElement = "p",
-		List<Emoji>? emoji = null
+		List<Emoji>? emoji = null, List<MfmInlineMedia>? media = null
 	)
 	{
 		var context    = BrowsingContext.New();
@@ -86,7 +107,8 @@ public class MfmConverter(
 			element.AppendChild(wrapper);
 		}
 
-		foreach (var node in nodeList) element.AppendNodes(FromMfmNode(document, node, mentions, host, emoji));
+		var usedMedia = new List<MfmInlineMedia>();
+		foreach (var node in nodeList) element.AppendNodes(FromMfmNode(document, node, mentions, host, ref usedMedia, emoji, media));
 
 		if (quoteUri != null)
 		{
@@ -126,45 +148,87 @@ public class MfmConverter(
 
 		await using var sw = new StringWriter();
 		await element.ToHtmlAsync(sw);
-		return sw.ToString();
+		return (sw.ToString(), usedMedia);
 	}
 
-	public async Task<string> ToHtmlAsync(
+	public async Task<(string Html, List<MfmInlineMedia> InlineMedia)> ToHtmlAsync(
 		string mfm, List<Note.MentionedUser> mentions, string? host, string? quoteUri = null,
 		bool quoteInaccessible = false, bool replyInaccessible = false, string rootElement = "p",
-		List<Emoji>? emoji = null
+		List<Emoji>? emoji = null, List<MfmInlineMedia>? media = null
 	)
 	{
 		var nodes = MfmParser.Parse(mfm);
 		return await ToHtmlAsync(nodes, mentions, host, quoteUri, quoteInaccessible,
-		                         replyInaccessible, rootElement, emoji);
+		                         replyInaccessible, rootElement, emoji, media);
 	}
 
 	private INode FromMfmNode(
-		IDocument document, IMfmNode node, List<Note.MentionedUser> mentions, string? host, List<Emoji>? emoji = null
+		IDocument document, IMfmNode node, List<Note.MentionedUser> mentions, string? host, ref List<MfmInlineMedia> usedMedia, 
+		List<Emoji>? emoji = null, List<MfmInlineMedia>? media = null
 	)
 	{
 		switch (node)
 		{
+			case MfmFnNode { Name: "media" } fn when media != null:
+			{
+				var urlNode = fn.Children.FirstOrDefault();
+				if (urlNode is MfmUrlNode url)
+				{
+					var current = media.FirstOrDefault(m => m.Src == url.Url);
+					if (current != null)
+					{
+						var nodeName = current.Type switch
+						{
+							MfmInlineMedia.MediaType.Image   => "img",
+							MfmInlineMedia.MediaType.Video   => "video",
+							MfmInlineMedia.MediaType.Audio   => "audio",
+							_                                => "a",
+						};
+						var el = document.CreateElement(nodeName);
+						if (current.Type == MfmInlineMedia.MediaType.Other)
+						{
+							el.SetAttribute("href", current.Src);
+							el.SetAttribute("download", "true");
+							el.TextContent = $"\ud83d\udcbe {current.Alt ?? current.Src}"; // floppy disk emoji
+						}
+						else
+						{
+							el.SetAttribute("src", current.Src);
+							el.SetAttribute("alt", current.Alt);
+						}
+						
+						usedMedia.Add(current);
+						return el;
+					}
+				}
+
+				{
+					var el = CreateInlineFormattingElement(document, "i");
+					AddHtmlMarkup(document, el, "*");
+					AppendChildren(el, document, node, mentions, host, ref usedMedia);
+					AddHtmlMarkup(document, el, "*");
+					return el;
+				}
+			}
 			case MfmBoldNode:
 			{
 				var el = CreateInlineFormattingElement(document, "b");
 				AddHtmlMarkup(document, el, "**");
-				AppendChildren(el, document, node, mentions, host);
+				AppendChildren(el, document, node, mentions, host, ref usedMedia);
 				AddHtmlMarkup(document, el, "**");
 				return el;
 			}
 			case MfmSmallNode:
 			{
 				var el = document.CreateElement("small");
-				AppendChildren(el, document, node, mentions, host);
+				AppendChildren(el, document, node, mentions, host, ref usedMedia);
 				return el;
 			}
 			case MfmStrikeNode:
 			{
 				var el = CreateInlineFormattingElement(document, "del");
 				AddHtmlMarkup(document, el, "~~");
-				AppendChildren(el, document, node, mentions, host);
+				AppendChildren(el, document, node, mentions, host, ref usedMedia);
 				AddHtmlMarkup(document, el, "~~");
 				return el;
 			}
@@ -173,7 +237,7 @@ public class MfmConverter(
 			{
 				var el = CreateInlineFormattingElement(document, "i");
 				AddHtmlMarkup(document, el, "*");
-				AppendChildren(el, document, node, mentions, host);
+				AppendChildren(el, document, node, mentions, host, ref usedMedia);
 				AddHtmlMarkup(document, el, "*");
 				return el;
 			}
@@ -188,7 +252,7 @@ public class MfmConverter(
 			case MfmCenterNode:
 			{
 				var el = document.CreateElement("div");
-				AppendChildren(el, document, node, mentions, host);
+				AppendChildren(el, document, node, mentions, host, ref usedMedia);
 				return el;
 			}
 			case MfmEmojiCodeNode emojiCodeNode:
@@ -278,7 +342,7 @@ public class MfmConverter(
 			{
 				var el = CreateInlineFormattingElement(document, "blockquote");
 				AddHtmlMarkup(document, el, "> ");
-				AppendChildren(el, document, node, mentions, host);
+				AppendChildren(el, document, node, mentions, host, ref usedMedia);
 				el.AppendChild(document.CreateElement("br"));
 				return el;
 			}
@@ -312,7 +376,7 @@ public class MfmConverter(
 			case MfmPlainNode:
 			{
 				var el = document.CreateElement("span");
-				AppendChildren(el, document, node, mentions, host);
+				AppendChildren(el, document, node, mentions, host, ref usedMedia);
 				return el;
 			}
 			default:
@@ -324,10 +388,11 @@ public class MfmConverter(
 
 	private void AppendChildren(
 		INode element, IDocument document, IMfmNode parent,
-		List<Note.MentionedUser> mentions, string? host, List<Emoji>? emoji = null
+		List<Note.MentionedUser> mentions, string? host, ref List<MfmInlineMedia> usedMedia, 
+		List<Emoji>? emoji = null, List<MfmInlineMedia>? media = null
 	)
 	{
-		foreach (var node in parent.Children) element.AppendNodes(FromMfmNode(document, node, mentions, host, emoji));
+		foreach (var node in parent.Children) element.AppendNodes(FromMfmNode(document, node, mentions, host, ref usedMedia, emoji, media));
 	}
 
 	private IElement CreateInlineFormattingElement(IDocument document, string name)
diff --git a/Iceshrimp.Backend/Core/Helpers/LibMfm/Parsing/HtmlParser.cs b/Iceshrimp.Backend/Core/Helpers/LibMfm/Parsing/HtmlParser.cs
index 978e52db..6dbb9ccb 100644
--- a/Iceshrimp.Backend/Core/Helpers/LibMfm/Parsing/HtmlParser.cs
+++ b/Iceshrimp.Backend/Core/Helpers/LibMfm/Parsing/HtmlParser.cs
@@ -1,10 +1,11 @@
 using AngleSharp.Dom;
 using AngleSharp.Html.Dom;
 using Iceshrimp.Backend.Core.Database.Tables;
+using Iceshrimp.Backend.Core.Helpers.LibMfm.Conversion;
 
 namespace Iceshrimp.Backend.Core.Helpers.LibMfm.Parsing;
 
-internal class HtmlParser(IEnumerable<Note.MentionedUser> mentions)
+internal class HtmlParser(IEnumerable<Note.MentionedUser> mentions, ICollection<MfmInlineMedia> media)
 {
 	internal string? ParseNode(INode node)
 	{
@@ -21,26 +22,23 @@ internal class HtmlParser(IEnumerable<Note.MentionedUser> mentions)
 			}
 			case "A":
 			{
-				if (node is HtmlElement el)
+				if (node is not HtmlElement el) return node.TextContent;
+
+				var href = el.GetAttribute("href");
+				if (href == null) return $"<plain>{el.TextContent}</plain>";
+
+				if (el.ClassList.Contains("u-url") && el.ClassList.Contains("mention"))
 				{
-					var href = el.GetAttribute("href");
-					if (href == null) return $"<plain>{el.TextContent}</plain>";
-
-					if (el.ClassList.Contains("u-url") && el.ClassList.Contains("mention"))
-					{
-						var mention = mentions.FirstOrDefault(p => p.Uri == href || p.Url == href);
-						return mention != null
-							? $"@{mention.Username}@{mention.Host}"
-							: $"<plain>{el.TextContent}</plain>";
-					}
-
-					if (el.TextContent == href && (href.StartsWith("http://") || href.StartsWith("https://")))
-						return href;
-
-					return $"[{el.TextContent}]({href})";
+					var mention = mentions.FirstOrDefault(p => p.Uri == href || p.Url == href);
+					return mention != null
+						? $"@{mention.Username}@{mention.Host}"
+						: $"<plain>{el.TextContent}</plain>";
 				}
 
-				return node.TextContent;
+				if (el.TextContent == href && (href.StartsWith("http://") || href.StartsWith("https://")))
+					return href;
+
+				return $"[{el.TextContent}]({href})";
 			}
 			case "H1":
 			{
@@ -81,6 +79,31 @@ internal class HtmlParser(IEnumerable<Note.MentionedUser> mentions)
 					? $"\n> {string.Join("\n> ", node.TextContent.Split("\n"))}"
 					: null;
 			}
+			
+			case "VIDEO":
+			case "AUDIO":
+			case "IMG":
+			{
+				if (node is not HtmlElement el) return node.TextContent;
+
+				var src = el.GetAttribute("src");
+				if (!Uri.IsWellFormedUriString(src, UriKind.Absolute))
+					return node.TextContent;
+				
+				var alt = el.GetAttribute("alt") ?? el.GetAttribute("title");
+
+				var type = node.NodeName switch
+				{
+					"VIDEO" => MfmInlineMedia.MediaType.Video,
+					"AUDIO" => MfmInlineMedia.MediaType.Audio,
+					"IMG"   => MfmInlineMedia.MediaType.Image,
+					_       => MfmInlineMedia.MediaType.Other,
+				};
+				
+				media.Add(new MfmInlineMedia(type, src, alt));
+
+				return $"$[media {src} ]";
+			}
 
 			case "P":
 			case "H2":
diff --git a/Iceshrimp.Backend/Core/Services/NoteService.cs b/Iceshrimp.Backend/Core/Services/NoteService.cs
index a8b86643..24623d3b 100644
--- a/Iceshrimp.Backend/Core/Services/NoteService.cs
+++ b/Iceshrimp.Backend/Core/Services/NoteService.cs
@@ -532,6 +532,7 @@ public class NoteService(
 		{
 			nodes = MfmParser.Parse(data.Text);
 			mentionsResolver.ResolveMentions(nodes.AsSpan(), note.User.Host, mentions, splitDomainMapping);
+			// todo scan nodes recursively to find inline media and make sure note.fileids has them
 			data.Text = nodes.Serialize();
 		}
 
@@ -942,7 +943,15 @@ public class NoteService(
 		var renote     = quoteUrl != null ? await ResolveNoteAsync(quoteUrl, user: user) : null;
 		var renoteUri  = renote == null ? quoteUrl : null;
 		var visibility = note.GetVisibility(actor);
-		var text       = note.MkContent ?? await MfmConverter.FromHtmlAsync(note.Content, mentionData.Mentions);
+
+		var                   text            = note.MkContent;
+		List<MfmInlineMedia>? htmlInlineMedia = null;
+
+		if (text == null)
+		{
+			(text, htmlInlineMedia) = await MfmConverter.FromHtmlAsync(note.Content, mentionData.Mentions);
+		}
+
 		var cw         = note.Summary;
 		var url        = note.Url?.Link;
 		var uri        = note.Id;
@@ -969,7 +978,7 @@ public class NoteService(
 		}
 
 		var sensitive = (note.Sensitive ?? false) || !string.IsNullOrWhiteSpace(cw);
-		var files     = await ProcessAttachmentsAsync(note.Attachments, actor, sensitive);
+		var files     = await ProcessAttachmentsAsync(note.Attachments, htmlInlineMedia, actor, sensitive);
 		var emoji = (await emojiSvc.ProcessEmojiAsync(note.Tags?.OfType<ASEmoji>().ToList(), actor.Host))
 		            .Select(p => p.Id)
 		            .ToList();
@@ -1026,7 +1035,14 @@ public class NoteService(
 
 		var mentionData = await ResolveNoteMentionsAsync(note);
 
-		var text = note.MkContent ?? await MfmConverter.FromHtmlAsync(note.Content, mentionData.Mentions);
+		var                   text            = note.MkContent;
+		List<MfmInlineMedia>? htmlInlineMedia = null;
+
+		if (text == null)
+		{
+			(text, htmlInlineMedia) = await MfmConverter.FromHtmlAsync(note.Content, mentionData.Mentions);
+		}
+
 		var cw   = note.Summary;
 
 		Poll? poll = null;
@@ -1055,7 +1071,7 @@ public class NoteService(
 		}
 
 		var sensitive = (note.Sensitive ?? false) || !string.IsNullOrWhiteSpace(cw);
-		var files     = await ProcessAttachmentsAsync(note.Attachments, actor, sensitive, false);
+		var files     = await ProcessAttachmentsAsync(note.Attachments, htmlInlineMedia, actor, sensitive, false);
 		var emoji = (await emojiSvc.ProcessEmojiAsync(note.Tags?.OfType<ASEmoji>().ToList(), actor.Host))
 		            .Select(p => p.Id)
 		            .ToList();
@@ -1186,13 +1202,28 @@ public class NoteService(
 	}
 
 	private async Task<List<DriveFile>> ProcessAttachmentsAsync(
-		List<ASAttachment>? attachments, User user, bool sensitive, bool logExisting = true
+		List<ASAttachment>? attachments, List<MfmInlineMedia>? htmlInlineMedia, User user, bool sensitive,
+		bool logExisting = true
 	)
 	{
-		if (attachments is not { Count: > 0 }) return [];
-		var result = await attachments
-		                   .OfType<ASDocument>()
-		                   .Take(10)
+		var allAttachments = attachments?.OfType<ASDocument>().Take(10).ToList() ?? [];
+
+		if (htmlInlineMedia != null)
+		{
+			var inlineUrls     = htmlInlineMedia.Select(p => p.Src);
+			var unattachedUrls = inlineUrls.Except(allAttachments.Select(a => a.Url?.Id)).ToArray();
+
+			allAttachments.AddRange(htmlInlineMedia.Where(p => unattachedUrls.Contains(p.Src))
+			                                       .DistinctBy(p => p.Src)
+			                                       .Select(p => new ASDocument
+			                                       {
+				                                       Url = new ASLink(p.Src),
+				                                       Description = p.Alt,
+			                                       }));
+		}
+
+		if (allAttachments is not { Count: > 0 }) return [];
+		var result = await allAttachments
 		                   .Select(p => driveSvc.StoreFileAsync(p.Url?.Id, user, p.Sensitive ?? sensitive,
 		                                                        p.Description, p.MediaType, logExisting))
 		                   .AwaitAllNoConcurrencyAsync();
diff --git a/Iceshrimp.Backend/Core/Services/UserService.cs b/Iceshrimp.Backend/Core/Services/UserService.cs
index 8072e4be..063564cc 100644
--- a/Iceshrimp.Backend/Core/Services/UserService.cs
+++ b/Iceshrimp.Backend/Core/Services/UserService.cs
@@ -150,12 +150,12 @@ public class UserService(
 			             .Where(p => p is { Name: not null, Value: not null })
 			             .Select(async p => new UserProfile.Field
 			             {
-				             Name = p.Name!, Value = await MfmConverter.FromHtmlAsync(p.Value) ?? ""
+				             Name = p.Name!, Value = (await MfmConverter.FromHtmlAsync(p.Value)).Mfm
 			             })
 			             .AwaitAllAsync()
 			: null;
 
-		var bio = actor.MkSummary ?? await MfmConverter.FromHtmlAsync(actor.Summary);
+		var bio = actor.MkSummary ?? (await MfmConverter.FromHtmlAsync(actor.Summary)).Mfm;
 
 		var tags = ResolveHashtags(bio, actor);
 
@@ -303,7 +303,7 @@ public class UserService(
 			             .Where(p => p is { Name: not null, Value: not null })
 			             .Select(async p => new UserProfile.Field
 			             {
-				             Name = p.Name!, Value = await MfmConverter.FromHtmlAsync(p.Value) ?? ""
+				             Name = p.Name!, Value = (await MfmConverter.FromHtmlAsync(p.Value)).Mfm
 			             })
 			             .AwaitAllAsync()
 			: null;
@@ -316,7 +316,7 @@ public class UserService(
 
 		var processPendingDeletes = await ResolveAvatarAndBannerAsync(user, actor);
 
-		user.UserProfile.Description = actor.MkSummary ?? await MfmConverter.FromHtmlAsync(actor.Summary);
+		user.UserProfile.Description = actor.MkSummary ?? (await MfmConverter.FromHtmlAsync(actor.Summary)).Mfm;
 		//user.UserProfile.Birthday = TODO;
 		//user.UserProfile.Location = TODO;
 		user.UserProfile.Fields   = fields?.ToArray() ?? [];
@@ -1065,14 +1065,14 @@ public class UserService(
 						             .Select(async p => new UserProfile.Field
 						             {
 							             Name  = p.Name!,
-							             Value = await MfmConverter.FromHtmlAsync(p.Value, mentions) ?? ""
+							             Value = (await MfmConverter.FromHtmlAsync(p.Value, mentions)).Mfm
 						             })
 						             .AwaitAllAsync()
 						: null;
 
 					var description = actor.MkSummary != null
 						? mentionsResolver.ResolveMentions(actor.MkSummary, bgUser.Host, mentions, splitDomainMapping)
-						: await MfmConverter.FromHtmlAsync(actor.Summary, mentions);
+						: (await MfmConverter.FromHtmlAsync(actor.Summary, mentions)).Mfm;
 
 					bgUser.UserProfile.Mentions    = mentions;
 					bgUser.UserProfile.Fields      = fields?.ToArray() ?? [];