[backend/drive] Add basic image processing & thumbnail generation (ISH-63, ISH-64)

Iceshrimp.Backend/Controllers/Mastodon/Renderers/NoteRenderer.cs

@@ -37,7 +37,7 @@ public class NoteRenderer(
 			attachments = await db.DriveFiles.Where(p => note.FileIds.Contains(p.Id))
 			                      .Select(f => new Attachment {
 				                      Id          = f.Id,
-				                      Url         = f.Url,
+				                      Url         = f.WebpublicUrl ?? f.Url,
 				                      Blurhash    = f.Blurhash,
 				                      PreviewUrl  = f.ThumbnailUrl,
 				                      Description = f.Comment,

Iceshrimp.Backend/Core/Database/Tables/DriveFile.cs

@@ -192,7 +192,13 @@ public class DriveFile : IEntity {
 	public class FileProperties {
 		[J("width")]  public int? Width  { get; set; }
 		[J("height")] public int? Height { get; set; }
-		[J("orientation")] public int?    Orientation  { get; set; }
+
-		[J("avgColor")]    public string? AverageColor { get; set; }
+		[Obsolete("Deprecated property")]
+		[J("orientation")]
+		public int? Orientation { get; set; }
+
+		[Obsolete("Deprecated property")]
+		[J("avgColor")]
+		public string? AverageColor { get; set; }
 	}
 }

Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs

@@ -55,7 +55,7 @@ public class NoteRenderer(IOptions<Config.InstanceSection> config, MfmConverter
 			          .Select(p => new ASDocument {
 				          Type        = $"{Constants.ActivityStreamsNs}#Document",
 				          Sensitive   = p.IsSensitive,
-				          Url       = new ASObjectBase(p.Url),
+				          Url         = new ASObjectBase(p.WebpublicUrl ?? p.Url),
 				          MediaType   = p.Type,
 				          Description = p.Comment
 			          })

Iceshrimp.Backend/Core/Services/DriveService.cs

@@ -9,7 +9,9 @@ using Iceshrimp.Backend.Core.Queues;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Options;
 using SixLabors.ImageSharp;
+using SixLabors.ImageSharp.Formats.Webp;
 using SixLabors.ImageSharp.PixelFormats;
+using SixLabors.ImageSharp.Processing;
 
 namespace Iceshrimp.Backend.Core.Services;
 
@@ -101,7 +103,17 @@ public class DriveService(
 
 		buf.Seek(0, SeekOrigin.Begin);
 
+		var shouldStore    = storageConfig.Value.MediaRetention != null || user.Host == null;
+		var storedInternal = storageConfig.Value.Mode == Enums.FileStorage.Local;
+
+		if (request.Uri == null && user.Host != null)
+			throw GracefulException.UnprocessableEntity("Refusing to store file without uri for remote user");
+
 		string? blurhash  = null;
+		Stream? thumbnail = null;
+		Stream? webpublic = null;
+
+		DriveFile.FileProperties? properties = null;
 
 		if (request.MimeType.StartsWith("image/") || request.MimeType == "image") {
 			try {
@@ -111,22 +123,62 @@ public class DriveService(
 				// Correct mime type
 				if (request.MimeType == "image" && image.Metadata.DecodedImageFormat?.DefaultMimeType != null)
 					request.MimeType = image.Metadata.DecodedImageFormat.DefaultMimeType;
+
+				properties = new DriveFile.FileProperties {
+					Width  = image.Size.Width,
+					Height = image.Size.Height
+				};
+
+				if (shouldStore) {
+					// Generate thumbnail
+					var thumbnailImage = image.Clone();
+					if (image.Size.Width > 1000)
+						thumbnailImage.Mutate(p => p.Resize(new Size(1000, 0)));
+
+					thumbnail = new MemoryStream();
+					await thumbnailImage.SaveAsWebpAsync(thumbnail);
+					thumbnail.Seek(0, SeekOrigin.Begin);
+
+					// Generate webpublic for local users
+					if (user.Host == null) {
+						var webpublicImage = image.Clone();
+						webpublicImage.Metadata.ExifProfile = null;
+						webpublicImage.Metadata.XmpProfile  = null;
+						if (image.Size.Width > 2048)
+							webpublicImage.Mutate(p => p.Resize(new Size(2048, 0)));
+
+						var encoder = request.MimeType == "image/png"
+							? new WebpEncoder {
+								Quality             = 100,
+								NearLossless        = true,
+								NearLosslessQuality = 60
+							}
+							: new WebpEncoder { Quality = 75 };
+
+						webpublic = new MemoryStream();
+						await webpublicImage.SaveAsWebpAsync(webpublic, encoder);
+						webpublic.Seek(0, SeekOrigin.Begin);
+					}
+				}
 			}
 			catch {
-				logger.LogError("Failed to generate blurhash for image with mime type {type}", request.MimeType);
+				logger.LogError("Failed to generate blurhash & thumbnail for image with mime type {type}",
+				                request.MimeType);
+
+				// We want to make sure no images are federated out without stripping metadata & converting to webp
+				if (user.Host == null) throw;
 			}
 
 			buf.Seek(0, SeekOrigin.Begin);
 		}
 
-		var (filename, guid) = GenerateFilenameKeepingExtension(request.Filename);
-		var shouldStore    = storageConfig.Value.MediaRetention != null || user.Host == null;
-		var storedInternal = storageConfig.Value.Mode == Enums.FileStorage.Local;
-
 		string  url;
+		string? thumbnailUrl = null;
+		string? webpublicUrl = null;
 
-		if (request.Uri == null && user.Host != null)
+		var filename          = GenerateFilenameKeepingExtension(request.Filename);
-			throw GracefulException.UnprocessableEntity("Refusing to store file without uri for remote user");
+		var thumbnailFilename = thumbnail != null ? GenerateWebpFilename("thumbnail-") : null;
+		var webpublicFilename = webpublic != null ? GenerateWebpFilename("webpublic-") : null;
 
 		if (shouldStore) {
 			if (storedInternal) {
@@ -137,10 +189,32 @@ public class DriveService(
 				await using var writer = File.OpenWrite(path);
 				await buf.CopyToAsync(writer);
 				url = $"https://{instanceConfig.Value.WebDomain}/files/{filename}";
+
+				if (thumbnailFilename != null && thumbnail is { Length: > 0 }) {
+					var             thumbPath   = Path.Combine(pathBase, thumbnailFilename);
+					await using var thumbWriter = File.OpenWrite(thumbPath);
+					await thumbnail.CopyToAsync(thumbWriter);
+				}
+
+				if (webpublicFilename != null && webpublic is { Length: > 0 }) {
+					var             webpPath   = Path.Combine(pathBase, webpublicFilename);
+					await using var webpWriter = File.OpenWrite(webpPath);
+					await webpublic.CopyToAsync(webpWriter);
+				}
 			}
 			else {
 				await storageSvc.UploadFileAsync(filename, data);
 				url = storageSvc.GetFilePublicUrl(filename).AbsoluteUri;
+
+				if (thumbnailFilename != null && thumbnail is { Length: > 0 }) {
+					await storageSvc.UploadFileAsync(thumbnailFilename, thumbnail);
+					thumbnailUrl = storageSvc.GetFilePublicUrl(thumbnailFilename).AbsoluteUri;
+				}
+
+				if (webpublicFilename != null && webpublic is { Length: > 0 }) {
+					await storageSvc.UploadFileAsync(webpublicFilename, webpublic);
+					webpublicUrl = storageSvc.GetFilePublicUrl(webpublicFilename).AbsoluteUri;
+				}
 			}
 		}
 		else {
@@ -165,12 +239,12 @@ public class DriveService(
 			RequestHeaders     = request.RequestHeaders,
 			RequestIp          = request.RequestIp,
 			Blurhash           = blurhash,
-			//Properties         = TODO,
+			Properties         = properties,
-			//ThumbnailUrl       = TODO,
+			ThumbnailUrl       = thumbnailUrl,
-			//ThumbnailAccessKey = TODO,
+			ThumbnailAccessKey = thumbnailFilename,
-			//WebpublicType      = TODO,
+			WebpublicType      = webpublicUrl != null ? "image/webp" : null,
-			//WebpublicUrl       = TODO,
+			WebpublicUrl       = webpublicUrl,
-			//WebpublicAccessKey = TODO,
+			WebpublicAccessKey = webpublicFilename,
 		};
 
 		await db.AddAsync(file);
@@ -194,10 +268,15 @@ public class DriveService(
 			: file.WebpublicUrl ?? file.Url;
 	}
 
-	private static (string filename, string guid) GenerateFilenameKeepingExtension(string filename) {
+	private static string GenerateFilenameKeepingExtension(string filename) {
 		var guid = Guid.NewGuid().ToString().ToLowerInvariant();
 		var ext  = Path.GetExtension(filename);
-		return (guid + ext, guid);
+		return guid + ext;
+	}
+
+	private static string GenerateWebpFilename(string prefix = "") {
+		var guid = Guid.NewGuid().ToString().ToLowerInvariant();
+		return $"{prefix}{guid}.webp";
 	}
 
 	private static string CleanMimeType(string? mimeType) {