[backend/drive] Significantly improve ImageSharp blurhash performance & memory efficiency

Iceshrimp.Backend/Core/Helpers/BlurhashHelper.cs

@@ -1,23 +1,22 @@
 using System.Collections.Immutable;
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-using Blurhash;
 using CommunityToolkit.HighPerformance;
+using SixLabors.ImageSharp.PixelFormats;
 
 namespace Iceshrimp.Backend.Core.Helpers;
 
+// Adapted from https://github.com/MarkusPalcer/blurhash.net under MIT
 public static class BlurhashHelper
 {
 	private static readonly ImmutableArray<float> PrecomputedLut = [..Enumerable.Range(0, 256).Select(SRgbToLinear)];
 
 	/// <summary>
-	/// Encodes a Span2D of raw rgb data into a Blurhash string
+	/// Encodes a Span2D of raw pixel data into a Blurhash string
 	/// </summary>
-	/// <param name="pixels">The 2-dimensional array of pixels to encode</param>
+	/// <param name="pixels">The Span2D of raw pixel data to encode</param>
 	/// <param name="componentsX">The number of components used on the X-Axis for the DCT</param>
 	/// <param name="componentsY">The number of components used on the Y-Axis for the DCT</param>
 	/// <returns>The resulting Blurhash string</returns>
-	public static string Encode(Span2D<RgbPixel> pixels, int componentsX, int componentsY)
+	public static string Encode(Span2D<Rgb24> pixels, int componentsX, int componentsY)
 	{
 		if (componentsX < 1) throw new ArgumentException("componentsX needs to be at least 1");
 		if (componentsX > 9) throw new ArgumentException("componentsX needs to be at most 9");
@@ -115,30 +114,21 @@ public static class BlurhashHelper
 
 	private static int EncodeAc(double r, double g, double b, double maximumValue)
 	{
-		var quantizedR = (int)Math.Max(0, Math.Min(18, Math.Floor(MathUtils.SignPow(r / maximumValue, 0.5) * 9 + 9.5)));
+		var quantizedR = (int)Math.Max(0, Math.Min(18, Math.Floor(SignPow(r / maximumValue, 0.5) * 9 + 9.5)));
-		var quantizedG = (int)Math.Max(0, Math.Min(18, Math.Floor(MathUtils.SignPow(g / maximumValue, 0.5) * 9 + 9.5)));
+		var quantizedG = (int)Math.Max(0, Math.Min(18, Math.Floor(SignPow(g / maximumValue, 0.5) * 9 + 9.5)));
-		var quantizedB = (int)Math.Max(0, Math.Min(18, Math.Floor(MathUtils.SignPow(b / maximumValue, 0.5) * 9 + 9.5)));
+		var quantizedB = (int)Math.Max(0, Math.Min(18, Math.Floor(SignPow(b / maximumValue, 0.5) * 9 + 9.5)));
 
 		return quantizedR * 19 * 19 + quantizedG * 19 + quantizedB;
 	}
 
 	private static int EncodeDc(double r, double g, double b)
 	{
-		var roundedR = MathUtils.LinearTosRgb(r);
+		var roundedR = LinearTosRgb(r);
-		var roundedG = MathUtils.LinearTosRgb(g);
+		var roundedG = LinearTosRgb(g);
-		var roundedB = MathUtils.LinearTosRgb(b);
+		var roundedB = LinearTosRgb(b);
 		return (roundedR << 16) + (roundedG << 8) + roundedB;
 	}
 
-	[StructLayout(LayoutKind.Sequential)]
-	[method: MethodImpl(MethodImplOptions.AggressiveInlining)]
-	public struct RgbPixel(byte r, byte g, byte b)
-	{
-		public readonly byte R = r;
-		public readonly byte G = g;
-		public readonly byte B = b;
-	}
-
 	private static void EncodeBase83(this int number, Span<char> output)
 	{
 		var length = output.Length;
@@ -156,4 +146,23 @@ public static class BlurhashHelper
 		var num = value / (float)byte.MaxValue;
 		return (float)(num <= 0.04045 ? num / 12.92 : float.Pow((num + 0.055f) / 1.055f, 2.4f));
 	}
+
+	private static int LinearTosRgb(double value)
+	{
+		var v = Math.Max(0.0, Math.Min(1.0, value));
+		if (v <= 0.0031308) return (int)(v * 12.92 * 255 + 0.5);
+		return (int)((1.055 * Math.Pow(v, 1 / 2.4) - 0.055) * 255 + 0.5);
+	}
+
+	private static double SignPow(double @base, double exponent)
+	{
+		return Math.Sign(@base) * Math.Pow(Math.Abs(@base), exponent);
+	}
+
+	private struct Pixel(double red, double green, double blue)
+	{
+		public double Red   = red;
+		public double Green = green;
+		public double Blue  = blue;
+	}
 }

Iceshrimp.Backend/Core/Services/ImageProcessor.cs

@@ -1,5 +1,4 @@
 using System.Runtime.InteropServices;
-using Blurhash.ImageSharp;
 using CommunityToolkit.HighPerformance;
 using Iceshrimp.Backend.Core.Configuration;
 using Iceshrimp.Backend.Core.Database.Tables;
@@ -14,10 +13,6 @@ using SixLabors.ImageSharp.PixelFormats;
 using SixLabors.ImageSharp.Processing;
 using ImageSharp = SixLabors.ImageSharp.Image;
 
-#if EnableLibVips
-using Blurhash;
-#endif
-
 namespace Iceshrimp.Backend.Core.Services;
 
 public class ImageProcessor
@@ -177,17 +172,18 @@ public class ImageProcessor
 		var properties = new DriveFile.FileProperties { Width = ident.Size.Width, Height = ident.Size.Height };
 		var res        = new Result { Properties              = properties };
 
-		// Calculate blurhash using a x200px image for improved performance
+		// Calculate blurhash using a x100px image for improved performance
 		{
-			using var image = await GetImage(data, ident, 200);
+			using var image = await GetImage<Rgb24>(data, ident, 100);
-			res.Blurhash = Blurhasher.Encode(image, 7, 7);
+			image.DangerousTryGetSinglePixelMemory(out var mem);
+			res.Blurhash = BlurhashHelper.Encode(mem.Span.AsSpan2D(image.Height, image.Width), 7, 7);
 		}
 
 		if (genThumb)
 		{
 			res.RenderThumbnail = async stream =>
 			{
-				using var image        = await GetImage(data, ident, 1000);
+				using var image        = await GetImage<Rgba32>(data, ident, 1000);
 				var       thumbEncoder = new WebpEncoder { Quality = 75, FileFormat = WebpFileFormatType.Lossy };
 				await image.SaveAsWebpAsync(stream, thumbEncoder);
 			};
@@ -197,7 +193,7 @@ public class ImageProcessor
 		{
 			res.RenderWebpublic = async stream =>
 			{
-				using var image        = await GetImage(data, ident, 2048);
+				using var image        = await GetImage<Rgba32>(data, ident, 2048);
 				var       q            = request.MimeType == "image/png" ? 100 : 75;
 				var       thumbEncoder = new WebpEncoder { Quality = q, FileFormat = WebpFileFormatType.Lossy };
 				await image.SaveAsWebpAsync(stream, thumbEncoder);
@@ -207,7 +203,9 @@ public class ImageProcessor
 		return res;
 	}
 
-	private static async Task<Image<Rgba32>> GetImage(Stream data, ImageInfo ident, int width, int? height = null)
+	private static async Task<Image<TPixel>> GetImage<TPixel>(
+		Stream data, ImageInfo ident, int width, int? height = null
+	) where TPixel : unmanaged, IPixel<TPixel>
 	{
 		width  = Math.Min(ident.Width, width);
 		height = Math.Min(ident.Height, height ?? width);
@@ -215,7 +213,7 @@ public class ImageProcessor
 		var options = new DecoderOptions { MaxFrames = 1, TargetSize = size };
 
 		data.Seek(0, SeekOrigin.Begin);
-		var image = await ImageSharp.LoadAsync<Rgba32>(options, data);
+		var image = await ImageSharp.LoadAsync<TPixel>(options, data);
 		image.Mutate(x => x.AutoOrient());
 		var opts = new ResizeOptions { Size = size, Mode = ResizeMode.Max };
 		image.Mutate(p => p.Resize(opts));
@@ -223,7 +221,7 @@ public class ImageProcessor
 	}
 
 	#if EnableLibVips
-	private Task<Result> ProcessImageVips(
+	private static Task<Result> ProcessImageVips(
 		byte[] buf, ImageInfo ident, DriveFileCreationRequest request, bool genThumb, bool genWebp
 	)
 	{
@@ -239,9 +237,8 @@ public class ImageProcessor
 		using var blurhashImageFlattened = blurhashImage.HasAlpha() ? blurhashImage.Flatten() : blurhashImage;
 		using var blurhashImageActual    = blurhashImageFlattened.Cast(NetVips.Enums.BandFormat.Uchar);
 
-		var blurBuf = blurhashImageActual.WriteToMemory();
+		var blurBuf    = blurhashImageActual.WriteToMemory();
-		var blurPixels = MemoryMarshal.Cast<byte, BlurhashHelper.RgbPixel>(blurBuf)
+		var blurPixels = MemoryMarshal.Cast<byte, Rgb24>(blurBuf).AsSpan2D(blurhashImage.Height, blurhashImage.Width);
-		                              .AsSpan2D(blurhashImage.Height, blurhashImage.Width);
 		res.Blurhash = BlurhashHelper.Encode(blurPixels, 7, 7);
 
 		if (genThumb)

Iceshrimp.Backend/Iceshrimp.Backend.csproj

@@ -16,7 +16,6 @@
     <ItemGroup>
         <PackageReference Include="AngleSharp" Version="1.1.2" />
         <PackageReference Include="AsyncKeyedLock" Version="7.0.0" />
-        <PackageReference Include="Blurhash.ImageSharp" Version="3.0.0" />
         <PackageReference Include="CommunityToolkit.HighPerformance" Version="8.2.2" />
         <PackageReference Include="dotNetRdf.Core" Version="3.2.9-iceshrimp" />
         <PackageReference Include="EntityFrameworkCore.Exceptions.PostgreSQL" Version="8.1.2" />