[backend/drive] Significantly improve LibVips blurhash performance & memory efficiency

This commit saves ~2.5ms & ~1.4MB of heap allocations during blurhash computation using the LibVips image processor.

Iceshrimp.Backend/Core/Helpers/BlurhashHelper.cs

@@ -0,0 +1,159 @@
+using System.Collections.Immutable;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using Blurhash;
+using CommunityToolkit.HighPerformance;
+
+namespace Iceshrimp.Backend.Core.Helpers;
+
+public static class BlurhashHelper
+{
+	private static readonly ImmutableArray<float> PrecomputedLut = [..Enumerable.Range(0, 256).Select(SRgbToLinear)];
+
+	/// <summary>
+	/// Encodes a Span2D of raw rgb data into a Blurhash string
+	/// </summary>
+	/// <param name="pixels">The 2-dimensional array of pixels to encode</param>
+	/// <param name="componentsX">The number of components used on the X-Axis for the DCT</param>
+	/// <param name="componentsY">The number of components used on the Y-Axis for the DCT</param>
+	/// <returns>The resulting Blurhash string</returns>
+	public static string Encode(Span2D<RgbPixel> pixels, int componentsX, int componentsY)
+	{
+		if (componentsX < 1) throw new ArgumentException("componentsX needs to be at least 1");
+		if (componentsX > 9) throw new ArgumentException("componentsX needs to be at most 9");
+		if (componentsY < 1) throw new ArgumentException("componentsY needs to be at least 1");
+		if (componentsY > 9) throw new ArgumentException("componentsY needs to be at most 9");
+
+		Span<Pixel> factors      = stackalloc Pixel[componentsX * componentsY];
+		Span<char>  resultBuffer = stackalloc char[4 + 2 * componentsX * componentsY];
+		Span<float> lut          = stackalloc float[256];
+		PrecomputedLut.CopyTo(lut);
+
+		var width  = pixels.Width;
+		var height = pixels.Height;
+
+		var xCosines = new double[width];
+		var yCosines = new double[height];
+
+		for (var yComponent = 0; yComponent < componentsY; yComponent++)
+		for (var xComponent = 0; xComponent < componentsX; xComponent++)
+		{
+			double r             = 0, g = 0, b = 0;
+			double normalization = xComponent == 0 && yComponent == 0 ? 1 : 2;
+
+			for (var xPixel = 0; xPixel < width; xPixel++)
+				xCosines[xPixel] = Math.Cos(Math.PI * xComponent * xPixel / width);
+			for (var yPixel = 0; yPixel < height; yPixel++)
+				yCosines[yPixel] = Math.Cos(Math.PI * yComponent * yPixel / height);
+
+			for (var xPixel = 0; xPixel < width; xPixel++)
+			for (var yPixel = 0; yPixel < height; yPixel++)
+			{
+				var basis = xCosines[xPixel] * yCosines[yPixel];
+				var pixel = pixels[yPixel, xPixel];
+				r += basis * lut[pixel.R];
+				g += basis * lut[pixel.G];
+				b += basis * lut[pixel.B];
+			}
+
+			var scale = normalization / (width * height);
+			factors[componentsX * yComponent + xComponent].Red   = r * scale;
+			factors[componentsX * yComponent + xComponent].Green = g * scale;
+			factors[componentsX * yComponent + xComponent].Blue  = b * scale;
+		}
+
+		var dc      = factors[0];
+		var acCount = componentsX * componentsY - 1;
+
+		var sizeFlag = (componentsX - 1) + (componentsY - 1) * 9;
+		sizeFlag.EncodeBase83(resultBuffer[..1]);
+
+		float maximumValue;
+		if (acCount > 0)
+		{
+			// Get maximum absolute value of all AC components
+			var actualMaximumValue = 0.0;
+			for (var yComponent = 0; yComponent < componentsY; yComponent++)
+			for (var xComponent = 0; xComponent < componentsX; xComponent++)
+			{
+				// Ignore DC component
+				if (xComponent == 0 && yComponent == 0) continue;
+
+				var factorIndex = componentsX * yComponent + xComponent;
+
+				actualMaximumValue = Math.Max(Math.Abs(factors[factorIndex].Red), actualMaximumValue);
+				actualMaximumValue = Math.Max(Math.Abs(factors[factorIndex].Green), actualMaximumValue);
+				actualMaximumValue = Math.Max(Math.Abs(factors[factorIndex].Blue), actualMaximumValue);
+			}
+
+			var quantizedMaximumValue = (int)Math.Max(0.0, Math.Min(82.0, Math.Floor(actualMaximumValue * 166 - 0.5)));
+			maximumValue = ((float)quantizedMaximumValue + 1) / 166;
+			quantizedMaximumValue.EncodeBase83(resultBuffer.Slice(1, 1));
+		}
+		else
+		{
+			maximumValue    = 1;
+			resultBuffer[1] = '0';
+		}
+
+		EncodeDc(dc.Red, dc.Green, dc.Blue).EncodeBase83(resultBuffer.Slice(2, 4));
+
+		for (var yComponent = 0; yComponent < componentsY; yComponent++)
+		for (var xComponent = 0; xComponent < componentsX; xComponent++)
+		{
+			// Ignore DC component
+			if (xComponent == 0 && yComponent == 0) continue;
+
+			var factorIndex = componentsX * yComponent + xComponent;
+
+			EncodeAc(factors[factorIndex].Red, factors[factorIndex].Green, factors[factorIndex].Blue, maximumValue)
+				.EncodeBase83(resultBuffer.Slice(6 + (factorIndex - 1) * 2, 2));
+		}
+
+		return resultBuffer.ToString();
+	}
+
+	private static int EncodeAc(double r, double g, double b, double maximumValue)
+	{
+		var quantizedR = (int)Math.Max(0, Math.Min(18, Math.Floor(MathUtils.SignPow(r / maximumValue, 0.5) * 9 + 9.5)));
+		var quantizedG = (int)Math.Max(0, Math.Min(18, Math.Floor(MathUtils.SignPow(g / maximumValue, 0.5) * 9 + 9.5)));
+		var quantizedB = (int)Math.Max(0, Math.Min(18, Math.Floor(MathUtils.SignPow(b / maximumValue, 0.5) * 9 + 9.5)));
+
+		return quantizedR * 19 * 19 + quantizedG * 19 + quantizedB;
+	}
+
+	private static int EncodeDc(double r, double g, double b)
+	{
+		var roundedR = MathUtils.LinearTosRgb(r);
+		var roundedG = MathUtils.LinearTosRgb(g);
+		var roundedB = MathUtils.LinearTosRgb(b);
+		return (roundedR << 16) + (roundedG << 8) + roundedB;
+	}
+
+	[StructLayout(LayoutKind.Sequential)]
+	[method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+	public struct RgbPixel(byte r, byte g, byte b)
+	{
+		public readonly byte R = r;
+		public readonly byte G = g;
+		public readonly byte B = b;
+	}
+
+	private static void EncodeBase83(this int number, Span<char> output)
+	{
+		var length = output.Length;
+		for (var index1 = 0; index1 < length; ++index1)
+		{
+			var index2 = number % 83;
+			number /= 83;
+			output[length - index1 - 1] =
+				"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz#$%*+,-.:;=?@[]^_{|}~"[index2];
+		}
+	}
+
+	private static float SRgbToLinear(int value)
+	{
+		var num = value / (float)byte.MaxValue;
+		return (float)(num <= 0.04045 ? num / 12.92 : float.Pow((num + 0.055f) / 1.055f, 2.4f));
+	}
+}

Iceshrimp.Backend/Core/Services/ImageProcessor.cs

@@ -1,6 +1,9 @@
+using System.Runtime.InteropServices;
 using Blurhash.ImageSharp;
+using CommunityToolkit.HighPerformance;
 using Iceshrimp.Backend.Core.Configuration;
 using Iceshrimp.Backend.Core.Database.Tables;
+using Iceshrimp.Backend.Core.Helpers;
 using Microsoft.Extensions.Options;
 using SixLabors.ImageSharp;
 using SixLabors.ImageSharp.Formats;
@@ -55,9 +58,9 @@ public class ImageProcessor
 		NetVips.NetVips.Leak = true;
 
 		// We don't need the VIPS operation or file cache
-		NetVips.Cache.Max = 0;
+		NetVips.Cache.Max      = 0;
 		NetVips.Cache.MaxFiles = 0;
-		NetVips.Cache.MaxMem = 0;
+		NetVips.Cache.MaxMem   = 0;
 
 		NetVips.Log.SetLogHandler("VIPS", NetVips.Enums.LogLevelFlags.Warning | NetVips.Enums.LogLevelFlags.Error,
 		                          VipsLogDelegate);
@@ -225,30 +228,21 @@ public class ImageProcessor
 	)
 	{
 		var properties = new DriveFile.FileProperties { Width = ident.Size.Width, Height = ident.Size.Height };
-		var res = new Result { Properties = properties };
+		var res        = new Result { Properties              = properties };
 
-		// Calculate blurhash using a x200px image for improved performance
+		// Calculate blurhash using a x100px image for improved performance
 		using var blurhashImageSource =
-			NetVips.Image.ThumbnailBuffer(buf, width: 200, height: 200, size: NetVips.Enums.Size.Down);
+			NetVips.Image.ThumbnailBuffer(buf, width: 100, height: 100, size: NetVips.Enums.Size.Down);
 		using var blurhashImage = blurhashImageSource.Interpretation == NetVips.Enums.Interpretation.Srgb
 			? blurhashImageSource
 			: blurhashImageSource.Colourspace(NetVips.Enums.Interpretation.Srgb);
-		var blurBuf = blurhashImage.WriteToMemory();
+		using var blurhashImageFlattened = blurhashImage.HasAlpha() ? blurhashImage.Flatten() : blurhashImage;
-		var blurArr = new Pixel[blurhashImage.Width, blurhashImage.Height];
+		using var blurhashImageActual    = blurhashImageFlattened.Cast(NetVips.Enums.BandFormat.Uchar);
 
-		var idx = 0;
+		var blurBuf = blurhashImageActual.WriteToMemory();
-		var incr = blurhashImage.Bands - 3;
+		var blurPixels = MemoryMarshal.Cast<byte, BlurhashHelper.RgbPixel>(blurBuf)
-		for (var i = 0; i < blurhashImage.Height; i++)
+		                              .AsSpan2D(blurhashImage.Height, blurhashImage.Width);
-		{
+		res.Blurhash = BlurhashHelper.Encode(blurPixels, 7, 7);
-			for (var j = 0; j < blurhashImage.Width; j++)
-			{
-				blurArr[j, i] = new Pixel(blurBuf[idx++] / 255d, blurBuf[idx++] / 255d,
-				                          blurBuf[idx++] / 255d);
-				idx += incr;
-			}
-		}
-
-		res.Blurhash = Blurhash.Core.Encode(blurArr, 7, 7, new Progress<int>());
 
 		if (genThumb)
 		{

Iceshrimp.Backend/Iceshrimp.Backend.csproj

@@ -17,6 +17,7 @@
         <PackageReference Include="AngleSharp" Version="1.1.2" />
         <PackageReference Include="AsyncKeyedLock" Version="7.0.0" />
         <PackageReference Include="Blurhash.ImageSharp" Version="3.0.0" />
+        <PackageReference Include="CommunityToolkit.HighPerformance" Version="8.2.2" />
         <PackageReference Include="dotNetRdf.Core" Version="3.2.9-iceshrimp" />
         <PackageReference Include="EntityFrameworkCore.Exceptions.PostgreSQL" Version="8.1.2" />
         <PackageReference Include="EntityFrameworkCore.Projectables" Version="3.0.4" />