From d371e6732cd8dd0e4e83d88d3476e59398645c64 Mon Sep 17 00:00:00 2001 From: Laura Hausmann Date: Tue, 6 Aug 2024 22:43:04 +0200 Subject: [PATCH] [backend/drive] Significantly improve ImageSharp blurhash performance & memory efficiency --- .../Core/Helpers/BlurhashHelper.cs | 51 +++++++++++-------- .../Core/Services/ImageProcessor.cs | 29 +++++------ Iceshrimp.Backend/Iceshrimp.Backend.csproj | 1 - 3 files changed, 43 insertions(+), 38 deletions(-) diff --git a/Iceshrimp.Backend/Core/Helpers/BlurhashHelper.cs b/Iceshrimp.Backend/Core/Helpers/BlurhashHelper.cs index 47cd2fd6..a06d90ab 100644 --- a/Iceshrimp.Backend/Core/Helpers/BlurhashHelper.cs +++ b/Iceshrimp.Backend/Core/Helpers/BlurhashHelper.cs @@ -1,23 +1,22 @@ using System.Collections.Immutable; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using Blurhash; using CommunityToolkit.HighPerformance; +using SixLabors.ImageSharp.PixelFormats; namespace Iceshrimp.Backend.Core.Helpers; +// Adapted from https://github.com/MarkusPalcer/blurhash.net under MIT public static class BlurhashHelper { private static readonly ImmutableArray PrecomputedLut = [..Enumerable.Range(0, 256).Select(SRgbToLinear)]; /// - /// Encodes a Span2D of raw rgb data into a Blurhash string + /// Encodes a Span2D of raw pixel data into a Blurhash string /// - /// The 2-dimensional array of pixels to encode + /// The Span2D of raw pixel data to encode /// The number of components used on the X-Axis for the DCT /// The number of components used on the Y-Axis for the DCT /// The resulting Blurhash string - public static string Encode(Span2D pixels, int componentsX, int componentsY) + public static string Encode(Span2D pixels, int componentsX, int componentsY) { if (componentsX < 1) throw new ArgumentException("componentsX needs to be at least 1"); if (componentsX > 9) throw new ArgumentException("componentsX needs to be at most 9"); @@ -115,30 +114,21 @@ public static class BlurhashHelper private static int EncodeAc(double r, double g, double b, double maximumValue) { - var quantizedR = (int)Math.Max(0, Math.Min(18, Math.Floor(MathUtils.SignPow(r / maximumValue, 0.5) * 9 + 9.5))); - var quantizedG = (int)Math.Max(0, Math.Min(18, Math.Floor(MathUtils.SignPow(g / maximumValue, 0.5) * 9 + 9.5))); - var quantizedB = (int)Math.Max(0, Math.Min(18, Math.Floor(MathUtils.SignPow(b / maximumValue, 0.5) * 9 + 9.5))); + var quantizedR = (int)Math.Max(0, Math.Min(18, Math.Floor(SignPow(r / maximumValue, 0.5) * 9 + 9.5))); + var quantizedG = (int)Math.Max(0, Math.Min(18, Math.Floor(SignPow(g / maximumValue, 0.5) * 9 + 9.5))); + var quantizedB = (int)Math.Max(0, Math.Min(18, Math.Floor(SignPow(b / maximumValue, 0.5) * 9 + 9.5))); return quantizedR * 19 * 19 + quantizedG * 19 + quantizedB; } private static int EncodeDc(double r, double g, double b) { - var roundedR = MathUtils.LinearTosRgb(r); - var roundedG = MathUtils.LinearTosRgb(g); - var roundedB = MathUtils.LinearTosRgb(b); + var roundedR = LinearTosRgb(r); + var roundedG = LinearTosRgb(g); + var roundedB = LinearTosRgb(b); return (roundedR << 16) + (roundedG << 8) + roundedB; } - [StructLayout(LayoutKind.Sequential)] - [method: MethodImpl(MethodImplOptions.AggressiveInlining)] - public struct RgbPixel(byte r, byte g, byte b) - { - public readonly byte R = r; - public readonly byte G = g; - public readonly byte B = b; - } - private static void EncodeBase83(this int number, Span output) { var length = output.Length; @@ -156,4 +146,23 @@ public static class BlurhashHelper var num = value / (float)byte.MaxValue; return (float)(num <= 0.04045 ? num / 12.92 : float.Pow((num + 0.055f) / 1.055f, 2.4f)); } + + private static int LinearTosRgb(double value) + { + var v = Math.Max(0.0, Math.Min(1.0, value)); + if (v <= 0.0031308) return (int)(v * 12.92 * 255 + 0.5); + return (int)((1.055 * Math.Pow(v, 1 / 2.4) - 0.055) * 255 + 0.5); + } + + private static double SignPow(double @base, double exponent) + { + return Math.Sign(@base) * Math.Pow(Math.Abs(@base), exponent); + } + + private struct Pixel(double red, double green, double blue) + { + public double Red = red; + public double Green = green; + public double Blue = blue; + } } \ No newline at end of file diff --git a/Iceshrimp.Backend/Core/Services/ImageProcessor.cs b/Iceshrimp.Backend/Core/Services/ImageProcessor.cs index e3ad7889..0a0c5ef7 100644 --- a/Iceshrimp.Backend/Core/Services/ImageProcessor.cs +++ b/Iceshrimp.Backend/Core/Services/ImageProcessor.cs @@ -1,5 +1,4 @@ using System.Runtime.InteropServices; -using Blurhash.ImageSharp; using CommunityToolkit.HighPerformance; using Iceshrimp.Backend.Core.Configuration; using Iceshrimp.Backend.Core.Database.Tables; @@ -14,10 +13,6 @@ using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.Processing; using ImageSharp = SixLabors.ImageSharp.Image; -#if EnableLibVips -using Blurhash; -#endif - namespace Iceshrimp.Backend.Core.Services; public class ImageProcessor @@ -177,17 +172,18 @@ public class ImageProcessor var properties = new DriveFile.FileProperties { Width = ident.Size.Width, Height = ident.Size.Height }; var res = new Result { Properties = properties }; - // Calculate blurhash using a x200px image for improved performance + // Calculate blurhash using a x100px image for improved performance { - using var image = await GetImage(data, ident, 200); - res.Blurhash = Blurhasher.Encode(image, 7, 7); + using var image = await GetImage(data, ident, 100); + image.DangerousTryGetSinglePixelMemory(out var mem); + res.Blurhash = BlurhashHelper.Encode(mem.Span.AsSpan2D(image.Height, image.Width), 7, 7); } if (genThumb) { res.RenderThumbnail = async stream => { - using var image = await GetImage(data, ident, 1000); + using var image = await GetImage(data, ident, 1000); var thumbEncoder = new WebpEncoder { Quality = 75, FileFormat = WebpFileFormatType.Lossy }; await image.SaveAsWebpAsync(stream, thumbEncoder); }; @@ -197,7 +193,7 @@ public class ImageProcessor { res.RenderWebpublic = async stream => { - using var image = await GetImage(data, ident, 2048); + using var image = await GetImage(data, ident, 2048); var q = request.MimeType == "image/png" ? 100 : 75; var thumbEncoder = new WebpEncoder { Quality = q, FileFormat = WebpFileFormatType.Lossy }; await image.SaveAsWebpAsync(stream, thumbEncoder); @@ -207,7 +203,9 @@ public class ImageProcessor return res; } - private static async Task> GetImage(Stream data, ImageInfo ident, int width, int? height = null) + private static async Task> GetImage( + Stream data, ImageInfo ident, int width, int? height = null + ) where TPixel : unmanaged, IPixel { width = Math.Min(ident.Width, width); height = Math.Min(ident.Height, height ?? width); @@ -215,7 +213,7 @@ public class ImageProcessor var options = new DecoderOptions { MaxFrames = 1, TargetSize = size }; data.Seek(0, SeekOrigin.Begin); - var image = await ImageSharp.LoadAsync(options, data); + var image = await ImageSharp.LoadAsync(options, data); image.Mutate(x => x.AutoOrient()); var opts = new ResizeOptions { Size = size, Mode = ResizeMode.Max }; image.Mutate(p => p.Resize(opts)); @@ -223,7 +221,7 @@ public class ImageProcessor } #if EnableLibVips - private Task ProcessImageVips( + private static Task ProcessImageVips( byte[] buf, ImageInfo ident, DriveFileCreationRequest request, bool genThumb, bool genWebp ) { @@ -239,9 +237,8 @@ public class ImageProcessor using var blurhashImageFlattened = blurhashImage.HasAlpha() ? blurhashImage.Flatten() : blurhashImage; using var blurhashImageActual = blurhashImageFlattened.Cast(NetVips.Enums.BandFormat.Uchar); - var blurBuf = blurhashImageActual.WriteToMemory(); - var blurPixels = MemoryMarshal.Cast(blurBuf) - .AsSpan2D(blurhashImage.Height, blurhashImage.Width); + var blurBuf = blurhashImageActual.WriteToMemory(); + var blurPixels = MemoryMarshal.Cast(blurBuf).AsSpan2D(blurhashImage.Height, blurhashImage.Width); res.Blurhash = BlurhashHelper.Encode(blurPixels, 7, 7); if (genThumb) diff --git a/Iceshrimp.Backend/Iceshrimp.Backend.csproj b/Iceshrimp.Backend/Iceshrimp.Backend.csproj index 85f00fe1..ac24f1c2 100644 --- a/Iceshrimp.Backend/Iceshrimp.Backend.csproj +++ b/Iceshrimp.Backend/Iceshrimp.Backend.csproj @@ -16,7 +16,6 @@ -