[backend/drive] Significantly improve LibVips blurhash performance & memory efficiency
This commit saves ~2.5ms & ~1.4MB of heap allocations during blurhash computation using the LibVips image processor.
This commit is contained in:
Laura Hausmann
parent
fcc04d5fd2
commit
b5a1c1ba85
3 changed files with 174 additions and 20 deletions
159
Iceshrimp.Backend/Core/Helpers/BlurhashHelper.cs
Normal file
159
Iceshrimp.Backend/Core/Helpers/BlurhashHelper.cs
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
using System.Collections.Immutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using Blurhash;
|
||||
using CommunityToolkit.HighPerformance;
|
||||
|
||||
namespace Iceshrimp.Backend.Core.Helpers;
|
||||
|
||||
public static class BlurhashHelper
|
||||
{
|
||||
private static readonly ImmutableArray<float> PrecomputedLut = [..Enumerable.Range(0, 256).Select(SRgbToLinear)];
|
||||
|
||||
/// <summary>
|
||||
/// Encodes a Span2D of raw rgb data into a Blurhash string
|
||||
/// </summary>
|
||||
/// <param name="pixels">The 2-dimensional array of pixels to encode</param>
|
||||
/// <param name="componentsX">The number of components used on the X-Axis for the DCT</param>
|
||||
/// <param name="componentsY">The number of components used on the Y-Axis for the DCT</param>
|
||||
/// <returns>The resulting Blurhash string</returns>
|
||||
public static string Encode(Span2D<RgbPixel> pixels, int componentsX, int componentsY)
|
||||
{
|
||||
if (componentsX < 1) throw new ArgumentException("componentsX needs to be at least 1");
|
||||
if (componentsX > 9) throw new ArgumentException("componentsX needs to be at most 9");
|
||||
if (componentsY < 1) throw new ArgumentException("componentsY needs to be at least 1");
|
||||
if (componentsY > 9) throw new ArgumentException("componentsY needs to be at most 9");
|
||||
|
||||
Span<Pixel> factors = stackalloc Pixel[componentsX * componentsY];
|
||||
Span<char> resultBuffer = stackalloc char[4 + 2 * componentsX * componentsY];
|
||||
Span<float> lut = stackalloc float[256];
|
||||
PrecomputedLut.CopyTo(lut);
|
||||
|
||||
var width = pixels.Width;
|
||||
var height = pixels.Height;
|
||||
|
||||
var xCosines = new double[width];
|
||||
var yCosines = new double[height];
|
||||
|
||||
for (var yComponent = 0; yComponent < componentsY; yComponent++)
|
||||
for (var xComponent = 0; xComponent < componentsX; xComponent++)
|
||||
{
|
||||
double r = 0, g = 0, b = 0;
|
||||
double normalization = xComponent == 0 && yComponent == 0 ? 1 : 2;
|
||||
|
||||
for (var xPixel = 0; xPixel < width; xPixel++)
|
||||
xCosines[xPixel] = Math.Cos(Math.PI * xComponent * xPixel / width);
|
||||
for (var yPixel = 0; yPixel < height; yPixel++)
|
||||
yCosines[yPixel] = Math.Cos(Math.PI * yComponent * yPixel / height);
|
||||
|
||||
for (var xPixel = 0; xPixel < width; xPixel++)
|
||||
for (var yPixel = 0; yPixel < height; yPixel++)
|
||||
{
|
||||
var basis = xCosines[xPixel] * yCosines[yPixel];
|
||||
var pixel = pixels[yPixel, xPixel];
|
||||
r += basis * lut[pixel.R];
|
||||
g += basis * lut[pixel.G];
|
||||
b += basis * lut[pixel.B];
|
||||
}
|
||||
|
||||
var scale = normalization / (width * height);
|
||||
factors[componentsX * yComponent + xComponent].Red = r * scale;
|
||||
factors[componentsX * yComponent + xComponent].Green = g * scale;
|
||||
factors[componentsX * yComponent + xComponent].Blue = b * scale;
|
||||
}
|
||||
|
||||
var dc = factors[0];
|
||||
var acCount = componentsX * componentsY - 1;
|
||||
|
||||
var sizeFlag = (componentsX - 1) + (componentsY - 1) * 9;
|
||||
sizeFlag.EncodeBase83(resultBuffer[..1]);
|
||||
|
||||
float maximumValue;
|
||||
if (acCount > 0)
|
||||
{
|
||||
// Get maximum absolute value of all AC components
|
||||
var actualMaximumValue = 0.0;
|
||||
for (var yComponent = 0; yComponent < componentsY; yComponent++)
|
||||
for (var xComponent = 0; xComponent < componentsX; xComponent++)
|
||||
{
|
||||
// Ignore DC component
|
||||
if (xComponent == 0 && yComponent == 0) continue;
|
||||
|
||||
var factorIndex = componentsX * yComponent + xComponent;
|
||||
|
||||
actualMaximumValue = Math.Max(Math.Abs(factors[factorIndex].Red), actualMaximumValue);
|
||||
actualMaximumValue = Math.Max(Math.Abs(factors[factorIndex].Green), actualMaximumValue);
|
||||
actualMaximumValue = Math.Max(Math.Abs(factors[factorIndex].Blue), actualMaximumValue);
|
||||
}
|
||||
|
||||
var quantizedMaximumValue = (int)Math.Max(0.0, Math.Min(82.0, Math.Floor(actualMaximumValue * 166 - 0.5)));
|
||||
maximumValue = ((float)quantizedMaximumValue + 1) / 166;
|
||||
quantizedMaximumValue.EncodeBase83(resultBuffer.Slice(1, 1));
|
||||
}
|
||||
else
|
||||
{
|
||||
maximumValue = 1;
|
||||
resultBuffer[1] = '0';
|
||||
}
|
||||
|
||||
EncodeDc(dc.Red, dc.Green, dc.Blue).EncodeBase83(resultBuffer.Slice(2, 4));
|
||||
|
||||
for (var yComponent = 0; yComponent < componentsY; yComponent++)
|
||||
for (var xComponent = 0; xComponent < componentsX; xComponent++)
|
||||
{
|
||||
// Ignore DC component
|
||||
if (xComponent == 0 && yComponent == 0) continue;
|
||||
|
||||
var factorIndex = componentsX * yComponent + xComponent;
|
||||
|
||||
EncodeAc(factors[factorIndex].Red, factors[factorIndex].Green, factors[factorIndex].Blue, maximumValue)
|
||||
.EncodeBase83(resultBuffer.Slice(6 + (factorIndex - 1) * 2, 2));
|
||||
}
|
||||
|
||||
return resultBuffer.ToString();
|
||||
}
|
||||
|
||||
private static int EncodeAc(double r, double g, double b, double maximumValue)
|
||||
{
|
||||
var quantizedR = (int)Math.Max(0, Math.Min(18, Math.Floor(MathUtils.SignPow(r / maximumValue, 0.5) * 9 + 9.5)));
|
||||
var quantizedG = (int)Math.Max(0, Math.Min(18, Math.Floor(MathUtils.SignPow(g / maximumValue, 0.5) * 9 + 9.5)));
|
||||
var quantizedB = (int)Math.Max(0, Math.Min(18, Math.Floor(MathUtils.SignPow(b / maximumValue, 0.5) * 9 + 9.5)));
|
||||
|
||||
return quantizedR * 19 * 19 + quantizedG * 19 + quantizedB;
|
||||
}
|
||||
|
||||
private static int EncodeDc(double r, double g, double b)
|
||||
{
|
||||
var roundedR = MathUtils.LinearTosRgb(r);
|
||||
var roundedG = MathUtils.LinearTosRgb(g);
|
||||
var roundedB = MathUtils.LinearTosRgb(b);
|
||||
return (roundedR << 16) + (roundedG << 8) + roundedB;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
[method: MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public struct RgbPixel(byte r, byte g, byte b)
|
||||
{
|
||||
public readonly byte R = r;
|
||||
public readonly byte G = g;
|
||||
public readonly byte B = b;
|
||||
}
|
||||
|
||||
private static void EncodeBase83(this int number, Span<char> output)
|
||||
{
|
||||
var length = output.Length;
|
||||
for (var index1 = 0; index1 < length; ++index1)
|
||||
{
|
||||
var index2 = number % 83;
|
||||
number /= 83;
|
||||
output[length - index1 - 1] =
|
||||
"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz#$%*+,-.:;=?@[]^_{|}~"[index2];
|
||||
}
|
||||
}
|
||||
|
||||
private static float SRgbToLinear(int value)
|
||||
{
|
||||
var num = value / (float)byte.MaxValue;
|
||||
return (float)(num <= 0.04045 ? num / 12.92 : float.Pow((num + 0.055f) / 1.055f, 2.4f));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,9 @@
|
|||
using System.Runtime.InteropServices;
|
||||
using Blurhash.ImageSharp;
|
||||
using CommunityToolkit.HighPerformance;
|
||||
using Iceshrimp.Backend.Core.Configuration;
|
||||
using Iceshrimp.Backend.Core.Database.Tables;
|
||||
using Iceshrimp.Backend.Core.Helpers;
|
||||
using Microsoft.Extensions.Options;
|
||||
using SixLabors.ImageSharp;
|
||||
using SixLabors.ImageSharp.Formats;
|
||||
|
|
@ -227,28 +230,19 @@ public class ImageProcessor
|
|||
var properties = new DriveFile.FileProperties { Width = ident.Size.Width, Height = ident.Size.Height };
|
||||
var res = new Result { Properties = properties };
|
||||
|
||||
// Calculate blurhash using a x200px image for improved performance
|
||||
// Calculate blurhash using a x100px image for improved performance
|
||||
using var blurhashImageSource =
|
||||
NetVips.Image.ThumbnailBuffer(buf, width: 200, height: 200, size: NetVips.Enums.Size.Down);
|
||||
NetVips.Image.ThumbnailBuffer(buf, width: 100, height: 100, size: NetVips.Enums.Size.Down);
|
||||
using var blurhashImage = blurhashImageSource.Interpretation == NetVips.Enums.Interpretation.Srgb
|
||||
? blurhashImageSource
|
||||
: blurhashImageSource.Colourspace(NetVips.Enums.Interpretation.Srgb);
|
||||
var blurBuf = blurhashImage.WriteToMemory();
|
||||
var blurArr = new Pixel[blurhashImage.Width, blurhashImage.Height];
|
||||
using var blurhashImageFlattened = blurhashImage.HasAlpha() ? blurhashImage.Flatten() : blurhashImage;
|
||||
using var blurhashImageActual = blurhashImageFlattened.Cast(NetVips.Enums.BandFormat.Uchar);
|
||||
|
||||
var idx = 0;
|
||||
var incr = blurhashImage.Bands - 3;
|
||||
for (var i = 0; i < blurhashImage.Height; i++)
|
||||
{
|
||||
for (var j = 0; j < blurhashImage.Width; j++)
|
||||
{
|
||||
blurArr[j, i] = new Pixel(blurBuf[idx++] / 255d, blurBuf[idx++] / 255d,
|
||||
blurBuf[idx++] / 255d);
|
||||
idx += incr;
|
||||
}
|
||||
}
|
||||
|
||||
res.Blurhash = Blurhash.Core.Encode(blurArr, 7, 7, new Progress<int>());
|
||||
var blurBuf = blurhashImageActual.WriteToMemory();
|
||||
var blurPixels = MemoryMarshal.Cast<byte, BlurhashHelper.RgbPixel>(blurBuf)
|
||||
.AsSpan2D(blurhashImage.Height, blurhashImage.Width);
|
||||
res.Blurhash = BlurhashHelper.Encode(blurPixels, 7, 7);
|
||||
|
||||
if (genThumb)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
<PackageReference Include="AngleSharp" Version="1.1.2" />
|
||||
<PackageReference Include="AsyncKeyedLock" Version="7.0.0" />
|
||||
<PackageReference Include="Blurhash.ImageSharp" Version="3.0.0" />
|
||||
<PackageReference Include="CommunityToolkit.HighPerformance" Version="8.2.2" />
|
||||
<PackageReference Include="dotNetRdf.Core" Version="3.2.9-iceshrimp" />
|
||||
<PackageReference Include="EntityFrameworkCore.Exceptions.PostgreSQL" Version="8.1.2" />
|
||||
<PackageReference Include="EntityFrameworkCore.Projectables" Version="3.0.4" />
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue