notfire/Iceshrimp.NET
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[backend/drive] Significantly improve ImageSharp blurhash performance & memory efficiency

Browse source
This commit is contained in:
Laura Hausmann 2024-08-06 22:43:04 +02:00
parent b5a1c1ba85
commit d371e6732c
No known key found for this signature in database
GPG key ID: D044E84C5BE01605
3 changed files with 43 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
Iceshrimp.Backend/Core/Helpers/BlurhashHelper.cs
View file

@ -1,23 +1,22 @@
using System.Collections.Immutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using Blurhash;
using CommunityToolkit.HighPerformance;
using SixLabors.ImageSharp.PixelFormats;
namespace Iceshrimp.Backend.Core.Helpers;
// Adapted from https://github.com/MarkusPalcer/blurhash.net under MIT
public static class BlurhashHelper
{
private static readonly ImmutableArray<float> PrecomputedLut = [..Enumerable.Range(0, 256).Select(SRgbToLinear)];
/// <summary>
/// Encodes a Span2D of raw rgb data into a Blurhash string
/// Encodes a Span2D of raw pixel data into a Blurhash string
/// </summary>
/// <param name="pixels">The 2-dimensional array of pixels to encode</param>
/// <param name="pixels">The Span2D of raw pixel data to encode</param>
/// <param name="componentsX">The number of components used on the X-Axis for the DCT</param>
/// <param name="componentsY">The number of components used on the Y-Axis for the DCT</param>
/// <returns>The resulting Blurhash string</returns>
public static string Encode(Span2D<RgbPixel> pixels, int componentsX, int componentsY)
public static string Encode(Span2D<Rgb24> pixels, int componentsX, int componentsY)
{
if (componentsX < 1) throw new ArgumentException("componentsX needs to be at least 1");
if (componentsX > 9) throw new ArgumentException("componentsX needs to be at most 9");
@ -115,30 +114,21 @@ public static class BlurhashHelper
private static int EncodeAc(double r, double g, double b, double maximumValue)
{
var quantizedR = (int)Math.Max(0, Math.Min(18, Math.Floor(MathUtils.SignPow(r / maximumValue, 0.5) * 9 + 9.5)));
var quantizedG = (int)Math.Max(0, Math.Min(18, Math.Floor(MathUtils.SignPow(g / maximumValue, 0.5) * 9 + 9.5)));
var quantizedB = (int)Math.Max(0, Math.Min(18, Math.Floor(MathUtils.SignPow(b / maximumValue, 0.5) * 9 + 9.5)));
var quantizedR = (int)Math.Max(0, Math.Min(18, Math.Floor(SignPow(r / maximumValue, 0.5) * 9 + 9.5)));
var quantizedG = (int)Math.Max(0, Math.Min(18, Math.Floor(SignPow(g / maximumValue, 0.5) * 9 + 9.5)));
var quantizedB = (int)Math.Max(0, Math.Min(18, Math.Floor(SignPow(b / maximumValue, 0.5) * 9 + 9.5)));
return quantizedR * 19 * 19 + quantizedG * 19 + quantizedB;
}
private static int EncodeDc(double r, double g, double b)
{
var roundedR = MathUtils.LinearTosRgb(r);
var roundedG = MathUtils.LinearTosRgb(g);
var roundedB = MathUtils.LinearTosRgb(b);
var roundedR = LinearTosRgb(r);
var roundedG = LinearTosRgb(g);
var roundedB = LinearTosRgb(b);
return (roundedR << 16) + (roundedG << 8) + roundedB;
}
[StructLayout(LayoutKind.Sequential)]
[method: MethodImpl(MethodImplOptions.AggressiveInlining)]
public struct RgbPixel(byte r, byte g, byte b)
{
public readonly byte R = r;
public readonly byte G = g;
public readonly byte B = b;
}
private static void EncodeBase83(this int number, Span<char> output)
{
var length = output.Length;
@ -156,4 +146,23 @@ public static class BlurhashHelper
var num = value / (float)byte.MaxValue;
return (float)(num <= 0.04045 ? num / 12.92 : float.Pow((num + 0.055f) / 1.055f, 2.4f));
}
private static int LinearTosRgb(double value)
{
var v = Math.Max(0.0, Math.Min(1.0, value));
if (v <= 0.0031308) return (int)(v * 12.92 * 255 + 0.5);
return (int)((1.055 * Math.Pow(v, 1 / 2.4) - 0.055) * 255 + 0.5);
}
private static double SignPow(double @base, double exponent)
{
return Math.Sign(@base) * Math.Pow(Math.Abs(@base), exponent);
}
private struct Pixel(double red, double green, double blue)
{
public double Red = red;
public double Green = green;
public double Blue = blue;
}
}

29
Iceshrimp.Backend/Core/Services/ImageProcessor.cs
View file

@ -1,5 +1,4 @@
using System.Runtime.InteropServices;
using Blurhash.ImageSharp;
using CommunityToolkit.HighPerformance;
using Iceshrimp.Backend.Core.Configuration;
using Iceshrimp.Backend.Core.Database.Tables;
@ -14,10 +13,6 @@ using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
using ImageSharp = SixLabors.ImageSharp.Image;
#if EnableLibVips
using Blurhash;
#endif
namespace Iceshrimp.Backend.Core.Services;
public class ImageProcessor
@ -177,17 +172,18 @@ public class ImageProcessor
var properties = new DriveFile.FileProperties { Width = ident.Size.Width, Height = ident.Size.Height };
var res = new Result { Properties = properties };
// Calculate blurhash using a x200px image for improved performance
// Calculate blurhash using a x100px image for improved performance
{
using var image = await GetImage(data, ident, 200);
res.Blurhash = Blurhasher.Encode(image, 7, 7);
using var image = await GetImage<Rgb24>(data, ident, 100);
image.DangerousTryGetSinglePixelMemory(out var mem);
res.Blurhash = BlurhashHelper.Encode(mem.Span.AsSpan2D(image.Height, image.Width), 7, 7);
}
if (genThumb)
{
res.RenderThumbnail = async stream =>
{
using var image = await GetImage(data, ident, 1000);
using var image = await GetImage<Rgba32>(data, ident, 1000);
var thumbEncoder = new WebpEncoder { Quality = 75, FileFormat = WebpFileFormatType.Lossy };
await image.SaveAsWebpAsync(stream, thumbEncoder);
};
@ -197,7 +193,7 @@ public class ImageProcessor
{
res.RenderWebpublic = async stream =>
{
using var image = await GetImage(data, ident, 2048);
using var image = await GetImage<Rgba32>(data, ident, 2048);
var q = request.MimeType == "image/png" ? 100 : 75;
var thumbEncoder = new WebpEncoder { Quality = q, FileFormat = WebpFileFormatType.Lossy };
await image.SaveAsWebpAsync(stream, thumbEncoder);
@ -207,7 +203,9 @@ public class ImageProcessor
return res;
}
private static async Task<Image<Rgba32>> GetImage(Stream data, ImageInfo ident, int width, int? height = null)
private static async Task<Image<TPixel>> GetImage<TPixel>(
Stream data, ImageInfo ident, int width, int? height = null
) where TPixel : unmanaged, IPixel<TPixel>
{
width = Math.Min(ident.Width, width);
height = Math.Min(ident.Height, height ?? width);
@ -215,7 +213,7 @@ public class ImageProcessor
var options = new DecoderOptions { MaxFrames = 1, TargetSize = size };
data.Seek(0, SeekOrigin.Begin);
var image = await ImageSharp.LoadAsync<Rgba32>(options, data);
var image = await ImageSharp.LoadAsync<TPixel>(options, data);
image.Mutate(x => x.AutoOrient());
var opts = new ResizeOptions { Size = size, Mode = ResizeMode.Max };
image.Mutate(p => p.Resize(opts));
@ -223,7 +221,7 @@ public class ImageProcessor
}
#if EnableLibVips
private Task<Result> ProcessImageVips(
private static Task<Result> ProcessImageVips(
byte[] buf, ImageInfo ident, DriveFileCreationRequest request, bool genThumb, bool genWebp
)
{
@ -239,9 +237,8 @@ public class ImageProcessor
using var blurhashImageFlattened = blurhashImage.HasAlpha() ? blurhashImage.Flatten() : blurhashImage;
using var blurhashImageActual = blurhashImageFlattened.Cast(NetVips.Enums.BandFormat.Uchar);
var blurBuf = blurhashImageActual.WriteToMemory();
var blurPixels = MemoryMarshal.Cast<byte, BlurhashHelper.RgbPixel>(blurBuf)
.AsSpan2D(blurhashImage.Height, blurhashImage.Width);
var blurBuf = blurhashImageActual.WriteToMemory();
var blurPixels = MemoryMarshal.Cast<byte, Rgb24>(blurBuf).AsSpan2D(blurhashImage.Height, blurhashImage.Width);
res.Blurhash = BlurhashHelper.Encode(blurPixels, 7, 7);
if (genThumb)

1
Iceshrimp.Backend/Iceshrimp.Backend.csproj
View file

@ -16,7 +16,6 @@
<ItemGroup>
<PackageReference Include="AngleSharp" Version="1.1.2" />
<PackageReference Include="AsyncKeyedLock" Version="7.0.0" />
<PackageReference Include="Blurhash.ImageSharp" Version="3.0.0" />
<PackageReference Include="CommunityToolkit.HighPerformance" Version="8.2.2" />
<PackageReference Include="dotNetRdf.Core" Version="3.2.9-iceshrimp" />
<PackageReference Include="EntityFrameworkCore.Exceptions.PostgreSQL" Version="8.1.2" />