notfire/Iceshrimp.NET
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[backend/drive] Add advanced image processing pipeline configuration options (ISH-436, ISH-446)

Browse source 
This commit allows advanced configuration of the image processing pipeline. Exposed options include the format (WebP, AVIF & JXL), as well as encoding parameters. These can be set individually for any combination of image version (original/thumbnail/public) & image origin (local/remote).
This commit is contained in:
Laura Hausmann 2024-08-08 05:34:17 +02:00
parent c07bb35548
commit aaf3be209d
No known key found for this signature in database
GPG key ID: D044E84C5BE01605
3 changed files with 183 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
Iceshrimp.Backend/Core/Configuration/Config.cs
View file

@ -1,6 +1,7 @@
using System.ComponentModel.DataAnnotations;
using System.Diagnostics.CodeAnalysis;
using Iceshrimp.Backend.Core.Middleware;
using Iceshrimp.Backend.Core.Services.ImageProcessing;
using Iceshrimp.Shared.Helpers;
namespace Iceshrimp.Backend.Core.Configuration;
@ -63,8 +64,8 @@ public sealed class Config
public sealed class StorageSection
{
public readonly long? MaxCacheSizeBytes;
public readonly long? MaxUploadSizeBytes;
public readonly long? MaxCacheSizeBytes;
public readonly long? MaxUploadSizeBytes;
public readonly TimeSpan? MediaRetentionTimeSpan;
public bool CleanAvatars = false;
@ -199,6 +200,8 @@ public sealed class Config
public sealed class MediaProcessingSection
{
public ImagePipelineSection ImagePipeline { get; init; } = new();
public readonly int MaxFileSizeBytes = 10 * 1024 * 1024;
public Enums.ImageProcessor ImageProcessor { get; init; } = Enums.ImageProcessor.ImageSharp;
public int MaxResolutionMpx { get; init; } = 30;
@ -236,6 +239,54 @@ public sealed class Config
}
}
public sealed class ImagePipelineSection
{
public ImageVersion Original { get; init; } = new()
{
Local = new ImageFormatConfiguration { Format = ImageFormatEnum.Keep },
Remote = new ImageFormatConfiguration { Format = ImageFormatEnum.Keep }
};
public ImageVersion Thumbnail { get; init; } = new()
{
Local = new ImageFormatConfiguration { Format = ImageFormatEnum.Webp, TargetRes = 1000 },
Remote = new ImageFormatConfiguration
{
Format = ImageFormatEnum.Webp,
TargetRes = 1000,
QualityFactorPngSource = 75
}
};
public ImageVersion Public { get; init; } = new()
{
Local = new ImageFormatConfiguration { Format = ImageFormatEnum.Webp, TargetRes = 2048 },
Remote = new ImageFormatConfiguration { Format = ImageFormatEnum.None }
};
}
public class ImageVersion
{
[Required] public required ImageFormatConfiguration Local { get; init; }
[Required] public required ImageFormatConfiguration Remote { get; init; }
}
public class ImageFormatConfiguration
{
[Required] public required ImageFormatEnum Format { get; init; }
[Range(1, 100)] public int QualityFactor { get; init; } = 75;
[Range(1, 100)] public int QualityFactorPngSource { get; init; } = 100;
[Range(1, 10240)] public int? TargetRes { get; init; }
public ImageFormat.Webp.Compression WebpCompressionMode { get; init; } = ImageFormat.Webp.Compression.Lossy;
public ImageFormat.Avif.Compression AvifCompressionMode { get; init; } = ImageFormat.Avif.Compression.Lossy;
public ImageFormat.Jxl.Compression JxlCompressionMode { get; init; } = ImageFormat.Jxl.Compression.Lossy;
[Range(8, 12)] public int? AvifBitDepth { get; init; }
[Range(1, 9)] public int JxlEffort { get; init; } = 7;
}
public sealed class PerformanceSection
{
public QueueConcurrencySection QueueConcurrency { get; init; } = new();

56
Iceshrimp.Backend/Core/Services/DriveService.cs
View file

@ -262,7 +262,7 @@ public class DriveService(
skipImageProcessing = true;
}
var formats = GetFormats(user, ident, request, skipImageProcessing);
var formats = GetFormats(user, request, skipImageProcessing);
var res = imageProcessor.ProcessImage(buf, ident, request, formats);
properties = res;
blurhash = res.Blurhash;
@ -440,27 +440,49 @@ public class DriveService(
: mimeType;
}
private static List<ImageVersion> GetFormats(
User user, IImageInfo ident, DriveFileCreationRequest request, bool skipImageProcessing
private IReadOnlyCollection<ImageVersion> GetFormats(
User user, DriveFileCreationRequest request, bool skipImageProcessing
)
{
//TODO: make this configurable
var origFormat = new ImageFormat.Keep(Path.GetExtension(request.Filename), request.MimeType);
var orig = new ImageVersion(KeyEnum.Original, origFormat);
List<ImageVersion> res = [orig];
if (skipImageProcessing) return res;
res.Add(new ImageVersion(KeyEnum.Thumbnail, new ImageFormat.Webp(75, 1000)));
if (user.IsLocalUser)
if (skipImageProcessing)
{
var q = ident.MimeType is "image/png" ? 100 : 75;
res.Add(new ImageVersion(KeyEnum.Public, new ImageFormat.Webp(q, 2048)));
var origFormat = new ImageFormat.Keep(Path.GetExtension(request.Filename), request.MimeType);
return [new ImageVersion(KeyEnum.Original, origFormat)];
}
return res;
return Enum.GetValues<KeyEnum>()
.ToDictionary(p => p, p => GetFormatFromConfig(request, user, p))
.Where(p => p.Value != null)
.Select(p => new ImageVersion(p.Key, p.Value!))
.ToImmutableArray()
.AsReadOnly();
}
private ImageFormat? GetFormatFromConfig(DriveFileCreationRequest request, User user, KeyEnum key)
{
var ver = key switch
{
KeyEnum.Original => storageConfig.Value.MediaProcessing.ImagePipeline.Original,
KeyEnum.Thumbnail => storageConfig.Value.MediaProcessing.ImagePipeline.Thumbnail,
KeyEnum.Public => storageConfig.Value.MediaProcessing.ImagePipeline.Public,
_ => throw new ArgumentOutOfRangeException()
};
var config = user.IsLocalUser ? ver.Local : ver.Remote;
// @formatter:off
return config.Format switch
{
ImageFormatEnum.None => null,
ImageFormatEnum.Keep => new ImageFormat.Keep(Path.GetExtension(request.Filename), request.MimeType),
ImageFormatEnum.Webp => new ImageFormat.Webp(config.WebpCompressionMode, GetQualityFactor(), GetTargetRes()),
ImageFormatEnum.Avif => new ImageFormat.Avif(config.AvifCompressionMode, GetQualityFactor(), config.AvifBitDepth, GetTargetRes()),
ImageFormatEnum.Jxl => new ImageFormat.Jxl(config.JxlCompressionMode, GetQualityFactor(), config.JxlEffort, GetTargetRes()),
_ => throw new ArgumentOutOfRangeException()
};
int GetQualityFactor() => request.MimeType == "image/png" ? config.QualityFactorPngSource : config.QualityFactor;
int GetTargetRes() => config.TargetRes ?? throw new Exception("TargetRes is required to encode images");
// @formatter:on
}
/// <summary>

94
Iceshrimp.Backend/configuration.ini
View file

@ -161,9 +161,97 @@ MaxFileSize = 10M
;; Caution: metadata (e.g. location data) for locally originating images will *not* be stripped for files larger than this
MaxResolutionMpx = 30
[Storage:MediaProcessing:Formats]
Local = Webp
Remote = Webp
;; Should you prefer to reject locally originating images that exceed MaxResolutionMpx, set this option to true.
;; Note that this does not apply to remote images, or to local images in a format not supported by the configured image processor.
FailIfImageExceedsMaxRes = false
;; Maxmimum concurrent image encode tasks to run. (0 = no limit)
ImageProcessorConcurrency = 8
;; --------------------------------------------------------------------------------------------------------------------------------------------------------- ;;
;; The below section allows for detailed customization of the image processing pipeline. The respective defaults are listed below. ;;
;; Caution: this is an advanced feature, it's quite easy to break media / media federation by messing with this. Make sure you know what you are doing. ;;
;; ;;
;; Section keys follow the pattern Storage:MediaProcessing:ImagePipeline:<Version>:<Origin> ;;
;; Versions: ;;
;; - 'Original' is the canonical file. It's used when there is no 'Public' version available. ;;
;; - 'Thumbnail' is a compact thumbnail. It's used when a client requests it, usually for timeline rendering. ;;
;; - 'Public' is used in place of 'Original'. Its default purpose is to serve as a smaller version with stripped metadata for locally originating images. ;;
;; Origins: ;;
;; - 'Local' means that the owner of the file is a local user. ;;
;; - 'Remote' means that the owner of the file is a remote user. ;;
;; The full selection of encoding options is only specified once (for brevity). ;;
;; --------------------------------------------------------------------------------------------------------------------------------------------------------- ;;
;;[Storage:MediaProcessing:ImagePipeline:<Version>:<Origin>]
;; Which image format to use.
;; Options: [None, Keep, Webp, Avif, Jxl]
;; - 'None' doesn't store an image of the respective type. It is not valid for the 'Original' image version.
;; - 'Keep' doesn't transcode the image, but still performs other image processing tasks (e.g. blurhash computation & deduplication).
;; - 'Webp' encodes the image as WebP
;; - 'Avif' encodes the image as AVIF. Only available when ImageProcessor is set to LibVips.
;; - 'Jxl' encodes the image as JPEG-XL. Only available when ImageProcessor is set to LibVips.
;;Format = Keep
;;; - Generic encoding options - ;;;
;; The quality factor. Valid range: 1-100
;;QualityFactor = 75
;; The quality factor, when processing lossless png images. Valid range: 1-100
;;QualityFactorPngSource = 100
;; The resolution to scale the largest dimension to, in pixels. If the source image is smaller, no scaling is performed.
;;TargetRes = 2048
;;; - Webp encoding options - ;;;
;; The compression mode.
;; Options: [Lossy, NearLossless, Lossless]
;;WebpCompressionMode = Lossy
;;; - Avif encoding options - ;;;
;; The compression mode.
;; Options: [Lossy, Lossless]
;;AvifCompressionMode = Lossy
;; The bit depth. Valid range: 8-12. Leave unset to use source image bit depth.
;;AvifBitDepth = 8
;;; - Jxl encoding options - ;;;
;; The compression mode.
;; Options: [Lossy, Lossless]
;;JxlCompressionMode = Lossy
;; The encoding effort. Valid range: 1-9
;;JxlEffort = 7
[Storage:MediaProcessing:ImagePipeline:Original:Local]
Format = Keep
[Storage:MediaProcessing:ImagePipeline:Original:Remote]
Format = Keep
[Storage:MediaProcessing:ImagePipeline:Thumbnail:Local]
Format = Webp
TargetRes = 1000
[Storage:MediaProcessing:ImagePipeline:Thumbnail:Remote]
Format = Webp
TargetRes = 1000
QualityFactorPngSource = 75
[Storage:MediaProcessing:ImagePipeline:Public:Local]
;; Caution: locally originating public images are federated.
;; If remote instance software doesn't understand the format, they might fail to ingest the image or associated note.
Format = Webp
TargetRes = 2048
[Storage:MediaProcessing:ImagePipeline:Public:Remote]
Format = None
[Logging:LogLevel]
Default = Information