notfire/Iceshrimp.NET
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[backend/drive] Add basic image processing & thumbnail generation (ISH-63, ISH-64)

Browse source
This commit is contained in:
Laura Hausmann 2024-02-14 16:59:49 +01:00
parent 6f4d6df602
commit fc0f40f8ce
No known key found for this signature in database
GPG key ID: D044E84C5BE01605
4 changed files with 128 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Iceshrimp.Backend/Controllers/Mastodon/Renderers/NoteRenderer.cs
View file

@ -37,7 +37,7 @@ public class NoteRenderer(
attachments = await db.DriveFiles.Where(p => note.FileIds.Contains(p.Id))
.Select(f => new Attachment {
Id = f.Id,
Url = f.Url,
Url = f.WebpublicUrl ?? f.Url,
Blurhash = f.Blurhash,
PreviewUrl = f.ThumbnailUrl,
Description = f.Comment,

14
Iceshrimp.Backend/Core/Database/Tables/DriveFile.cs
View file

@ -190,9 +190,15 @@ public class DriveFile : IEntity {
public string Id { get; set; } = IdHelpers.GenerateSlowflakeId();
public class FileProperties {
[J("width")] public int? Width { get; set; }
[J("height")] public int? Height { get; set; }
[J("orientation")] public int? Orientation { get; set; }
[J("avgColor")] public string? AverageColor { get; set; }
[J("width")] public int? Width { get; set; }
[J("height")] public int? Height { get; set; }
[Obsolete("Deprecated property")]
[J("orientation")]
public int? Orientation { get; set; }
[Obsolete("Deprecated property")]
[J("avgColor")]
public string? AverageColor { get; set; }
}
}

8
Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs
View file

@ -53,10 +53,10 @@ public class NoteRenderer(IOptions<Config.InstanceSection> config, MfmConverter
? await db.DriveFiles
.Where(p => note.FileIds.Contains(p.Id) && p.UserHost == null)
.Select(p => new ASDocument {
Type = $"{Constants.ActivityStreamsNs}#Document",
Sensitive = p.IsSensitive,
Url = new ASObjectBase(p.Url),
MediaType = p.Type,
Type = $"{Constants.ActivityStreamsNs}#Document",
Sensitive = p.IsSensitive,
Url = new ASObjectBase(p.WebpublicUrl ?? p.Url),
MediaType = p.Type,
Description = p.Comment
})
.Cast<ASAttachment>()

147
Iceshrimp.Backend/Core/Services/DriveService.cs
View file

@ -9,7 +9,9 @@ using Iceshrimp.Backend.Core.Queues;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.Formats.Webp;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
namespace Iceshrimp.Backend.Core.Services;
@ -101,7 +103,17 @@ public class DriveService(
buf.Seek(0, SeekOrigin.Begin);
string? blurhash = null;
var shouldStore = storageConfig.Value.MediaRetention != null || user.Host == null;
var storedInternal = storageConfig.Value.Mode == Enums.FileStorage.Local;
if (request.Uri == null && user.Host != null)
throw GracefulException.UnprocessableEntity("Refusing to store file without uri for remote user");
string? blurhash = null;
Stream? thumbnail = null;
Stream? webpublic = null;
DriveFile.FileProperties? properties = null;
if (request.MimeType.StartsWith("image/") || request.MimeType == "image") {
try {
@ -111,22 +123,62 @@ public class DriveService(
// Correct mime type
if (request.MimeType == "image" && image.Metadata.DecodedImageFormat?.DefaultMimeType != null)
request.MimeType = image.Metadata.DecodedImageFormat.DefaultMimeType;
properties = new DriveFile.FileProperties {
Width = image.Size.Width,
Height = image.Size.Height
};
if (shouldStore) {
// Generate thumbnail
var thumbnailImage = image.Clone();
if (image.Size.Width > 1000)
thumbnailImage.Mutate(p => p.Resize(new Size(1000, 0)));
thumbnail = new MemoryStream();
await thumbnailImage.SaveAsWebpAsync(thumbnail);
thumbnail.Seek(0, SeekOrigin.Begin);
// Generate webpublic for local users
if (user.Host == null) {
var webpublicImage = image.Clone();
webpublicImage.Metadata.ExifProfile = null;
webpublicImage.Metadata.XmpProfile = null;
if (image.Size.Width > 2048)
webpublicImage.Mutate(p => p.Resize(new Size(2048, 0)));
var encoder = request.MimeType == "image/png"
? new WebpEncoder {
Quality = 100,
NearLossless = true,
NearLosslessQuality = 60
}
: new WebpEncoder { Quality = 75 };
webpublic = new MemoryStream();
await webpublicImage.SaveAsWebpAsync(webpublic, encoder);
webpublic.Seek(0, SeekOrigin.Begin);
}
}
}
catch {
logger.LogError("Failed to generate blurhash for image with mime type {type}", request.MimeType);
logger.LogError("Failed to generate blurhash & thumbnail for image with mime type {type}",
request.MimeType);
// We want to make sure no images are federated out without stripping metadata & converting to webp
if (user.Host == null) throw;
}
buf.Seek(0, SeekOrigin.Begin);
}
var (filename, guid) = GenerateFilenameKeepingExtension(request.Filename);
var shouldStore = storageConfig.Value.MediaRetention != null || user.Host == null;
var storedInternal = storageConfig.Value.Mode == Enums.FileStorage.Local;
string url;
string? thumbnailUrl = null;
string? webpublicUrl = null;
string url;
if (request.Uri == null && user.Host != null)
throw GracefulException.UnprocessableEntity("Refusing to store file without uri for remote user");
var filename = GenerateFilenameKeepingExtension(request.Filename);
var thumbnailFilename = thumbnail != null ? GenerateWebpFilename("thumbnail-") : null;
var webpublicFilename = webpublic != null ? GenerateWebpFilename("webpublic-") : null;
if (shouldStore) {
if (storedInternal) {
@ -137,10 +189,32 @@ public class DriveService(
await using var writer = File.OpenWrite(path);
await buf.CopyToAsync(writer);
url = $"https://{instanceConfig.Value.WebDomain}/files/{filename}";
if (thumbnailFilename != null && thumbnail is { Length: > 0 }) {
var thumbPath = Path.Combine(pathBase, thumbnailFilename);
await using var thumbWriter = File.OpenWrite(thumbPath);
await thumbnail.CopyToAsync(thumbWriter);
}
if (webpublicFilename != null && webpublic is { Length: > 0 }) {
var webpPath = Path.Combine(pathBase, webpublicFilename);
await using var webpWriter = File.OpenWrite(webpPath);
await webpublic.CopyToAsync(webpWriter);
}
}
else {
await storageSvc.UploadFileAsync(filename, data);
url = storageSvc.GetFilePublicUrl(filename).AbsoluteUri;
if (thumbnailFilename != null && thumbnail is { Length: > 0 }) {
await storageSvc.UploadFileAsync(thumbnailFilename, thumbnail);
thumbnailUrl = storageSvc.GetFilePublicUrl(thumbnailFilename).AbsoluteUri;
}
if (webpublicFilename != null && webpublic is { Length: > 0 }) {
await storageSvc.UploadFileAsync(webpublicFilename, webpublic);
webpublicUrl = storageSvc.GetFilePublicUrl(webpublicFilename).AbsoluteUri;
}
}
}
else {
@ -148,29 +222,29 @@ public class DriveService(
}
file = new DriveFile {
User = user,
UserHost = user.Host,
Sha256 = digest,
Size = (int)buf.Length,
IsLink = !shouldStore && user.Host != null,
AccessKey = filename,
IsSensitive = request.IsSensitive,
StoredInternal = storedInternal,
Src = request.Source,
Uri = request.Uri,
Url = url,
Name = request.Filename,
Comment = request.Comment,
Type = CleanMimeType(request.MimeType),
RequestHeaders = request.RequestHeaders,
RequestIp = request.RequestIp,
Blurhash = blurhash,
//Properties = TODO,
//ThumbnailUrl = TODO,
//ThumbnailAccessKey = TODO,
//WebpublicType = TODO,
//WebpublicUrl = TODO,
//WebpublicAccessKey = TODO,
User = user,
UserHost = user.Host,
Sha256 = digest,
Size = (int)buf.Length,
IsLink = !shouldStore && user.Host != null,
AccessKey = filename,
IsSensitive = request.IsSensitive,
StoredInternal = storedInternal,
Src = request.Source,
Uri = request.Uri,
Url = url,
Name = request.Filename,
Comment = request.Comment,
Type = CleanMimeType(request.MimeType),
RequestHeaders = request.RequestHeaders,
RequestIp = request.RequestIp,
Blurhash = blurhash,
Properties = properties,
ThumbnailUrl = thumbnailUrl,
ThumbnailAccessKey = thumbnailFilename,
WebpublicType = webpublicUrl != null ? "image/webp" : null,
WebpublicUrl = webpublicUrl,
WebpublicAccessKey = webpublicFilename,
};
await db.AddAsync(file);
@ -194,10 +268,15 @@ public class DriveService(
: file.WebpublicUrl ?? file.Url;
}
private static (string filename, string guid) GenerateFilenameKeepingExtension(string filename) {
private static string GenerateFilenameKeepingExtension(string filename) {
var guid = Guid.NewGuid().ToString().ToLowerInvariant();
var ext = Path.GetExtension(filename);
return (guid + ext, guid);
return guid + ext;
}
private static string GenerateWebpFilename(string prefix = "") {
var guid = Guid.NewGuid().ToString().ToLowerInvariant();
return $"{prefix}{guid}.webp";
}
private static string CleanMimeType(string? mimeType) {