From 27f6e3790fcf69ca5ed029fe14f6c0020e689843 Mon Sep 17 00:00:00 2001 From: Kopper Date: Tue, 2 Jul 2024 03:37:10 +0300 Subject: [PATCH] [backend/api] Add emoji pack importer --- .../Controllers/EmojiController.cs | 12 ++ .../Core/Extensions/ServiceExtensions.cs | 1 + .../Core/Middleware/ErrorHandlerMiddleware.cs | 3 + .../Core/Services/EmojiImportService.cs | 111 ++++++++++++++++++ .../Core/Services/EmojiService.cs | 16 ++- 5 files changed, 137 insertions(+), 6 deletions(-) create mode 100644 Iceshrimp.Backend/Core/Services/EmojiImportService.cs diff --git a/Iceshrimp.Backend/Controllers/EmojiController.cs b/Iceshrimp.Backend/Controllers/EmojiController.cs index b818c19b..ac9a2d77 100644 --- a/Iceshrimp.Backend/Controllers/EmojiController.cs +++ b/Iceshrimp.Backend/Controllers/EmojiController.cs @@ -69,6 +69,7 @@ public class EmojiController( [HttpPost] [Authorize("role:admin")] [ProducesResponseType(StatusCodes.Status200OK, Type = typeof(EmojiResponse))] + [ProducesResponseType(StatusCodes.Status409Conflict, Type = typeof(ErrorResponse))] public async Task UploadEmoji(IFormFile file, [FromServices] IOptions config) { var emoji = await emojiSvc.CreateEmojiFromStream(file.OpenReadStream(), file.FileName, file.ContentType, @@ -104,6 +105,17 @@ public class EmojiController( return Ok(await emojiSvc.CloneEmoji(emojo)); } + [HttpPost("import")] + [Authorize("role:admin")] + [ProducesResponseType(StatusCodes.Status202Accepted)] + public async Task ImportEmoji(IFormFile file, [FromServices] EmojiImportService emojiImportSvc) + { + var zip = await emojiImportSvc.Parse(file.OpenReadStream()); + await emojiImportSvc.Import(zip); // TODO: run in background. this will take a while + + return Accepted(); + } + [HttpPatch("{id}")] [Authorize("role:admin")] [Consumes(MediaTypeNames.Application.Json)] diff --git a/Iceshrimp.Backend/Core/Extensions/ServiceExtensions.cs b/Iceshrimp.Backend/Core/Extensions/ServiceExtensions.cs index 19043ba1..2949b729 100644 --- a/Iceshrimp.Backend/Core/Extensions/ServiceExtensions.cs +++ b/Iceshrimp.Backend/Core/Extensions/ServiceExtensions.cs @@ -55,6 +55,7 @@ public static class ServiceExtensions .AddScoped() .AddScoped() .AddScoped() + .AddScoped() .AddScoped() .AddScoped() .AddScoped() diff --git a/Iceshrimp.Backend/Core/Middleware/ErrorHandlerMiddleware.cs b/Iceshrimp.Backend/Core/Middleware/ErrorHandlerMiddleware.cs index 9dff8dfc..def7228e 100644 --- a/Iceshrimp.Backend/Core/Middleware/ErrorHandlerMiddleware.cs +++ b/Iceshrimp.Backend/Core/Middleware/ErrorHandlerMiddleware.cs @@ -180,6 +180,9 @@ public class GracefulException( public static GracefulException RequestTimeout(string message, string? details = null) => new(HttpStatusCode.RequestTimeout, message, details); + public static GracefulException Conflict(string message, string? details = null) => + new(HttpStatusCode.Conflict, message, details); + public static GracefulException RecordNotFound() => new(HttpStatusCode.NotFound, "Record not found"); public static GracefulException MisdirectedRequest() => diff --git a/Iceshrimp.Backend/Core/Services/EmojiImportService.cs b/Iceshrimp.Backend/Core/Services/EmojiImportService.cs new file mode 100644 index 00000000..c12660e4 --- /dev/null +++ b/Iceshrimp.Backend/Core/Services/EmojiImportService.cs @@ -0,0 +1,111 @@ +using System.IO.Compression; +using System.Text.Json; +using Iceshrimp.Backend.Core.Database; +using Iceshrimp.Backend.Core.Middleware; +using Microsoft.AspNetCore.StaticFiles; +using Microsoft.Extensions.Options; +using static Iceshrimp.Backend.Core.Configuration.Config; + +namespace Iceshrimp.Backend.Core.Services; + +public class EmojiZipEmoji +{ + public string? Name { get; set; } + public string? Category { get; set; } + public List Aliases { get; set; } = []; +} + +public class EmojiZipEntry +{ + public required string FileName { get; set; } + public required EmojiZipEmoji Emoji { get; set; } +} + +public class EmojiZipMeta +{ + public required ushort MetaVersion { get; set; } + public required EmojiZipEntry[] Emojis { get; set; } +} + +public record EmojiZip(EmojiZipMeta Metadata, ZipArchive Archive); + +public class EmojiImportService( + EmojiService emojiSvc, + ILogger logger, + IOptions config +) +{ + public static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + public async Task Parse(Stream zipStream) + { + var archive = new ZipArchive(zipStream, ZipArchiveMode.Read); + + try + { + var meta = archive.GetEntry("meta.json") + ?? throw GracefulException.BadRequest("Invalid emoji zip. Only Misskey-style emoji zips are supported."); + + var metaJson = await JsonSerializer.DeserializeAsync(meta.Open(), SerializerOptions) + ?? throw GracefulException.BadRequest("Invalid emoji zip metadata"); + + if (metaJson.MetaVersion < 1 || metaJson.MetaVersion > 2) + throw GracefulException.BadRequest("Unrecognized metaVersion {version}, expected 1 or 2", metaJson.MetaVersion.ToString()); + + return new(metaJson, archive); + } + catch + { + // We don't want to dispose of archive on success, as Import will do it when it's done. + archive.Dispose(); + throw; + } + } + + public async Task Import(EmojiZip zip) + { + using var archive = zip.Archive; + var contentTypeProvider = new FileExtensionContentTypeProvider(); + + foreach (var emoji in zip.Metadata.Emojis) + { + var file = archive.GetEntry(emoji.FileName); + if (file == null) + { + logger.LogWarning("Skipping {file} as no such file was found in the zip.", emoji.FileName); + continue; + } + + if (!contentTypeProvider.TryGetContentType(emoji.FileName, out var mimeType)) + { + logger.LogWarning("Skipping {file} as the mime type could not be detemrined.", emoji.FileName); + continue; + } + + // DriveService requires a seekable and .Length-able stream, which the DeflateStream from file.Open does not support. + using var buffer = new MemoryStream((int)file.Length); + await file.Open().CopyToAsync(buffer); + buffer.Seek(0, SeekOrigin.Begin); + + var name = emoji.Emoji.Name ?? emoji.FileName; + + try + { + await emojiSvc.CreateEmojiFromStream( + buffer, + name, + mimeType, + config.Value, + emoji.Emoji.Aliases, + emoji.Emoji.Category + ); + + logger.LogDebug("Imported emoji {emoji}", name); + } + catch (GracefulException e) when (e.StatusCode == System.Net.HttpStatusCode.Conflict) + { + logger.LogDebug("Skipping {emoji} as it already exists.", name); + } + } + } +} \ No newline at end of file diff --git a/Iceshrimp.Backend/Core/Services/EmojiService.cs b/Iceshrimp.Backend/Core/Services/EmojiService.cs index 08db99f0..f0a1c4f8 100644 --- a/Iceshrimp.Backend/Core/Services/EmojiService.cs +++ b/Iceshrimp.Backend/Core/Services/EmojiService.cs @@ -26,9 +26,15 @@ public partial class EmojiService(DatabaseContext db, DriveService driveSvc, Sys new(@"^:?([\w+-]+)@([a-zA-Z0-9._\-]+\.[a-zA-Z0-9._\-]+):?$", RegexOptions.Compiled); public async Task CreateEmojiFromStream( - Stream input, string fileName, string mimeType, Config.InstanceSection config + Stream input, string fileName, string mimeType, Config.InstanceSection config, List? aliases = null, + string? category = null ) { + var name = fileName.Split(".")[0]; + var existing = await db.Emojis.AnyAsync(p => p.Host == null && p.Name == name); + if (existing) + throw GracefulException.Conflict("An emoji with that name already exists."); + var user = await sysUserSvc.GetInstanceActorAsync(); var request = new DriveFileCreationRequest { @@ -38,15 +44,13 @@ public partial class EmojiService(DatabaseContext db, DriveService driveSvc, Sys }; var driveFile = await driveSvc.StoreFile(input, user, request); - var name = fileName.Split(".")[0]; - - var existing = await db.Emojis.FirstOrDefaultAsync(p => p.Host == null && p.Name == name); - var id = IdHelpers.GenerateSlowflakeId(); var emoji = new Emoji { Id = id, - Name = existing == null && CustomEmojiRegex.IsMatch(name) ? name : id, + Name = name, + Aliases = aliases ?? [], + Category = category, UpdatedAt = DateTime.UtcNow, OriginalUrl = driveFile.Url, PublicUrl = driveFile.PublicUrl,