From 9d4d4a027e39676daf62622adb8dde824117ef7c Mon Sep 17 00:00:00 2001 From: pancakes Date: Wed, 12 Mar 2025 22:34:08 +1000 Subject: [PATCH] [backend/api] Add note export endpoint --- .../Controllers/Web/Schemas/ExportNote.cs | 39 +++++++ .../Controllers/Web/SettingsController.cs | 24 +++++ .../Core/Queues/BackgroundTaskQueue.cs | 101 ++++++++++++++++++ 3 files changed, 164 insertions(+) create mode 100644 Iceshrimp.Backend/Controllers/Web/Schemas/ExportNote.cs diff --git a/Iceshrimp.Backend/Controllers/Web/Schemas/ExportNote.cs b/Iceshrimp.Backend/Controllers/Web/Schemas/ExportNote.cs new file mode 100644 index 00000000..da08496e --- /dev/null +++ b/Iceshrimp.Backend/Controllers/Web/Schemas/ExportNote.cs @@ -0,0 +1,39 @@ +using Iceshrimp.Backend.Core.Database.Tables; +using Iceshrimp.Shared.Helpers; + +namespace Iceshrimp.Backend.Controllers.Web.Schemas; + +public class ExportNote : IIdentifiable +{ + public required string Id { get; set; } + public required string? Text { get; set; } + public required DateTime CreatedAt { get; set; } + public required List FileIds { get; set; } + public required List Files { get; set; } + public required string? ReplyId { get; set; } + public required string? RenoteId { get; set; } + public required object? Poll { get; set; } + public required string? Cw { get; set; } + public required Note.NoteVisibility Visibility { get; set; } + public required List VisibleUserIds { get; set; } + public required bool LocalOnly { get; set; } +} + +public class ExportFile : IIdentifiable +{ + public required string Id { get; set; } + public required DateTime CreatedAt { get; set; } + public required string Name { get; set; } + public required string Type { get; set; } + public required int Size { get; set; } + public required bool IsSensitive { get; set; } + public required string? Blurhash { get; set; } + public required DriveFile.FileProperties Properties { get; set; } + public required string Url { get; set; } + public required string ThumbnailUrl { get; set; } + public required string? Comment { get; set; } + public string? FolderId { get; set; } + public object? Folder { get; set; } + public string? UserId { get; set; } + public object? User { get; set; } +} diff --git a/Iceshrimp.Backend/Controllers/Web/SettingsController.cs b/Iceshrimp.Backend/Controllers/Web/SettingsController.cs index afb4d143..7f61a4e8 100644 --- a/Iceshrimp.Backend/Controllers/Web/SettingsController.cs +++ b/Iceshrimp.Backend/Controllers/Web/SettingsController.cs @@ -9,6 +9,7 @@ using Iceshrimp.Backend.Core.Database.Tables; using Iceshrimp.Backend.Core.Extensions; using Iceshrimp.Backend.Core.Helpers; using Iceshrimp.Backend.Core.Middleware; +using Iceshrimp.Backend.Core.Queues; using Iceshrimp.Backend.Core.Services; using Iceshrimp.Shared.Schemas.Web; using Microsoft.AspNetCore.Mvc; @@ -29,6 +30,7 @@ public class SettingsController( DatabaseContext db, ImportExportService importExportSvc, MetaService meta, + QueueService queueSvc, IOptions instance ) : ControllerBase { @@ -164,6 +166,28 @@ public class SettingsController( return File(Encoding.UTF8.GetBytes(following), "text/csv", $"following-{DateTime.Now:yyyy-MM-dd-HH-mm-ss}.csv"); } + // TODO: Choose a more appropriate rate limit + [HttpPost("export/notes")] + [EnableRateLimiting("imports")] + [ProducesResults(HttpStatusCode.Accepted)] + [ProducesErrors(HttpStatusCode.BadRequest)] + public async Task ExportNotes([FromQuery] bool includePrivate = false) + { + var user = HttpContext.GetUserOrFail(); + + var noteCount = await db.Notes + .CountAsync(p => p.UserId == user.Id); + if (noteCount < 1) + throw GracefulException.BadRequest("You do not have any notes"); + + await queueSvc.BackgroundTaskQueue.EnqueueAsync(new NoteExportJobData + { + UserId = user.Id, IncludePrivate = includePrivate + }); + + return Accepted(); + } + [HttpPost("import/following")] [EnableRateLimiting("imports")] [ProducesResults(HttpStatusCode.Accepted)] diff --git a/Iceshrimp.Backend/Core/Queues/BackgroundTaskQueue.cs b/Iceshrimp.Backend/Core/Queues/BackgroundTaskQueue.cs index 1f836c9a..542d2282 100644 --- a/Iceshrimp.Backend/Core/Queues/BackgroundTaskQueue.cs +++ b/Iceshrimp.Backend/Core/Queues/BackgroundTaskQueue.cs @@ -1,11 +1,14 @@ using System.Diagnostics.CodeAnalysis; +using System.Text.Json; using System.Text.Json.Serialization; +using Iceshrimp.Backend.Controllers.Web.Schemas; using Iceshrimp.Backend.Core.Configuration; using Iceshrimp.Backend.Core.Database; using Iceshrimp.Backend.Core.Database.Tables; using Iceshrimp.Backend.Core.Extensions; using Iceshrimp.Backend.Core.Services; using Iceshrimp.EntityFrameworkCore.Extensions; +using Iceshrimp.Shared.Configuration; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; using J = System.Text.Json.Serialization.JsonPropertyNameAttribute; @@ -50,6 +53,9 @@ public class BackgroundTaskQueue(int parallelism) case ProfileFieldUpdateJobData profileFieldUpdateJob: await ProcessProfileFieldUpdateAsync(profileFieldUpdateJob, scope, token); break; + case NoteExportJobData noteExportJob: + await ProcessNoteExportAsync(noteExportJob, scope, token); + break; } } @@ -342,6 +348,94 @@ public class BackgroundTaskQueue(int parallelism) logger.LogDebug("Profile fields for user {id} updated successfully", jobData.UserId); } + + private static async Task ProcessNoteExportAsync( + NoteExportJobData jobData, IServiceProvider scope, CancellationToken token + ) + { + var db = scope.GetRequiredService(); + var driveSvc = scope.GetRequiredService(); + var logger = scope.GetRequiredService>(); + + logger.LogDebug("Processing note export for user {id}", jobData.UserId); + + var user = await db.Users.FirstOrDefaultAsync(p => p.Id == jobData.UserId, token); + if (user == null) + { + logger.LogDebug("Failed to export notes for user {id}: user not found in database", jobData.UserId); + return; + } + + var fileIds = await db.Notes + .Where(p => p.UserId == user.Id + && (jobData.IncludePrivate || p.VisibilityIsPublicOrHome) + && p.FileIds.Count != 0) + .SelectMany(p => p.FileIds) + .ToListAsync(token); + + var files = await db.DriveFiles + .Where(p => fileIds.Any(id => p.Id == id)) + .Select(p => new ExportFile + { + Id = p.Id, + CreatedAt = p.CreatedAt, + Name = p.Name, + Type = p.Type, + Size = p.Size, + IsSensitive = p.IsSensitive, + Blurhash = p.Blurhash, + Properties = p.Properties, + Url = p.RawAccessUrl, + ThumbnailUrl = p.RawThumbnailAccessUrl, + Comment = p.Comment + }) + .ToListAsync(token); + + var notes = await db.Notes + .Where(p => p.UserId == user.Id && (jobData.IncludePrivate || p.VisibilityIsPublicOrHome)) + .Select(p => new ExportNote + { + Id = p.Id, + Text = p.Text, + CreatedAt = p.CreatedAt, + FileIds = p.FileIds, + Files = new List(), + ReplyId = p.ReplyId, + RenoteId = p.RenoteId, + Poll = null, + Cw = p.Cw, + Visibility = p.Visibility, + VisibleUserIds = p.VisibleUserIds, + LocalOnly = p.LocalOnly + }) + .ToListAsync(token); + + notes.ForEach(p => p.Files = files.Where(f => p.FileIds.Contains(f.Id)).ToList()); + + var request = new DriveFileCreationRequest + { + Comment = null, + Filename = $"notes-{DateTime.UtcNow:yyyy-MM-dd-HH-mm-ss}.json", + IsSensitive = jobData.IncludePrivate, + MimeType = "application/json", + RequestHeaders = null, + RequestIp = null, + Source = null, + Uri = null, + FolderId = null + }; + + var stream = new MemoryStream(); + await using var sr = new StreamWriter(stream); + + await sr.WriteAsync(JsonSerializer.Serialize(notes, JsonSerialization.Options)); + + await driveSvc.StoreFileAsync(stream, user, request, true); + + logger.LogDebug("Exported notes for user {id} successfully", jobData.UserId); + + // TODO: Send some kind of system notification letting the user know the file was created + } } [JsonDerivedType(typeof(DriveFileDeleteJobData), "driveFileDelete")] @@ -351,6 +445,7 @@ public class BackgroundTaskQueue(int parallelism) [JsonDerivedType(typeof(UserDeleteJobData), "userDelete")] [JsonDerivedType(typeof(UserPurgeJobData), "userPurge")] [JsonDerivedType(typeof(ProfileFieldUpdateJobData), "profileFieldUpdate")] +[JsonDerivedType(typeof(NoteExportJobData), "noteExport")] public abstract class BackgroundTaskJobData; public class DriveFileDeleteJobData : BackgroundTaskJobData @@ -387,4 +482,10 @@ public class UserPurgeJobData : BackgroundTaskJobData public class ProfileFieldUpdateJobData : BackgroundTaskJobData { [JR] [J("userId")] public required string UserId { get; set; } +} + +public class NoteExportJobData : BackgroundTaskJobData +{ + [JR] [J("userId")] public required string UserId { get; set; } + [JR] [J("includePrivate")] public required bool IncludePrivate { get; set; } } \ No newline at end of file