From 60545e25ae4ed78231c60c96d3ce5dc8470b29e0 Mon Sep 17 00:00:00 2001 From: Laura Hausmann Date: Sun, 7 Apr 2024 21:16:10 +0200 Subject: [PATCH] [backend/core] Fix sporadic "key not found" background task failures --- .../Core/Queues/BackgroundTaskQueue.cs | 14 ++-- .../Core/Services/ObjectStorageService.cs | 64 +++++++++++++++---- .../Core/Services/QueueService.cs | 2 +- 3 files changed, 59 insertions(+), 21 deletions(-) diff --git a/Iceshrimp.Backend/Core/Queues/BackgroundTaskQueue.cs b/Iceshrimp.Backend/Core/Queues/BackgroundTaskQueue.cs index fc180f30..2f7001b0 100644 --- a/Iceshrimp.Backend/Core/Queues/BackgroundTaskQueue.cs +++ b/Iceshrimp.Backend/Core/Queues/BackgroundTaskQueue.cs @@ -48,10 +48,12 @@ public class BackgroundTaskQueue() CancellationToken token ) { - var db = scope.GetRequiredService(); - var usedAsAvatarOrBanner = - await db.Users.AnyAsync(p => p.AvatarId == jobData.DriveFileId || - p.BannerId == jobData.DriveFileId, token); + var db = scope.GetRequiredService(); + var logger = scope.GetRequiredService>(); + logger.LogDebug("Expiring file {id}...", jobData.DriveFileId); + + var usedAsAvatarOrBanner = await db.Users.AnyAsync(p => p.AvatarId == jobData.DriveFileId || + p.BannerId == jobData.DriveFileId, token); var usedInNote = await db.Notes.AnyAsync(p => p.FileIds.Contains(jobData.DriveFileId), token); @@ -195,14 +197,14 @@ public class BackgroundTaskQueue() var eventSvc = scope.GetRequiredService(); eventSvc.RaiseUserUnmuted(null, muting.Muter, muting.Mutee); } - + private static async Task ProcessFilterExpiry( FilterExpiryJobData jobData, IServiceProvider scope, CancellationToken token ) { - var db = scope.GetRequiredService(); + var db = scope.GetRequiredService(); var filter = await db.Filters.FirstOrDefaultAsync(p => p.Id == jobData.FilterId, token); if (filter is not { Expiry: not null }) return; diff --git a/Iceshrimp.Backend/Core/Services/ObjectStorageService.cs b/Iceshrimp.Backend/Core/Services/ObjectStorageService.cs index 2517e3bc..60f0565f 100644 --- a/Iceshrimp.Backend/Core/Services/ObjectStorageService.cs +++ b/Iceshrimp.Backend/Core/Services/ObjectStorageService.cs @@ -9,42 +9,74 @@ using Microsoft.Extensions.Options; namespace Iceshrimp.Backend.Core.Services; -public class ObjectStorageService(IOptions config, HttpClient httpClient) +public class ObjectStorageService { - private readonly string? _accessUrl = config.Value.ObjectStorage?.AccessUrl; + private readonly string? _accessUrl; - private readonly S3Bucket? _bucket = GetBucketSafely(config); + private readonly S3Bucket? _bucket; + private readonly S3Client? _client; - private readonly string? _prefix = config.Value.ObjectStorage?.Prefix?.Trim('/'); + private readonly string? _prefix; + private readonly HttpClient _httpClient; - private static S3Bucket? GetBucketSafely(IOptions config) + public ObjectStorageService(IOptions config, HttpClient httpClient) { - if (config.Value.Mode != Enums.FileStorage.Local) return GetBucket(config); + _httpClient = httpClient; + _accessUrl = config.Value.ObjectStorage?.AccessUrl; + _client = GetClientSafely(config); + _bucket = GetBucketSafely(config, _client); + _prefix = config.Value.ObjectStorage?.Prefix?.Trim('/'); + } + private static S3Client? GetClientSafely(IOptions config) + { try { - return GetBucket(config); + return GetClient(config); } catch { + if (config.Value.Mode == Enums.FileStorage.ObjectStorage) + throw; + return null; } } - private static S3Bucket GetBucket(IOptions config) + private static S3Bucket? GetBucketSafely(IOptions config, S3Client? client) { - var s3Config = config.Value.ObjectStorage ?? throw new Exception("Invalid object storage configuration"); + try + { + return GetBucket(config, client); + } + catch + { + if (config.Value.Mode == Enums.FileStorage.ObjectStorage) + throw; + return null; + } + } + + private static S3Client GetClient(IOptions config) + { + var s3Config = config.Value.ObjectStorage ?? throw new Exception("Invalid object storage configuration"); var region = s3Config.Region ?? throw new Exception("Invalid object storage region"); var endpoint = s3Config.Endpoint ?? throw new Exception("Invalid object storage endpoint"); var accessKey = s3Config.KeyId ?? throw new Exception("Invalid object storage access key"); var secretKey = s3Config.SecretKey ?? throw new Exception("Invalid object storage secret key"); - var bucket = s3Config.Bucket ?? throw new Exception("Invalid object storage bucket"); + return new S3Client(new AwsRegion(region), endpoint, new AwsCredential(accessKey, secretKey)); + } + + private static S3Bucket GetBucket(IOptions config, S3Client? client) + { + if (client == null) throw new Exception("S3Client is null"); + var s3Config = config.Value.ObjectStorage ?? throw new Exception("Invalid object storage configuration"); + var bucket = s3Config.Bucket ?? throw new Exception("Invalid object storage bucket"); if (config.Value.ObjectStorage?.AccessUrl == null) throw new Exception("Invalid object storage access url"); - var client = new S3Client(new AwsRegion(region), endpoint, new AwsCredential(accessKey, secretKey)); return new S3Bucket(bucket, client); } @@ -58,7 +90,7 @@ public class ObjectStorageService(IOptions config, HttpCl string result; try { - result = await httpClient.GetStringAsync(GetFilePublicUrl(filename)); + result = await _httpClient.GetStringAsync(GetFilePublicUrl(filename)); } catch (Exception e) { @@ -113,9 +145,13 @@ public class ObjectStorageService(IOptions config, HttpCl public async Task RemoveFilesAsync(params string[] filenames) { - if (_bucket == null) + if (_bucket == null || _client == null) throw new Exception("Refusing to remove file from object storage with invalid configuration"); - await _bucket.DeleteAsync(filenames.Select(GetFilenameWithPrefix).ToImmutableList()); + + // We need to construct this request manually as the library will throw an exception on missing keys otherwise + var batch = new DeleteBatch(filenames.Select(GetFilenameWithPrefix).ToImmutableList(), quite: true); + var request = new DeleteObjectsRequest(_client.Host, _bucket.Name, batch); + await _client.DeleteObjectsAsync(request); } private string GetFilenameWithPrefix(string filename) diff --git a/Iceshrimp.Backend/Core/Services/QueueService.cs b/Iceshrimp.Backend/Core/Services/QueueService.cs index 2f0f3f4d..ce7ae2fe 100644 --- a/Iceshrimp.Backend/Core/Services/QueueService.cs +++ b/Iceshrimp.Backend/Core/Services/QueueService.cs @@ -436,7 +436,7 @@ public class PostgresJobQueue( } else { - logger.LogError("Failed to process job in {queue} queue: {error}", name, e.Message); + logger.LogError(e, "Failed to process job in {queue} queue:", name); } }