From 66c6f5be6f354a880f6766c63128ade950ca2cd8 Mon Sep 17 00:00:00 2001 From: Laura Hausmann Date: Fri, 9 Feb 2024 17:28:02 +0100 Subject: [PATCH] [backend/services] Add object storage service (ISH-10) --- .../Core/Configuration/Config.cs | 59 +++++++++----- Iceshrimp.Backend/Core/Configuration/Enums.cs | 5 ++ .../Core/Extensions/ServiceExtensions.cs | 5 ++ .../Extensions/WebApplicationExtensions.cs | 50 ++++++++++-- .../Core/Services/ObjectStorageService.cs | 81 +++++++++++++++++++ Iceshrimp.Backend/Iceshrimp.Backend.csproj | 5 +- Iceshrimp.Backend/Startup.cs | 2 +- Iceshrimp.Backend/configuration.ini | 18 +++++ 8 files changed, 195 insertions(+), 30 deletions(-) create mode 100644 Iceshrimp.Backend/Core/Services/ObjectStorageService.cs diff --git a/Iceshrimp.Backend/Core/Configuration/Config.cs b/Iceshrimp.Backend/Core/Configuration/Config.cs index 7fce83d2..5f3e69f7 100644 --- a/Iceshrimp.Backend/Core/Configuration/Config.cs +++ b/Iceshrimp.Backend/Core/Configuration/Config.cs @@ -4,10 +4,11 @@ using Iceshrimp.Backend.Core.Middleware; namespace Iceshrimp.Backend.Core.Configuration; public sealed class Config { - public required InstanceSection Instance { get; init; } - public required DatabaseSection Database { get; init; } - public required RedisSection Redis { get; init; } + public required InstanceSection Instance { get; init; } = new(); + public required DatabaseSection Database { get; init; } = new(); + public required RedisSection Redis { get; init; } = new(); public required SecuritySection Security { get; init; } = new(); + public required StorageSection Storage { get; init; } = new(); public sealed class InstanceSection { public readonly string Version; @@ -30,11 +31,11 @@ public sealed class Config { public string UserAgent => $"Iceshrimp.NET/{Version} (https://{WebDomain})"; - public required int ListenPort { get; init; } = 3000; - public required string ListenHost { get; init; } = "localhost"; - public required string WebDomain { get; init; } - public required string AccountDomain { get; init; } - public required int CharacterLimit { get; init; } = 8192; + public int ListenPort { get; init; } = 3000; + public string ListenHost { get; init; } = "localhost"; + public string WebDomain { get; init; } = null!; + public string AccountDomain { get; init; } = null!; + public int CharacterLimit { get; init; } = 8192; } public sealed class SecuritySection { @@ -47,21 +48,41 @@ public sealed class Config { } public sealed class DatabaseSection { - public required string Host { get; init; } = "localhost"; - public required int Port { get; init; } = 5432; - public required string Database { get; init; } - public required string Username { get; init; } - public string? Password { get; init; } + public string Host { get; init; } = "localhost"; + public int Port { get; init; } = 5432; + public string Database { get; init; } = null!; + public string Username { get; init; } = null!; + public string? Password { get; init; } } public sealed class RedisSection { - public required string Host { get; init; } = "localhost"; - public required int Port { get; init; } = 6379; - public string? Prefix { get; init; } - public string? Username { get; init; } - public string? Password { get; init; } - public int? Database { get; init; } + public string Host { get; init; } = "localhost"; + public int Port { get; init; } = 6379; + public string? Prefix { get; init; } + public string? Username { get; init; } + public string? Password { get; init; } + public int? Database { get; init; } //TODO: TLS settings } + + public sealed class StorageSection { + public Enums.FileStorage Mode { get; init; } = Enums.FileStorage.Local; + public LocalStorageSection? Local { get; init; } + public ObjectStorageSection? ObjectStorage { get; init; } + } + + public sealed class LocalStorageSection { + public string? Path { get; init; } + } + + public sealed class ObjectStorageSection { + public string? Endpoint { get; init; } + public string? Region { get; init; } + public string? AccessKey { get; init; } + public string? SecretKey { get; init; } + public string? Bucket { get; init; } + public string? Prefix { get; init; } + public string? AccessUrl { get; init; } + } } \ No newline at end of file diff --git a/Iceshrimp.Backend/Core/Configuration/Enums.cs b/Iceshrimp.Backend/Core/Configuration/Enums.cs index bccd4ef3..e608eb31 100644 --- a/Iceshrimp.Backend/Core/Configuration/Enums.cs +++ b/Iceshrimp.Backend/Core/Configuration/Enums.cs @@ -6,6 +6,11 @@ public static class Enums { AllowList = 1 } + public enum FileStorage { + Local = 0, + ObjectStorage = 1 + } + public enum ItemVisibility { Hide = 0, Registered = 1, diff --git a/Iceshrimp.Backend/Core/Extensions/ServiceExtensions.cs b/Iceshrimp.Backend/Core/Extensions/ServiceExtensions.cs index 27d92dc8..c1e5623c 100644 --- a/Iceshrimp.Backend/Core/Extensions/ServiceExtensions.cs +++ b/Iceshrimp.Backend/Core/Extensions/ServiceExtensions.cs @@ -46,6 +46,7 @@ public static class ServiceExtensions { .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() @@ -63,6 +64,10 @@ public static class ServiceExtensions { services.Configure(configuration.GetSection("Instance")); services.Configure(configuration.GetSection("Security")); services.Configure(configuration.GetSection("Database")); + services.Configure(configuration.GetSection("Redis")); + services.Configure(configuration.GetSection("Storage")); + services.Configure(configuration.GetSection("Storage:Local")); + services.Configure(configuration.GetSection("Storage:ObjectStorage")); } public static void AddDatabaseContext(this IServiceCollection services, IConfiguration configuration) { diff --git a/Iceshrimp.Backend/Core/Extensions/WebApplicationExtensions.cs b/Iceshrimp.Backend/Core/Extensions/WebApplicationExtensions.cs index fe7fe599..d95c11e9 100644 --- a/Iceshrimp.Backend/Core/Extensions/WebApplicationExtensions.cs +++ b/Iceshrimp.Backend/Core/Extensions/WebApplicationExtensions.cs @@ -1,6 +1,7 @@ using Iceshrimp.Backend.Core.Configuration; using Iceshrimp.Backend.Core.Database; using Iceshrimp.Backend.Core.Middleware; +using Iceshrimp.Backend.Core.Services; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Caching.Distributed; @@ -33,10 +34,13 @@ public static class WebApplicationExtensions { return app; } - public static Config.InstanceSection Initialize(this WebApplication app, string[] args) { + public static async Task Initialize(this WebApplication app, string[] args) { var instanceConfig = app.Configuration.GetSection("Instance").Get() ?? throw new Exception("Failed to read Instance config section"); + var storageConfig = app.Configuration.GetSection("Storage").Get() ?? + throw new Exception("Failed to read Storage config section"); + app.Logger.LogInformation("Iceshrimp.NET v{version} ({domain})", instanceConfig.Version, instanceConfig.AccountDomain); @@ -46,44 +50,74 @@ public static class WebApplicationExtensions { app.Logger.LogWarning("If this is not a local development instance, please set the environment to Production."); } - var provider = app.Services.CreateScope(); - var context = provider.ServiceProvider.GetService(); + var provider = app.Services.CreateScope().ServiceProvider; + var context = provider.GetService(); if (context == null) { app.Logger.LogCritical("Failed to initialize database context"); Environment.Exit(1); } app.Logger.LogInformation("Verifying database connection..."); - if (!context.Database.CanConnect()) { + if (!await context.Database.CanConnectAsync()) { app.Logger.LogCritical("Failed to connect to database"); Environment.Exit(1); } if (args.Contains("--migrate") || args.Contains("--migrate-and-start")) { app.Logger.LogInformation("Running migrations..."); - context.Database.Migrate(); + await context.Database.MigrateAsync(); if (args.Contains("--migrate")) Environment.Exit(0); } - else if (context.Database.GetPendingMigrations().Any()) { + else if ((await context.Database.GetPendingMigrationsAsync()).Any()) { app.Logger.LogCritical("Database has pending migrations, please restart with --migrate or --migrate-and-start"); Environment.Exit(1); } app.Logger.LogInformation("Verifying redis connection..."); - var cache = provider.ServiceProvider.GetService(); + var cache = provider.GetService(); if (cache == null) { app.Logger.LogCritical("Failed to initialize redis cache"); Environment.Exit(1); } try { - cache.Get("test"); + await cache.GetAsync("test"); } catch { app.Logger.LogCritical("Failed to connect to redis"); Environment.Exit(1); } + if (storageConfig.Mode == Enums.FileStorage.Local) { + if (string.IsNullOrWhiteSpace(storageConfig.Local?.Path) || + !Directory.Exists(storageConfig.Local.Path)) { + app.Logger.LogCritical("Local storage path does not exist"); + Environment.Exit(1); + } + else { + try { + var path = Path.Combine(storageConfig.Local.Path, Path.GetRandomFileName()); + + await using var fs = File.Create(path, 1, FileOptions.DeleteOnClose); + } + catch { + app.Logger.LogCritical("Local storage path is not accessible or not writable"); + Environment.Exit(1); + } + } + } + else if (storageConfig.Mode == Enums.FileStorage.ObjectStorage) { + app.Logger.LogInformation("Verifying object storage configuration..."); + var svc = provider.GetRequiredService(); + try { + await svc.VerifyCredentialsAsync(); + } + catch (Exception e) { + app.Logger.LogCritical("Failed to initialize object storage: {message}", e.Message); + Environment.Exit(1); + } + } + app.Logger.LogInformation("Initializing application, please wait..."); return instanceConfig; diff --git a/Iceshrimp.Backend/Core/Services/ObjectStorageService.cs b/Iceshrimp.Backend/Core/Services/ObjectStorageService.cs new file mode 100644 index 00000000..8bc652c6 --- /dev/null +++ b/Iceshrimp.Backend/Core/Services/ObjectStorageService.cs @@ -0,0 +1,81 @@ +using System.Text; +using Amazon; +using Amazon.S3; +using Carbon.Storage; +using Iceshrimp.Backend.Core.Configuration; +using Iceshrimp.Backend.Core.Helpers; +using Microsoft.Extensions.Options; + +namespace Iceshrimp.Backend.Core.Services; + +public class ObjectStorageService(IOptions config) { + private readonly string _accessUrl = config.Value.ObjectStorage?.AccessUrl ?? + throw new Exception("Invalid object storage access url"); + + private readonly S3Bucket _bucket = GetBucket(config); + + private readonly string? _prefix = config.Value.ObjectStorage?.Prefix?.Trim('/'); + + private static S3Bucket GetBucket(IOptions config) { + var s3Config = config.Value.ObjectStorage ?? throw new Exception("Invalid object storage configuration"); + + var region = s3Config.Region ?? throw new Exception("Invalid object storage region"); + var endpoint = s3Config.Endpoint ?? throw new Exception("Invalid object storage endpoint"); + var accessKey = s3Config.AccessKey ?? throw new Exception("Invalid object storage access key"); + var secretKey = s3Config.SecretKey ?? throw new Exception("Invalid object storage secret key"); + var bucket = s3Config.Bucket ?? throw new Exception("Invalid object storage bucket"); + + var client = new S3Client(new AwsRegion(region), endpoint, new AwsCredential(accessKey, secretKey)); + return new S3Bucket(bucket, client); + } + + public async Task VerifyCredentialsAsync() { + const string filename = ".iceshrimp-test"; + var content = CryptographyHelpers.GenerateRandomString(16); + + await UploadFileAsync(filename, Encoding.UTF8.GetBytes(content)); + + var httpClient = new HttpClient(); + + string result; + try { + result = await httpClient.GetStringAsync(GetFilePublicUrl(filename)); + } + catch (Exception e) { + throw new Exception($"Failed to verify access url: {e.Message}"); + } + + if (result == content) + return; + throw new Exception("Failed to verify access url (content mismatch)"); + } + + public async Task UploadFileAsync(string filename, byte[] data) { + await _bucket.PutAsync(new Blob(GetFilenameWithPrefix(filename), data)); + } + + public async Task UploadFileAsync(byte[] data) { + var filename = Guid.NewGuid().ToString().ToLowerInvariant(); + await _bucket.PutAsync(new Blob(GetFilenameWithPrefix(filename), data)); + return filename; + } + + public Uri GetFilePublicUrl(string filename) { + var baseUri = new Uri(_accessUrl); + return new Uri(baseUri, GetFilenameWithPrefix(filename)); + } + + public async ValueTask GetFileAsync(string filename) { + try { + var res = await _bucket.GetAsync(GetFilenameWithPrefix(filename)); + return await res.OpenAsync(); + } + catch { + return null; + } + } + + private string GetFilenameWithPrefix(string filename) { + return !string.IsNullOrWhiteSpace(_prefix) ? _prefix + "/" + filename : filename; + } +} \ No newline at end of file diff --git a/Iceshrimp.Backend/Iceshrimp.Backend.csproj b/Iceshrimp.Backend/Iceshrimp.Backend.csproj index 476c8b6c..4f32eb60 100644 --- a/Iceshrimp.Backend/Iceshrimp.Backend.csproj +++ b/Iceshrimp.Backend/Iceshrimp.Backend.csproj @@ -14,11 +14,12 @@ + - + @@ -48,7 +49,7 @@ - + diff --git a/Iceshrimp.Backend/Startup.cs b/Iceshrimp.Backend/Startup.cs index 1ac2da74..4d46a7dc 100644 --- a/Iceshrimp.Backend/Startup.cs +++ b/Iceshrimp.Backend/Startup.cs @@ -37,7 +37,7 @@ builder.Services.AddServices(); builder.Services.ConfigureServices(builder.Configuration); var app = builder.Build(); -var config = app.Initialize(args); +var config = await app.Initialize(args); // This determines the order of middleware execution in the request pipeline app.UseRouting(); diff --git a/Iceshrimp.Backend/configuration.ini b/Iceshrimp.Backend/configuration.ini index 388a9771..3d57f127 100644 --- a/Iceshrimp.Backend/configuration.ini +++ b/Iceshrimp.Backend/configuration.ini @@ -50,6 +50,24 @@ Port = 6379 ;;Password = ;;Database = 0 +[Storage] +;; Where to store media attachments +;; Options: [Local, ObjectStorage] +Mode = Local + +[Storage:Local] +;; Path where media is stored at. Must be writable for the service user. +Path = /path/to/media/location + +;;[Storage:ObjectStorage] +;;Endpoint = endpoint.example.org +;;Region = us-east-1 +;;KeyId = +;;SecretKey = +;;Bucket = +;;Prefix = +;;AccessUrl = https://endpoint.example.org/ + [Logging:LogLevel] Default = Information Microsoft.AspNetCore = Warning