[backend/services] Add object storage service (ISH-10)

Iceshrimp.Backend/Core/Configuration/Config.cs

@@ -4,10 +4,11 @@ using Iceshrimp.Backend.Core.Middleware;
 namespace Iceshrimp.Backend.Core.Configuration;
 
 public sealed class Config {
-	public required InstanceSection Instance { get; init; }
+	public required InstanceSection Instance { get; init; } = new();
-	public required DatabaseSection Database { get; init; }
+	public required DatabaseSection Database { get; init; } = new();
-	public required RedisSection    Redis    { get; init; }
+	public required RedisSection    Redis    { get; init; } = new();
 	public required SecuritySection Security { get; init; } = new();
+	public required StorageSection  Storage  { get; init; } = new();
 
 	public sealed class InstanceSection {
 		public readonly string Version;
@@ -30,11 +31,11 @@ public sealed class Config {
 
 		public string UserAgent => $"Iceshrimp.NET/{Version} (https://{WebDomain})";
 
-		public required int    ListenPort     { get; init; } = 3000;
+		public int    ListenPort     { get; init; } = 3000;
-		public required string ListenHost     { get; init; } = "localhost";
+		public string ListenHost     { get; init; } = "localhost";
-		public required string WebDomain      { get; init; }
+		public string WebDomain      { get; init; } = null!;
-		public required string AccountDomain  { get; init; }
+		public string AccountDomain  { get; init; } = null!;
-		public required int    CharacterLimit { get; init; } = 8192;
+		public int    CharacterLimit { get; init; } = 8192;
 	}
 
 	public sealed class SecuritySection {
@@ -47,16 +48,16 @@ public sealed class Config {
 	}
 
 	public sealed class DatabaseSection {
-		public required string  Host     { get; init; } = "localhost";
+		public string  Host     { get; init; } = "localhost";
-		public required int     Port     { get; init; } = 5432;
+		public int     Port     { get; init; } = 5432;
-		public required string  Database { get; init; }
+		public string  Database { get; init; } = null!;
-		public required string  Username { get; init; }
+		public string  Username { get; init; } = null!;
 		public string? Password { get; init; }
 	}
 
 	public sealed class RedisSection {
-		public required string  Host     { get; init; } = "localhost";
+		public string  Host     { get; init; } = "localhost";
-		public required int     Port     { get; init; } = 6379;
+		public int     Port     { get; init; } = 6379;
 		public string? Prefix   { get; init; }
 		public string? Username { get; init; }
 		public string? Password { get; init; }
@@ -64,4 +65,24 @@ public sealed class Config {
 
 		//TODO: TLS settings
 	}
+
+	public sealed class StorageSection {
+		public Enums.FileStorage     Mode          { get; init; } = Enums.FileStorage.Local;
+		public LocalStorageSection?  Local         { get; init; }
+		public ObjectStorageSection? ObjectStorage { get; init; }
+	}
+
+	public sealed class LocalStorageSection {
+		public string? Path { get; init; }
+	}
+
+	public sealed class ObjectStorageSection {
+		public string? Endpoint  { get; init; }
+		public string? Region    { get; init; }
+		public string? AccessKey { get; init; }
+		public string? SecretKey { get; init; }
+		public string? Bucket    { get; init; }
+		public string? Prefix    { get; init; }
+		public string? AccessUrl { get; init; }
+	}
 }

Iceshrimp.Backend/Core/Configuration/Enums.cs

@@ -6,6 +6,11 @@ public static class Enums {
 		AllowList = 1
 	}
 
+	public enum FileStorage {
+		Local         = 0,
+		ObjectStorage = 1
+	}
+
 	public enum ItemVisibility {
 		Hide       = 0,
 		Registered = 1,

Iceshrimp.Backend/Core/Extensions/ServiceExtensions.cs

@@ -46,6 +46,7 @@ public static class ServiceExtensions {
 			.AddSingleton<HttpClient>()
 			.AddSingleton<HttpRequestService>()
 			.AddSingleton<QueueService>()
+			.AddSingleton<ObjectStorageService>()
 			.AddSingleton<ErrorHandlerMiddleware>()
 			.AddSingleton<RequestBufferingMiddleware>()
 			.AddSingleton<AuthorizationMiddleware>()
@@ -63,6 +64,10 @@ public static class ServiceExtensions {
 		services.Configure<Config.InstanceSection>(configuration.GetSection("Instance"));
 		services.Configure<Config.SecuritySection>(configuration.GetSection("Security"));
 		services.Configure<Config.DatabaseSection>(configuration.GetSection("Database"));
+		services.Configure<Config.RedisSection>(configuration.GetSection("Redis"));
+		services.Configure<Config.StorageSection>(configuration.GetSection("Storage"));
+		services.Configure<Config.LocalStorageSection>(configuration.GetSection("Storage:Local"));
+		services.Configure<Config.ObjectStorageSection>(configuration.GetSection("Storage:ObjectStorage"));
 	}
 
 	public static void AddDatabaseContext(this IServiceCollection services, IConfiguration configuration) {

Iceshrimp.Backend/Core/Extensions/WebApplicationExtensions.cs

@@ -1,6 +1,7 @@
 using Iceshrimp.Backend.Core.Configuration;
 using Iceshrimp.Backend.Core.Database;
 using Iceshrimp.Backend.Core.Middleware;
+using Iceshrimp.Backend.Core.Services;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Caching.Distributed;
 
@@ -33,10 +34,13 @@ public static class WebApplicationExtensions {
 		return app;
 	}
 
-	public static Config.InstanceSection Initialize(this WebApplication app, string[] args) {
+	public static async Task<Config.InstanceSection> Initialize(this WebApplication app, string[] args) {
 		var instanceConfig = app.Configuration.GetSection("Instance").Get<Config.InstanceSection>() ??
 		                     throw new Exception("Failed to read Instance config section");
 
+		var storageConfig = app.Configuration.GetSection("Storage").Get<Config.StorageSection>() ??
+		                    throw new Exception("Failed to read Storage config section");
+
 		app.Logger.LogInformation("Iceshrimp.NET v{version} ({domain})", instanceConfig.Version,
 		                          instanceConfig.AccountDomain);
 
@@ -46,44 +50,74 @@ public static class WebApplicationExtensions {
 			app.Logger.LogWarning("If this is not a local development instance, please set the environment to Production.");
 		}
 
-		var provider = app.Services.CreateScope();
+		var provider = app.Services.CreateScope().ServiceProvider;
-		var context  = provider.ServiceProvider.GetService<DatabaseContext>();
+		var context  = provider.GetService<DatabaseContext>();
 		if (context == null) {
 			app.Logger.LogCritical("Failed to initialize database context");
 			Environment.Exit(1);
 		}
 
 		app.Logger.LogInformation("Verifying database connection...");
-		if (!context.Database.CanConnect()) {
+		if (!await context.Database.CanConnectAsync()) {
 			app.Logger.LogCritical("Failed to connect to database");
 			Environment.Exit(1);
 		}
 
 		if (args.Contains("--migrate") || args.Contains("--migrate-and-start")) {
 			app.Logger.LogInformation("Running migrations...");
-			context.Database.Migrate();
+			await context.Database.MigrateAsync();
 			if (args.Contains("--migrate")) Environment.Exit(0);
 		}
-		else if (context.Database.GetPendingMigrations().Any()) {
+		else if ((await context.Database.GetPendingMigrationsAsync()).Any()) {
 			app.Logger.LogCritical("Database has pending migrations, please restart with --migrate or --migrate-and-start");
 			Environment.Exit(1);
 		}
 
 		app.Logger.LogInformation("Verifying redis connection...");
-		var cache = provider.ServiceProvider.GetService<IDistributedCache>();
+		var cache = provider.GetService<IDistributedCache>();
 		if (cache == null) {
 			app.Logger.LogCritical("Failed to initialize redis cache");
 			Environment.Exit(1);
 		}
 
 		try {
-			cache.Get("test");
+			await cache.GetAsync("test");
 		}
 		catch {
 			app.Logger.LogCritical("Failed to connect to redis");
 			Environment.Exit(1);
 		}
 
+		if (storageConfig.Mode == Enums.FileStorage.Local) {
+			if (string.IsNullOrWhiteSpace(storageConfig.Local?.Path) ||
+			    !Directory.Exists(storageConfig.Local.Path)) {
+				app.Logger.LogCritical("Local storage path does not exist");
+				Environment.Exit(1);
+			}
+			else {
+				try {
+					var path = Path.Combine(storageConfig.Local.Path, Path.GetRandomFileName());
+
+					await using var fs = File.Create(path, 1, FileOptions.DeleteOnClose);
+				}
+				catch {
+					app.Logger.LogCritical("Local storage path is not accessible or not writable");
+					Environment.Exit(1);
+				}
+			}
+		}
+		else if (storageConfig.Mode == Enums.FileStorage.ObjectStorage) {
+			app.Logger.LogInformation("Verifying object storage configuration...");
+			var svc = provider.GetRequiredService<ObjectStorageService>();
+			try {
+				await svc.VerifyCredentialsAsync();
+			}
+			catch (Exception e) {
+				app.Logger.LogCritical("Failed to initialize object storage: {message}", e.Message);
+				Environment.Exit(1);
+			}
+		}
+
 		app.Logger.LogInformation("Initializing application, please wait...");
 
 		return instanceConfig;

Iceshrimp.Backend/Core/Services/ObjectStorageService.cs

@@ -0,0 +1,81 @@
+using System.Text;
+using Amazon;
+using Amazon.S3;
+using Carbon.Storage;
+using Iceshrimp.Backend.Core.Configuration;
+using Iceshrimp.Backend.Core.Helpers;
+using Microsoft.Extensions.Options;
+
+namespace Iceshrimp.Backend.Core.Services;
+
+public class ObjectStorageService(IOptions<Config.StorageSection> config) {
+	private readonly string _accessUrl = config.Value.ObjectStorage?.AccessUrl ??
+	                                     throw new Exception("Invalid object storage access url");
+
+	private readonly S3Bucket _bucket = GetBucket(config);
+
+	private readonly string? _prefix = config.Value.ObjectStorage?.Prefix?.Trim('/');
+
+	private static S3Bucket GetBucket(IOptions<Config.StorageSection> config) {
+		var s3Config = config.Value.ObjectStorage ?? throw new Exception("Invalid object storage configuration");
+
+		var region    = s3Config.Region ?? throw new Exception("Invalid object storage region");
+		var endpoint  = s3Config.Endpoint ?? throw new Exception("Invalid object storage endpoint");
+		var accessKey = s3Config.AccessKey ?? throw new Exception("Invalid object storage access key");
+		var secretKey = s3Config.SecretKey ?? throw new Exception("Invalid object storage secret key");
+		var bucket    = s3Config.Bucket ?? throw new Exception("Invalid object storage bucket");
+
+		var client = new S3Client(new AwsRegion(region), endpoint, new AwsCredential(accessKey, secretKey));
+		return new S3Bucket(bucket, client);
+	}
+
+	public async Task VerifyCredentialsAsync() {
+		const string filename = ".iceshrimp-test";
+		var          content  = CryptographyHelpers.GenerateRandomString(16);
+
+		await UploadFileAsync(filename, Encoding.UTF8.GetBytes(content));
+
+		var httpClient = new HttpClient();
+
+		string result;
+		try {
+			result = await httpClient.GetStringAsync(GetFilePublicUrl(filename));
+		}
+		catch (Exception e) {
+			throw new Exception($"Failed to verify access url: {e.Message}");
+		}
+
+		if (result == content)
+			return;
+		throw new Exception("Failed to verify access url (content mismatch)");
+	}
+
+	public async Task UploadFileAsync(string filename, byte[] data) {
+		await _bucket.PutAsync(new Blob(GetFilenameWithPrefix(filename), data));
+	}
+
+	public async Task<string> UploadFileAsync(byte[] data) {
+		var filename = Guid.NewGuid().ToString().ToLowerInvariant();
+		await _bucket.PutAsync(new Blob(GetFilenameWithPrefix(filename), data));
+		return filename;
+	}
+
+	public Uri GetFilePublicUrl(string filename) {
+		var baseUri = new Uri(_accessUrl);
+		return new Uri(baseUri, GetFilenameWithPrefix(filename));
+	}
+
+	public async ValueTask<Stream?> GetFileAsync(string filename) {
+		try {
+			var res = await _bucket.GetAsync(GetFilenameWithPrefix(filename));
+			return await res.OpenAsync();
+		}
+		catch {
+			return null;
+		}
+	}
+
+	private string GetFilenameWithPrefix(string filename) {
+		return !string.IsNullOrWhiteSpace(_prefix) ? _prefix + "/" + filename : filename;
+	}
+}

Iceshrimp.Backend/Iceshrimp.Backend.csproj

@@ -14,11 +14,12 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <PackageReference Include="Amazon.S3" Version="0.31.2"/>
         <PackageReference Include="AngleSharp" Version="1.1.0"/>
         <PackageReference Include="Asp.Versioning.Http" Version="8.0.0"/>
         <PackageReference Include="cuid.net" Version="5.0.2"/>
         <PackageReference Include="dotNetRdf.Core" Version="3.2.1-dev"/>
-        <PackageReference Include="EntityFrameworkCore.Projectables" Version="3.0.4" />
+        <PackageReference Include="EntityFrameworkCore.Projectables" Version="3.0.4"/>
         <PackageReference Include="Isopoh.Cryptography.Argon2" Version="2.0.0"/>
         <PackageReference Include="Microsoft.AspNetCore.DataProtection.EntityFrameworkCore" Version="8.0.1"/>
         <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.0"/>
@@ -48,7 +49,7 @@
     </ItemGroup>
 
     <ItemGroup>
-      <None Remove="migrate.sql" />
+        <None Remove="migrate.sql"/>
     </ItemGroup>
 
 </Project>

Iceshrimp.Backend/Startup.cs

@@ -37,7 +37,7 @@ builder.Services.AddServices();
 builder.Services.ConfigureServices(builder.Configuration);
 
 var app    = builder.Build();
-var config = app.Initialize(args);
+var config = await app.Initialize(args);
 
 // This determines the order of middleware execution in the request pipeline
 app.UseRouting();

Iceshrimp.Backend/configuration.ini

@@ -50,6 +50,24 @@ Port = 6379
 ;;Password =
 ;;Database = 0
 
+[Storage]
+;; Where to store media attachments
+;; Options: [Local, ObjectStorage]
+Mode = Local
+
+[Storage:Local]
+;; Path where media is stored at. Must be writable for the service user.
+Path = /path/to/media/location
+
+;;[Storage:ObjectStorage]
+;;Endpoint = endpoint.example.org
+;;Region = us-east-1
+;;KeyId = <yourAccessKey>
+;;SecretKey = <yourSecretKey>
+;;Bucket = <yourBucketName>
+;;Prefix =
+;;AccessUrl = https://endpoint.example.org/<yourBucketName>
+
 [Logging:LogLevel]
 Default = Information
 Microsoft.AspNetCore = Warning