notfire/Iceshrimp.NET
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[backend/services] Add object storage service (ISH-10)

Browse source
This commit is contained in:
Laura Hausmann 2024-02-09 17:28:02 +01:00
parent 0a2b8f53eb
commit 66c6f5be6f
No known key found for this signature in database
GPG key ID: D044E84C5BE01605
8 changed files with 195 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
Iceshrimp.Backend/Core/Configuration/Config.cs
View file

@ -4,10 +4,11 @@ using Iceshrimp.Backend.Core.Middleware;
namespace Iceshrimp.Backend.Core.Configuration;
public sealed class Config {
public required InstanceSection Instance { get; init; }
public required DatabaseSection Database { get; init; }
public required RedisSection Redis { get; init; }
public required InstanceSection Instance { get; init; } = new();
public required DatabaseSection Database { get; init; } = new();
public required RedisSection Redis { get; init; } = new();
public required SecuritySection Security { get; init; } = new();
public required StorageSection Storage { get; init; } = new();
public sealed class InstanceSection {
public readonly string Version;
@ -30,11 +31,11 @@ public sealed class Config {
public string UserAgent => $"Iceshrimp.NET/{Version} (https://{WebDomain})";
public required int ListenPort { get; init; } = 3000;
public required string ListenHost { get; init; } = "localhost";
public required string WebDomain { get; init; }
public required string AccountDomain { get; init; }
public required int CharacterLimit { get; init; } = 8192;
public int ListenPort { get; init; } = 3000;
public string ListenHost { get; init; } = "localhost";
public string WebDomain { get; init; } = null!;
public string AccountDomain { get; init; } = null!;
public int CharacterLimit { get; init; } = 8192;
}
public sealed class SecuritySection {
@ -47,16 +48,16 @@ public sealed class Config {
}
public sealed class DatabaseSection {
public required string Host { get; init; } = "localhost";
public required int Port { get; init; } = 5432;
public required string Database { get; init; }
public required string Username { get; init; }
public string Host { get; init; } = "localhost";
public int Port { get; init; } = 5432;
public string Database { get; init; } = null!;
public string Username { get; init; } = null!;
public string? Password { get; init; }
}
public sealed class RedisSection {
public required string Host { get; init; } = "localhost";
public required int Port { get; init; } = 6379;
public string Host { get; init; } = "localhost";
public int Port { get; init; } = 6379;
public string? Prefix { get; init; }
public string? Username { get; init; }
public string? Password { get; init; }
@ -64,4 +65,24 @@ public sealed class Config {
//TODO: TLS settings
}
public sealed class StorageSection {
public Enums.FileStorage Mode { get; init; } = Enums.FileStorage.Local;
public LocalStorageSection? Local { get; init; }
public ObjectStorageSection? ObjectStorage { get; init; }
}
public sealed class LocalStorageSection {
public string? Path { get; init; }
}
public sealed class ObjectStorageSection {
public string? Endpoint { get; init; }
public string? Region { get; init; }
public string? AccessKey { get; init; }
public string? SecretKey { get; init; }
public string? Bucket { get; init; }
public string? Prefix { get; init; }
public string? AccessUrl { get; init; }
}
}

5
Iceshrimp.Backend/Core/Configuration/Enums.cs
View file

@ -6,6 +6,11 @@ public static class Enums {
AllowList = 1
}
public enum FileStorage {
Local = 0,
ObjectStorage = 1
}
public enum ItemVisibility {
Hide = 0,
Registered = 1,

5
Iceshrimp.Backend/Core/Extensions/ServiceExtensions.cs
View file

@ -46,6 +46,7 @@ public static class ServiceExtensions {
.AddSingleton<HttpClient>()
.AddSingleton<HttpRequestService>()
.AddSingleton<QueueService>()
.AddSingleton<ObjectStorageService>()
.AddSingleton<ErrorHandlerMiddleware>()
.AddSingleton<RequestBufferingMiddleware>()
.AddSingleton<AuthorizationMiddleware>()
@ -63,6 +64,10 @@ public static class ServiceExtensions {
services.Configure<Config.InstanceSection>(configuration.GetSection("Instance"));
services.Configure<Config.SecuritySection>(configuration.GetSection("Security"));
services.Configure<Config.DatabaseSection>(configuration.GetSection("Database"));
services.Configure<Config.RedisSection>(configuration.GetSection("Redis"));
services.Configure<Config.StorageSection>(configuration.GetSection("Storage"));
services.Configure<Config.LocalStorageSection>(configuration.GetSection("Storage:Local"));
services.Configure<Config.ObjectStorageSection>(configuration.GetSection("Storage:ObjectStorage"));
}
public static void AddDatabaseContext(this IServiceCollection services, IConfiguration configuration) {

50
Iceshrimp.Backend/Core/Extensions/WebApplicationExtensions.cs
View file

@ -1,6 +1,7 @@
using Iceshrimp.Backend.Core.Configuration;
using Iceshrimp.Backend.Core.Database;
using Iceshrimp.Backend.Core.Middleware;
using Iceshrimp.Backend.Core.Services;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Caching.Distributed;
@ -33,10 +34,13 @@ public static class WebApplicationExtensions {
return app;
}
public static Config.InstanceSection Initialize(this WebApplication app, string[] args) {
public static async Task<Config.InstanceSection> Initialize(this WebApplication app, string[] args) {
var instanceConfig = app.Configuration.GetSection("Instance").Get<Config.InstanceSection>() ??
throw new Exception("Failed to read Instance config section");
var storageConfig = app.Configuration.GetSection("Storage").Get<Config.StorageSection>() ??
throw new Exception("Failed to read Storage config section");
app.Logger.LogInformation("Iceshrimp.NET v{version} ({domain})", instanceConfig.Version,
instanceConfig.AccountDomain);
@ -46,44 +50,74 @@ public static class WebApplicationExtensions {
app.Logger.LogWarning("If this is not a local development instance, please set the environment to Production.");
}
var provider = app.Services.CreateScope();
var context = provider.ServiceProvider.GetService<DatabaseContext>();
var provider = app.Services.CreateScope().ServiceProvider;
var context = provider.GetService<DatabaseContext>();
if (context == null) {
app.Logger.LogCritical("Failed to initialize database context");
Environment.Exit(1);
}
app.Logger.LogInformation("Verifying database connection...");
if (!context.Database.CanConnect()) {
if (!await context.Database.CanConnectAsync()) {
app.Logger.LogCritical("Failed to connect to database");
Environment.Exit(1);
}
if (args.Contains("--migrate") || args.Contains("--migrate-and-start")) {
app.Logger.LogInformation("Running migrations...");
context.Database.Migrate();
await context.Database.MigrateAsync();
if (args.Contains("--migrate")) Environment.Exit(0);
}
else if (context.Database.GetPendingMigrations().Any()) {
else if ((await context.Database.GetPendingMigrationsAsync()).Any()) {
app.Logger.LogCritical("Database has pending migrations, please restart with --migrate or --migrate-and-start");
Environment.Exit(1);
}
app.Logger.LogInformation("Verifying redis connection...");
var cache = provider.ServiceProvider.GetService<IDistributedCache>();
var cache = provider.GetService<IDistributedCache>();
if (cache == null) {
app.Logger.LogCritical("Failed to initialize redis cache");
Environment.Exit(1);
}
try {
cache.Get("test");
await cache.GetAsync("test");
}
catch {
app.Logger.LogCritical("Failed to connect to redis");
Environment.Exit(1);
}
if (storageConfig.Mode == Enums.FileStorage.Local) {
if (string.IsNullOrWhiteSpace(storageConfig.Local?.Path) ||
!Directory.Exists(storageConfig.Local.Path)) {
app.Logger.LogCritical("Local storage path does not exist");
Environment.Exit(1);
}
else {
try {
var path = Path.Combine(storageConfig.Local.Path, Path.GetRandomFileName());
await using var fs = File.Create(path, 1, FileOptions.DeleteOnClose);
}
catch {
app.Logger.LogCritical("Local storage path is not accessible or not writable");
Environment.Exit(1);
}
}
}
else if (storageConfig.Mode == Enums.FileStorage.ObjectStorage) {
app.Logger.LogInformation("Verifying object storage configuration...");
var svc = provider.GetRequiredService<ObjectStorageService>();
try {
await svc.VerifyCredentialsAsync();
}
catch (Exception e) {
app.Logger.LogCritical("Failed to initialize object storage: {message}", e.Message);
Environment.Exit(1);
}
}
app.Logger.LogInformation("Initializing application, please wait...");
return instanceConfig;

81
Iceshrimp.Backend/Core/Services/ObjectStorageService.cs Normal file
View file

@ -0,0 +1,81 @@
using System.Text;
using Amazon;
using Amazon.S3;
using Carbon.Storage;
using Iceshrimp.Backend.Core.Configuration;
using Iceshrimp.Backend.Core.Helpers;
using Microsoft.Extensions.Options;
namespace Iceshrimp.Backend.Core.Services;
public class ObjectStorageService(IOptions<Config.StorageSection> config) {
private readonly string _accessUrl = config.Value.ObjectStorage?.AccessUrl ??
throw new Exception("Invalid object storage access url");
private readonly S3Bucket _bucket = GetBucket(config);
private readonly string? _prefix = config.Value.ObjectStorage?.Prefix?.Trim('/');
private static S3Bucket GetBucket(IOptions<Config.StorageSection> config) {
var s3Config = config.Value.ObjectStorage ?? throw new Exception("Invalid object storage configuration");
var region = s3Config.Region ?? throw new Exception("Invalid object storage region");
var endpoint = s3Config.Endpoint ?? throw new Exception("Invalid object storage endpoint");
var accessKey = s3Config.AccessKey ?? throw new Exception("Invalid object storage access key");
var secretKey = s3Config.SecretKey ?? throw new Exception("Invalid object storage secret key");
var bucket = s3Config.Bucket ?? throw new Exception("Invalid object storage bucket");
var client = new S3Client(new AwsRegion(region), endpoint, new AwsCredential(accessKey, secretKey));
return new S3Bucket(bucket, client);
}
public async Task VerifyCredentialsAsync() {
const string filename = ".iceshrimp-test";
var content = CryptographyHelpers.GenerateRandomString(16);
await UploadFileAsync(filename, Encoding.UTF8.GetBytes(content));
var httpClient = new HttpClient();
string result;
try {
result = await httpClient.GetStringAsync(GetFilePublicUrl(filename));
}
catch (Exception e) {
throw new Exception($"Failed to verify access url: {e.Message}");
}
if (result == content)
return;
throw new Exception("Failed to verify access url (content mismatch)");
}
public async Task UploadFileAsync(string filename, byte[] data) {
await _bucket.PutAsync(new Blob(GetFilenameWithPrefix(filename), data));
}
public async Task<string> UploadFileAsync(byte[] data) {
var filename = Guid.NewGuid().ToString().ToLowerInvariant();
await _bucket.PutAsync(new Blob(GetFilenameWithPrefix(filename), data));
return filename;
}
public Uri GetFilePublicUrl(string filename) {
var baseUri = new Uri(_accessUrl);
return new Uri(baseUri, GetFilenameWithPrefix(filename));
}
public async ValueTask<Stream?> GetFileAsync(string filename) {
try {
var res = await _bucket.GetAsync(GetFilenameWithPrefix(filename));
return await res.OpenAsync();
}
catch {
return null;
}
}
private string GetFilenameWithPrefix(string filename) {
return !string.IsNullOrWhiteSpace(_prefix) ? _prefix + "/" + filename : filename;
}
}

5
Iceshrimp.Backend/Iceshrimp.Backend.csproj
View file

@ -14,11 +14,12 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Amazon.S3" Version="0.31.2"/>
<PackageReference Include="AngleSharp" Version="1.1.0"/>
<PackageReference Include="Asp.Versioning.Http" Version="8.0.0"/>
<PackageReference Include="cuid.net" Version="5.0.2"/>
<PackageReference Include="dotNetRdf.Core" Version="3.2.1-dev"/>
<PackageReference Include="EntityFrameworkCore.Projectables" Version="3.0.4" />
<PackageReference Include="EntityFrameworkCore.Projectables" Version="3.0.4"/>
<PackageReference Include="Isopoh.Cryptography.Argon2" Version="2.0.0"/>
<PackageReference Include="Microsoft.AspNetCore.DataProtection.EntityFrameworkCore" Version="8.0.1"/>
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.0"/>
@ -48,7 +49,7 @@
</ItemGroup>
<ItemGroup>
<None Remove="migrate.sql" />
<None Remove="migrate.sql"/>
</ItemGroup>
</Project>

2
Iceshrimp.Backend/Startup.cs
View file

@ -37,7 +37,7 @@ builder.Services.AddServices();
builder.Services.ConfigureServices(builder.Configuration);
var app = builder.Build();
var config = app.Initialize(args);
var config = await app.Initialize(args);
// This determines the order of middleware execution in the request pipeline
app.UseRouting();

18
Iceshrimp.Backend/configuration.ini
View file

@ -50,6 +50,24 @@ Port = 6379
;;Password =
;;Database = 0
[Storage]
;; Where to store media attachments
;; Options: [Local, ObjectStorage]
Mode = Local
[Storage:Local]
;; Path where media is stored at. Must be writable for the service user.
Path = /path/to/media/location
;;[Storage:ObjectStorage]
;;Endpoint = endpoint.example.org
;;Region = us-east-1
;;KeyId = <yourAccessKey>
;;SecretKey = <yourSecretKey>
;;Bucket = <yourBucketName>
;;Prefix =
;;AccessUrl = https://endpoint.example.org/<yourBucketName>
[Logging:LogLevel]
Default = Information
Microsoft.AspNetCore = Warning