[backend/core] Improve meta store (ISH-133)

Iceshrimp.Backend/Controllers/Mastodon/AuthController.cs

@@ -31,7 +31,7 @@ public class AuthController(DatabaseContext db, MetaService meta) : ControllerBa
 
 		var res = new AuthSchemas.VerifyAppCredentialsResponse
 		{
-			App = token.App, VapidKey = await meta.GetVapidPublicKey()
+			App = token.App, VapidKey = await meta.Get(MetaEntity.VapidPublicKey)
 		};
 
 		return Ok(res);
@@ -79,7 +79,10 @@ public class AuthController(DatabaseContext db, MetaService meta) : ControllerBa
 		await db.AddAsync(app);
 		await db.SaveChangesAsync();
 
-		var res = new AuthSchemas.RegisterAppResponse { App = app, VapidKey = await meta.GetVapidPublicKey() };
+		var res = new AuthSchemas.RegisterAppResponse
+		{
+			App = app, VapidKey = await meta.Get(MetaEntity.VapidPublicKey)
+		};
 
 		return Ok(res);
 	}

Iceshrimp.Backend/Controllers/Mastodon/PushController.cs

@@ -153,7 +153,7 @@ public class PushController(DatabaseContext db, MetaService meta) : ControllerBa
 		{
 			Id        = sub.Id,
 			Endpoint  = sub.Endpoint,
-			ServerKey = await meta.GetVapidPublicKey() ?? throw new Exception("Failed to fetch VAPID key"),
+			ServerKey = await meta.Get(MetaEntity.VapidPublicKey),
 			Policy    = GetPolicyString(sub.Policy),
 			Alerts = new PushSchemas.Alerts
 			{

Iceshrimp.Backend/Core/Database/DatabaseContext.cs

@@ -85,7 +85,7 @@ public class DatabaseContext(DbContextOptions<DatabaseContext> options)
 	public virtual DbSet<AllowedInstance>      AllowedInstances      { get; init; } = null!;
 	public virtual DbSet<BlockedInstance>      BlockedInstances      { get; init; } = null!;
 	public virtual DbSet<DataProtectionKey>    DataProtectionKeys    { get; init; } = null!;
-	public virtual DbSet<MetaStore>            MetaStore             { get; init; } = null!;
+	public virtual DbSet<MetaStoreEntry>       MetaStore             { get; init; } = null!;
 
 	public static NpgsqlDataSource GetDataSource(Config.DatabaseSection? config)
 	{

Iceshrimp.Backend/Core/Database/Migrations/20240316115734_MakeMetaStoreValueNullable.cs

@@ -0,0 +1,36 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Iceshrimp.Backend.Core.Database.Migrations
+{
+    /// <inheritdoc />
+    public partial class MakeMetaStoreValueNullable : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AlterColumn<string>(
+                name: "value",
+                table: "meta_store",
+                type: "text",
+                nullable: true,
+                oldClrType: typeof(string),
+                oldType: "text");
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AlterColumn<string>(
+                name: "value",
+                table: "meta_store",
+                type: "text",
+                nullable: false,
+                defaultValue: "",
+                oldClrType: typeof(string),
+                oldType: "text",
+                oldNullable: true);
+        }
+    }
+}

Iceshrimp.Backend/Core/Database/Migrations/DatabaseContextModelSnapshot.cs

@@ -2060,7 +2060,7 @@ namespace Iceshrimp.Backend.Core.Database.Migrations
                     b.ToTable("meta");
                 });
 
-            modelBuilder.Entity("Iceshrimp.Backend.Core.Database.Tables.MetaStore", b =>
+            modelBuilder.Entity("Iceshrimp.Backend.Core.Database.Tables.MetaStoreEntry", b =>
                 {
                     b.Property<string>("Key")
                         .HasMaxLength(128)
@@ -2068,7 +2068,6 @@ namespace Iceshrimp.Backend.Core.Database.Migrations
                         .HasColumnName("key");
 
                     b.Property<string>("Value")
-                        .IsRequired()
                         .HasColumnType("text")
                         .HasColumnName("value");
 

Iceshrimp.Backend/Core/Database/Tables/MetaStoreEntry.cs

@@ -6,12 +6,12 @@ namespace Iceshrimp.Backend.Core.Database.Tables;
 
 [Table("meta_store")]
 [Index("Key")]
-public class MetaStore
+public class MetaStoreEntry
 {
 	[Key]
 	[Column("key")]
 	[StringLength(128)]
 	public string Key { get; set; } = null!;
 
-	[Column("value")] public string Value { get; set; } = null!;
+	[Column("value")] public string? Value { get; set; } = null!;
 }

Iceshrimp.Backend/Core/Extensions/DistributedCacheExtensions.cs

@@ -8,18 +8,20 @@ public static class DistributedCacheExtensions
 {
 	//TODO: named caches, CacheService? (that optionally uses StackExchange.Redis directly)?
 	//TODO: thread-safe locks to prevent fetching data more than once
-	//TODO: sliding window ttl?
-	//TODO: renew option on GetAsync and FetchAsync
 	//TODO: check that this actually works for complex types (sigh)
 
 	private static readonly JsonSerializerOptions Options =
 		new(JsonSerializerOptions.Default) { ReferenceHandler = ReferenceHandler.Preserve };
 
-	public static async Task<T?> GetAsync<T>(this IDistributedCache cache, string key) where T : class?
+	public static async Task<T?> GetAsync<T>(this IDistributedCache cache, string key, bool renew = false)
+		where T : class?
 	{
 		var buffer = await cache.GetAsync(key);
 		if (buffer == null || buffer.Length == 0) return null;
 
+		if (renew)
+			await cache.RefreshAsync(key);
+
 		var stream = new MemoryStream(buffer);
 		try
 		{
@@ -32,11 +34,15 @@ public static class DistributedCacheExtensions
 		}
 	}
 
-	public static async Task<T?> GetAsyncValue<T>(this IDistributedCache cache, string key) where T : struct
+	public static async Task<T?> GetAsyncValue<T>(this IDistributedCache cache, string key, bool renew = false)
+		where T : struct
 	{
 		var buffer = await cache.GetAsync(key);
 		if (buffer == null || buffer.Length == 0) return null;
 
+		if (renew)
+			await cache.RefreshAsync(key);
+
 		var stream = new MemoryStream(buffer);
 		try
 		{
@@ -50,49 +56,53 @@ public static class DistributedCacheExtensions
 	}
 
 	public static async Task<T> FetchAsync<T>(
-		this IDistributedCache cache, string key, TimeSpan ttl, Func<Task<T>> fetcher
+		this IDistributedCache cache, string key, TimeSpan ttl, Func<Task<T>> fetcher, bool renew = false
 	) where T : class?
 	{
-		var hit = await cache.GetAsync<T>(key);
+		var hit = await cache.GetAsync<T>(key, renew);
 		if (hit != null) return hit;
 
 		var fetched = await fetcher();
-		await cache.SetAsync(key, fetched, ttl);
+		await cache.SetAsync(key, fetched, ttl, renew);
 		return fetched;
 	}
 
 	public static async Task<T> FetchAsync<T>(
-		this IDistributedCache cache, string key, TimeSpan ttl, Func<T> fetcher
+		this IDistributedCache cache, string key, TimeSpan ttl, Func<T> fetcher, bool renew = false
 	) where T : class
 	{
-		return await FetchAsync(cache, key, ttl, () => Task.FromResult(fetcher()));
+		return await FetchAsync(cache, key, ttl, () => Task.FromResult(fetcher()), renew);
 	}
 
 	public static async Task<T> FetchAsyncValue<T>(
-		this IDistributedCache cache, string key, TimeSpan ttl, Func<Task<T>> fetcher
+		this IDistributedCache cache, string key, TimeSpan ttl, Func<Task<T>> fetcher, bool renew = false
 	) where T : struct
 	{
-		var hit = await cache.GetAsyncValue<T>(key);
+		var hit = await cache.GetAsyncValue<T>(key, renew);
 		if (hit.HasValue) return hit.Value;
 
 		var fetched = await fetcher();
-		await cache.SetAsync(key, fetched, ttl);
+		await cache.SetAsync(key, fetched, ttl, renew);
 		return fetched;
 	}
 
 	public static async Task<T> FetchAsyncValue<T>(
-		this IDistributedCache cache, string key, TimeSpan ttl, Func<T> fetcher
+		this IDistributedCache cache, string key, TimeSpan ttl, Func<T> fetcher, bool renew = false
 	) where T : struct
 	{
-		return await FetchAsyncValue(cache, key, ttl, () => Task.FromResult(fetcher()));
+		return await FetchAsyncValue(cache, key, ttl, () => Task.FromResult(fetcher()), renew);
 	}
 
-	public static async Task SetAsync<T>(this IDistributedCache cache, string key, T data, TimeSpan ttl)
+	public static async Task SetAsync<T>(
+		this IDistributedCache cache, string key, T data, TimeSpan ttl, bool sliding = false
+	)
 	{
 		using var stream = new MemoryStream();
 		await JsonSerializer.SerializeAsync(stream, data, Options);
 		stream.Position = 0;
-		var options = new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = ttl };
+		var options = sliding
+			? new DistributedCacheEntryOptions { SlidingExpiration               = ttl }
+			: new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = ttl };
 		await cache.SetAsync(key, stream.ToArray(), options);
 	}
 }

Iceshrimp.Backend/Core/Extensions/WebApplicationExtensions.cs

@@ -1,7 +1,6 @@
 using System.Runtime.InteropServices;
 using Iceshrimp.Backend.Core.Configuration;
 using Iceshrimp.Backend.Core.Database;
-using Iceshrimp.Backend.Core.Database.Tables;
 using Iceshrimp.Backend.Core.Middleware;
 using Iceshrimp.Backend.Core.Services;
 using Microsoft.EntityFrameworkCore;
@@ -168,26 +167,15 @@ public static class WebApplicationExtensions
 		}
 
 		app.Logger.LogInformation("Initializing VAPID keys...");
-		var vapidPublicKey  = await db.MetaStore.FirstOrDefaultAsync(p => p.Key == "vapid_public_key");
+		var meta = provider.GetRequiredService<MetaService>();
-		var vapidPrivateKey = await db.MetaStore.FirstOrDefaultAsync(p => p.Key == "vapid_private_key");
+		await meta.EnsureSet([MetaEntity.VapidPublicKey, MetaEntity.VapidPrivateKey], () =>
-		if (vapidPrivateKey == null || vapidPublicKey == null)
 		{
 			var keypair = VapidHelper.GenerateVapidKeys();
-			vapidPrivateKey = new MetaStore
+			return [keypair.PublicKey, keypair.PrivateKey];
-			{
+		});
-				Key = "vapid_private_key",
-				Value = keypair.PrivateKey
-			};
 		
-			vapidPublicKey = new MetaStore
+		app.Logger.LogInformation("Warming up meta cache...");
-			{
+		await meta.WarmupCache();
-				Key   = "vapid_public_key",
-				Value = keypair.PublicKey
-			};
-
-			db.AddRange(vapidPublicKey, vapidPrivateKey);
-			await db.SaveChangesAsync();
-		}
 
 		app.Logger.LogInformation("Initializing application, please wait...");
 

Iceshrimp.Backend/Core/Services/MetaService.cs

@@ -1,4 +1,6 @@
+using EntityFramework.Exceptions.Common;
 using Iceshrimp.Backend.Core.Database;
+using Iceshrimp.Backend.Core.Database.Tables;
 using Iceshrimp.Backend.Core.Extensions;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Caching.Distributed;
@@ -7,23 +9,126 @@ namespace Iceshrimp.Backend.Core.Services;
 
 public class MetaService(IServiceScopeFactory scopeFactory, IDistributedCache cache)
 {
-	public async Task<string?> GetVapidPrivateKey() =>
+	public async Task<T> Get<T>(Meta<T> meta) where T : class? =>
-		await cache.FetchAsync("cache:meta:vapidPrivateKey", TimeSpan.FromDays(30),
+		await cache.FetchAsync($"cache:meta:{meta.Key}", meta.Ttl, async () => await Fetch(meta));
-		                       async () => await Fetch("vapid_private_key"));
 
-	public async Task<string?> GetVapidPublicKey() =>
+	public async Task<T> GetValue<T>(Meta<T> meta) where T : struct =>
-		await cache.FetchAsync("cache:meta:vapidPublicKey", TimeSpan.FromDays(30),
+		await cache.FetchAsyncValue($"cache:meta:{meta.Key}", meta.Ttl, async () => await Fetch(meta));
-		                       async () => await Fetch("vapid_public_key"));
 
-	private async Task<string?> Fetch(string key)
+	public async Task EnsureSet<T>(Meta<T> meta, T value) => await EnsureSet(meta, () => value);
+
+	public async Task EnsureSet<T>(Meta<T> meta, Func<T> value)
 	{
-		var db = scopeFactory.CreateScope().ServiceProvider.GetRequiredService<DatabaseContext>();
+		if (await Fetch(meta.Key) != null) return;
-		return await db.MetaStore.Where(p => p.Key == key).Select(p => p.Value).FirstOrDefaultAsync();
+		await Set(meta, value());
 	}
 
-	//TODO
+	public async Task EnsureSet<T>(Meta<T> meta, Func<Task<T>> value)
-	// private interface IMeta
+	{
-	// {
+		if (await Fetch(meta.Key) != null) return;
-	// 	string Key { get; }
+		await Set(meta, await value());
-	// }
+	}
+
+	public async Task EnsureSet<T>(IReadOnlyList<Meta<T>> metas, Func<List<T>> values)
+	{
+		if (await GetDbContext().MetaStore.CountAsync(p => metas.Select(m => m.Key).Contains(p.Key)) == metas.Count)
+			return;
+
+		var resolvedValues = values();
+		if (resolvedValues.Count != metas.Count)
+			throw new Exception("Metas count doesn't match values count");
+
+		for (var i = 0; i < metas.Count; i++)
+			await Set(metas[i], resolvedValues[i]);
+	}
+
+	public async Task Set<T>(Meta<T> meta, T value) => await Set(meta.Key, meta.ConvertSet(value), meta.Ttl);
+
+	public async Task WarmupCache()
+	{
+		await Get(MetaEntity.VapidPrivateKey);
+		await Get(MetaEntity.VapidPublicKey);
+		await Get(MetaEntity.InstanceName);
+		await Get(MetaEntity.InstanceDescription);
+	}
+
+	private async Task<T> Fetch<T>(Meta<T> meta) => meta.GetConverter(await Fetch(meta.Key));
+
+	private async Task<string?> Fetch(string key) =>
+		await GetDbContext().MetaStore.Where(p => p.Key == key).Select(p => p.Value).FirstOrDefaultAsync();
+
+	private async Task Set(string key, string? value, TimeSpan ttl)
+	{
+		var db     = GetDbContext();
+		var entity = await db.MetaStore.FirstOrDefaultAsync(p => p.Key == key);
+		if (entity != null)
+		{
+			entity.Value = value;
+			await db.SaveChangesAsync();
+		}
+		else
+		{
+			entity = new MetaStoreEntry { Key = key, Value = value };
+			db.Add(entity);
+			try
+			{
+				await db.SaveChangesAsync();
+			}
+			catch (UniqueConstraintException)
+			{
+				db.Remove(entity);
+				entity = await db.MetaStore.FirstOrDefaultAsync(p => p.Key == key) ??
+				         throw new Exception("Failed to fetch entity after UniqueConstraintException");
+				entity.Value = value;
+				await db.SaveChangesAsync();
+			}
+		}
+
+		await cache.SetAsync($"cache:meta:{key}", value, ttl, true);
+	}
+
+	private DatabaseContext GetDbContext() =>
+		scopeFactory.CreateScope().ServiceProvider.GetRequiredService<DatabaseContext>();
 }
+
+public static class MetaEntity
+{
+	public static readonly StringMeta         VapidPrivateKey     = new("vapid_private_key");
+	public static readonly StringMeta         VapidPublicKey      = new("vapid_public_key");
+	public static readonly NullableStringMeta InstanceName        = new("instance_name");
+	public static readonly NullableStringMeta InstanceDescription = new("instance_description");
+}
+
+public class Meta<T>(string key, TimeSpan? ttl, Func<string?, T> getConverter, Func<T, string?> setConverter)
+{
+	public string           Key          => key;
+	public TimeSpan         Ttl          => ttl ?? TimeSpan.FromDays(30);
+	public Func<string?, T> GetConverter => getConverter;
+	public Func<T, string?> ConvertSet   => setConverter;
+}
+
+public class StringMeta(string key, TimeSpan? ttl = null) : NonNullableMeta<string>(key, ttl, val => val, val => val);
+
+public class NullableStringMeta(string key, TimeSpan? ttl = null) : Meta<string?>(key, ttl, val => val, val => val);
+
+public class IntMeta(string key, TimeSpan? ttl = null)
+	: NonNullableValueMeta<int>(key, ttl,
+	                            int.Parse,
+	                            val => val.ToString());
+
+public class NullableIntMeta(string key, TimeSpan? ttl = null)
+	: Meta<int?>(key, ttl, val => int.TryParse(val, out var result) ? result : null, val => val?.ToString());
+
+public class NonNullableMeta<T>(string key, TimeSpan? ttl, Func<string, T> getConverter, Func<T, string> setConverter)
+	: Meta<T>(key, ttl,
+	          val => getConverter(val ?? throw new Exception($"Fetched meta value {key} was null")),
+	          setConverter) where T : class;
+
+public class NonNullableValueMeta<T>(
+	string key,
+	TimeSpan? ttl,
+	Func<string, T> getConverter,
+	Func<T, string> setConverter
+) : Meta<T>(key, ttl,
+            val => getConverter(val ?? throw new Exception($"Fetched meta value {key} was null")),
+            setConverter) where T : struct;

Iceshrimp.Backend/Core/Services/PushService.cs

@@ -22,7 +22,7 @@ public class PushService(
 	IServiceScopeFactory scopeFactory,
 	HttpClient httpClient,
 	IOptions<Config.InstanceSection> config,
-	MetaService metaService
+	MetaService meta
 ) : BackgroundService
 {
 	protected override Task ExecuteAsync(CancellationToken stoppingToken)
@@ -85,11 +85,8 @@ public class PushService(
 				if (body.Length > 137)
 					body = body.Truncate(137).TrimEnd() + "...";
 
-				var priv = await metaService.GetVapidPrivateKey();
+				var priv = await meta.Get(MetaEntity.VapidPrivateKey);
-				var pub  = await metaService.GetVapidPublicKey();
+				var pub  = await meta.Get(MetaEntity.VapidPublicKey);
-
-				if (priv == null || pub == null)
-					throw new Exception("Failed to fetch VAPID keys");
 
 				var client = new WebPushClient(httpClient);
 				client.SetVapidDetails(new VapidDetails($"https://{config.Value.WebDomain}", pub, priv));