notfire/Iceshrimp.NET
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[backend/core] Improve meta store (ISH-133)

Browse source
This commit is contained in:
Laura Hausmann 2024-03-16 12:48:49 +01:00
parent e9a67ef6a0
commit be27893c32
No known key found for this signature in database
GPG key ID: D044E84C5BE01605
11 changed files with 6102 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
Iceshrimp.Backend/Controllers/Mastodon/AuthController.cs
View file

@ -31,7 +31,7 @@ public class AuthController(DatabaseContext db, MetaService meta) : ControllerBa
var res = new AuthSchemas.VerifyAppCredentialsResponse
{
App = token.App, VapidKey = await meta.GetVapidPublicKey()
App = token.App, VapidKey = await meta.Get(MetaEntity.VapidPublicKey)
};
return Ok(res);
@ -79,7 +79,10 @@ public class AuthController(DatabaseContext db, MetaService meta) : ControllerBa
await db.AddAsync(app);
await db.SaveChangesAsync();
var res = new AuthSchemas.RegisterAppResponse { App = app, VapidKey = await meta.GetVapidPublicKey() };
var res = new AuthSchemas.RegisterAppResponse
{
App = app, VapidKey = await meta.Get(MetaEntity.VapidPublicKey)
};
return Ok(res);
}

2
Iceshrimp.Backend/Controllers/Mastodon/PushController.cs
View file

@ -153,7 +153,7 @@ public class PushController(DatabaseContext db, MetaService meta) : ControllerBa
{
Id = sub.Id,
Endpoint = sub.Endpoint,
ServerKey = await meta.GetVapidPublicKey() ?? throw new Exception("Failed to fetch VAPID key"),
ServerKey = await meta.Get(MetaEntity.VapidPublicKey),
Policy = GetPolicyString(sub.Policy),
Alerts = new PushSchemas.Alerts
{

2
Iceshrimp.Backend/Core/Database/DatabaseContext.cs
View file

@ -85,7 +85,7 @@ public class DatabaseContext(DbContextOptions<DatabaseContext> options)
public virtual DbSet<AllowedInstance> AllowedInstances { get; init; } = null!;
public virtual DbSet<BlockedInstance> BlockedInstances { get; init; } = null!;
public virtual DbSet<DataProtectionKey> DataProtectionKeys { get; init; } = null!;
public virtual DbSet<MetaStore> MetaStore { get; init; } = null!;
public virtual DbSet<MetaStoreEntry> MetaStore { get; init; } = null!;
public static NpgsqlDataSource GetDataSource(Config.DatabaseSection? config)
{

5900
Iceshrimp.Backend/Core/Database/Migrations/20240316115734_MakeMetaStoreValueNullable.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

36
Iceshrimp.Backend/Core/Database/Migrations/20240316115734_MakeMetaStoreValueNullable.cs Normal file
View file

@ -0,0 +1,36 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Iceshrimp.Backend.Core.Database.Migrations
{
/// <inheritdoc />
public partial class MakeMetaStoreValueNullable : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AlterColumn<string>(
name: "value",
table: "meta_store",
type: "text",
nullable: true,
oldClrType: typeof(string),
oldType: "text");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.AlterColumn<string>(
name: "value",
table: "meta_store",
type: "text",
nullable: false,
defaultValue: "",
oldClrType: typeof(string),
oldType: "text",
oldNullable: true);
}
}
}

3
Iceshrimp.Backend/Core/Database/Migrations/DatabaseContextModelSnapshot.cs
View file

@ -2060,7 +2060,7 @@ namespace Iceshrimp.Backend.Core.Database.Migrations
b.ToTable("meta");
});
modelBuilder.Entity("Iceshrimp.Backend.Core.Database.Tables.MetaStore", b =>
modelBuilder.Entity("Iceshrimp.Backend.Core.Database.Tables.MetaStoreEntry", b =>
{
b.Property<string>("Key")
.HasMaxLength(128)
@ -2068,7 +2068,6 @@ namespace Iceshrimp.Backend.Core.Database.Migrations
.HasColumnName("key");
b.Property<string>("Value")
.IsRequired()
.HasColumnType("text")
.HasColumnName("value");

4
Iceshrimp.Backend/Core/Database/Tables/MetaStore.cs → Iceshrimp.Backend/Core/Database/Tables/MetaStoreEntry.cs
View file

@ -6,12 +6,12 @@ namespace Iceshrimp.Backend.Core.Database.Tables;
[Table("meta_store")]
[Index("Key")]
public class MetaStore
public class MetaStoreEntry
{
[Key]
[Column("key")]
[StringLength(128)]
public string Key { get; set; } = null!;
[Column("value")] public string Value { get; set; } = null!;
[Column("value")] public string? Value { get; set; } = null!;
}

42
Iceshrimp.Backend/Core/Extensions/DistributedCacheExtensions.cs
View file

@ -8,18 +8,20 @@ public static class DistributedCacheExtensions
{
//TODO: named caches, CacheService? (that optionally uses StackExchange.Redis directly)?
//TODO: thread-safe locks to prevent fetching data more than once
//TODO: sliding window ttl?
//TODO: renew option on GetAsync and FetchAsync
//TODO: check that this actually works for complex types (sigh)
private static readonly JsonSerializerOptions Options =
new(JsonSerializerOptions.Default) { ReferenceHandler = ReferenceHandler.Preserve };
public static async Task<T?> GetAsync<T>(this IDistributedCache cache, string key) where T : class?
public static async Task<T?> GetAsync<T>(this IDistributedCache cache, string key, bool renew = false)
where T : class?
{
var buffer = await cache.GetAsync(key);
if (buffer == null || buffer.Length == 0) return null;
if (renew)
await cache.RefreshAsync(key);
var stream = new MemoryStream(buffer);
try
{
@ -32,11 +34,15 @@ public static class DistributedCacheExtensions
}
}
public static async Task<T?> GetAsyncValue<T>(this IDistributedCache cache, string key) where T : struct
public static async Task<T?> GetAsyncValue<T>(this IDistributedCache cache, string key, bool renew = false)
where T : struct
{
var buffer = await cache.GetAsync(key);
if (buffer == null || buffer.Length == 0) return null;
if (renew)
await cache.RefreshAsync(key);
var stream = new MemoryStream(buffer);
try
{
@ -50,49 +56,53 @@ public static class DistributedCacheExtensions
}
public static async Task<T> FetchAsync<T>(
this IDistributedCache cache, string key, TimeSpan ttl, Func<Task<T>> fetcher
this IDistributedCache cache, string key, TimeSpan ttl, Func<Task<T>> fetcher, bool renew = false
) where T : class?
{
var hit = await cache.GetAsync<T>(key);
var hit = await cache.GetAsync<T>(key, renew);
if (hit != null) return hit;
var fetched = await fetcher();
await cache.SetAsync(key, fetched, ttl);
await cache.SetAsync(key, fetched, ttl, renew);
return fetched;
}
public static async Task<T> FetchAsync<T>(
this IDistributedCache cache, string key, TimeSpan ttl, Func<T> fetcher
this IDistributedCache cache, string key, TimeSpan ttl, Func<T> fetcher, bool renew = false
) where T : class
{
return await FetchAsync(cache, key, ttl, () => Task.FromResult(fetcher()));
return await FetchAsync(cache, key, ttl, () => Task.FromResult(fetcher()), renew);
}
public static async Task<T> FetchAsyncValue<T>(
this IDistributedCache cache, string key, TimeSpan ttl, Func<Task<T>> fetcher
this IDistributedCache cache, string key, TimeSpan ttl, Func<Task<T>> fetcher, bool renew = false
) where T : struct
{
var hit = await cache.GetAsyncValue<T>(key);
var hit = await cache.GetAsyncValue<T>(key, renew);
if (hit.HasValue) return hit.Value;
var fetched = await fetcher();
await cache.SetAsync(key, fetched, ttl);
await cache.SetAsync(key, fetched, ttl, renew);
return fetched;
}
public static async Task<T> FetchAsyncValue<T>(
this IDistributedCache cache, string key, TimeSpan ttl, Func<T> fetcher
this IDistributedCache cache, string key, TimeSpan ttl, Func<T> fetcher, bool renew = false
) where T : struct
{
return await FetchAsyncValue(cache, key, ttl, () => Task.FromResult(fetcher()));
return await FetchAsyncValue(cache, key, ttl, () => Task.FromResult(fetcher()), renew);
}
public static async Task SetAsync<T>(this IDistributedCache cache, string key, T data, TimeSpan ttl)
public static async Task SetAsync<T>(
this IDistributedCache cache, string key, T data, TimeSpan ttl, bool sliding = false
)
{
using var stream = new MemoryStream();
await JsonSerializer.SerializeAsync(stream, data, Options);
stream.Position = 0;
var options = new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = ttl };
var options = sliding
? new DistributedCacheEntryOptions { SlidingExpiration = ttl }
: new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = ttl };
await cache.SetAsync(key, stream.ToArray(), options);
}
}

24
Iceshrimp.Backend/Core/Extensions/WebApplicationExtensions.cs
View file

@ -1,7 +1,6 @@
using System.Runtime.InteropServices;
using Iceshrimp.Backend.Core.Configuration;
using Iceshrimp.Backend.Core.Database;
using Iceshrimp.Backend.Core.Database.Tables;
using Iceshrimp.Backend.Core.Middleware;
using Iceshrimp.Backend.Core.Services;
using Microsoft.EntityFrameworkCore;
@ -168,26 +167,15 @@ public static class WebApplicationExtensions
}
app.Logger.LogInformation("Initializing VAPID keys...");
var vapidPublicKey = await db.MetaStore.FirstOrDefaultAsync(p => p.Key == "vapid_public_key");
var vapidPrivateKey = await db.MetaStore.FirstOrDefaultAsync(p => p.Key == "vapid_private_key");
if (vapidPrivateKey == null || vapidPublicKey == null)
var meta = provider.GetRequiredService<MetaService>();
await meta.EnsureSet([MetaEntity.VapidPublicKey, MetaEntity.VapidPrivateKey], () =>
{
var keypair = VapidHelper.GenerateVapidKeys();
vapidPrivateKey = new MetaStore
{
Key = "vapid_private_key",
Value = keypair.PrivateKey
};
return [keypair.PublicKey, keypair.PrivateKey];
});
vapidPublicKey = new MetaStore
{
Key = "vapid_public_key",
Value = keypair.PublicKey
};
db.AddRange(vapidPublicKey, vapidPrivateKey);
await db.SaveChangesAsync();
}
app.Logger.LogInformation("Warming up meta cache...");
await meta.WarmupCache();
app.Logger.LogInformation("Initializing application, please wait...");

133
Iceshrimp.Backend/Core/Services/MetaService.cs
View file

@ -1,4 +1,6 @@
using EntityFramework.Exceptions.Common;
using Iceshrimp.Backend.Core.Database;
using Iceshrimp.Backend.Core.Database.Tables;
using Iceshrimp.Backend.Core.Extensions;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Caching.Distributed;
@ -7,23 +9,126 @@ namespace Iceshrimp.Backend.Core.Services;
public class MetaService(IServiceScopeFactory scopeFactory, IDistributedCache cache)
{
public async Task<string?> GetVapidPrivateKey() =>
await cache.FetchAsync("cache:meta:vapidPrivateKey", TimeSpan.FromDays(30),
async () => await Fetch("vapid_private_key"));
public async Task<T> Get<T>(Meta<T> meta) where T : class? =>
await cache.FetchAsync($"cache:meta:{meta.Key}", meta.Ttl, async () => await Fetch(meta));
public async Task<string?> GetVapidPublicKey() =>
await cache.FetchAsync("cache:meta:vapidPublicKey", TimeSpan.FromDays(30),
async () => await Fetch("vapid_public_key"));
public async Task<T> GetValue<T>(Meta<T> meta) where T : struct =>
await cache.FetchAsyncValue($"cache:meta:{meta.Key}", meta.Ttl, async () => await Fetch(meta));
private async Task<string?> Fetch(string key)
public async Task EnsureSet<T>(Meta<T> meta, T value) => await EnsureSet(meta, () => value);
public async Task EnsureSet<T>(Meta<T> meta, Func<T> value)
{
var db = scopeFactory.CreateScope().ServiceProvider.GetRequiredService<DatabaseContext>();
return await db.MetaStore.Where(p => p.Key == key).Select(p => p.Value).FirstOrDefaultAsync();
if (await Fetch(meta.Key) != null) return;
await Set(meta, value());
}
//TODO
// private interface IMeta
// {
// string Key { get; }
// }
public async Task EnsureSet<T>(Meta<T> meta, Func<Task<T>> value)
{
if (await Fetch(meta.Key) != null) return;
await Set(meta, await value());
}
public async Task EnsureSet<T>(IReadOnlyList<Meta<T>> metas, Func<List<T>> values)
{
if (await GetDbContext().MetaStore.CountAsync(p => metas.Select(m => m.Key).Contains(p.Key)) == metas.Count)
return;
var resolvedValues = values();
if (resolvedValues.Count != metas.Count)
throw new Exception("Metas count doesn't match values count");
for (var i = 0; i < metas.Count; i++)
await Set(metas[i], resolvedValues[i]);
}
public async Task Set<T>(Meta<T> meta, T value) => await Set(meta.Key, meta.ConvertSet(value), meta.Ttl);
public async Task WarmupCache()
{
await Get(MetaEntity.VapidPrivateKey);
await Get(MetaEntity.VapidPublicKey);
await Get(MetaEntity.InstanceName);
await Get(MetaEntity.InstanceDescription);
}
private async Task<T> Fetch<T>(Meta<T> meta) => meta.GetConverter(await Fetch(meta.Key));
private async Task<string?> Fetch(string key) =>
await GetDbContext().MetaStore.Where(p => p.Key == key).Select(p => p.Value).FirstOrDefaultAsync();
private async Task Set(string key, string? value, TimeSpan ttl)
{
var db = GetDbContext();
var entity = await db.MetaStore.FirstOrDefaultAsync(p => p.Key == key);
if (entity != null)
{
entity.Value = value;
await db.SaveChangesAsync();
}
else
{
entity = new MetaStoreEntry { Key = key, Value = value };
db.Add(entity);
try
{
await db.SaveChangesAsync();
}
catch (UniqueConstraintException)
{
db.Remove(entity);
entity = await db.MetaStore.FirstOrDefaultAsync(p => p.Key == key) ??
throw new Exception("Failed to fetch entity after UniqueConstraintException");
entity.Value = value;
await db.SaveChangesAsync();
}
}
await cache.SetAsync($"cache:meta:{key}", value, ttl, true);
}
private DatabaseContext GetDbContext() =>
scopeFactory.CreateScope().ServiceProvider.GetRequiredService<DatabaseContext>();
}
public static class MetaEntity
{
public static readonly StringMeta VapidPrivateKey = new("vapid_private_key");
public static readonly StringMeta VapidPublicKey = new("vapid_public_key");
public static readonly NullableStringMeta InstanceName = new("instance_name");
public static readonly NullableStringMeta InstanceDescription = new("instance_description");
}
public class Meta<T>(string key, TimeSpan? ttl, Func<string?, T> getConverter, Func<T, string?> setConverter)
{
public string Key => key;
public TimeSpan Ttl => ttl ?? TimeSpan.FromDays(30);
public Func<string?, T> GetConverter => getConverter;
public Func<T, string?> ConvertSet => setConverter;
}
public class StringMeta(string key, TimeSpan? ttl = null) : NonNullableMeta<string>(key, ttl, val => val, val => val);
public class NullableStringMeta(string key, TimeSpan? ttl = null) : Meta<string?>(key, ttl, val => val, val => val);
public class IntMeta(string key, TimeSpan? ttl = null)
: NonNullableValueMeta<int>(key, ttl,
int.Parse,
val => val.ToString());
public class NullableIntMeta(string key, TimeSpan? ttl = null)
: Meta<int?>(key, ttl, val => int.TryParse(val, out var result) ? result : null, val => val?.ToString());
public class NonNullableMeta<T>(string key, TimeSpan? ttl, Func<string, T> getConverter, Func<T, string> setConverter)
: Meta<T>(key, ttl,
val => getConverter(val ?? throw new Exception($"Fetched meta value {key} was null")),
setConverter) where T : class;
public class NonNullableValueMeta<T>(
string key,
TimeSpan? ttl,
Func<string, T> getConverter,
Func<T, string> setConverter
) : Meta<T>(key, ttl,
val => getConverter(val ?? throw new Exception($"Fetched meta value {key} was null")),
setConverter) where T : struct;

9
Iceshrimp.Backend/Core/Services/PushService.cs
View file

@ -22,7 +22,7 @@ public class PushService(
IServiceScopeFactory scopeFactory,
HttpClient httpClient,
IOptions<Config.InstanceSection> config,
MetaService metaService
MetaService meta
) : BackgroundService
{
protected override Task ExecuteAsync(CancellationToken stoppingToken)
@ -85,11 +85,8 @@ public class PushService(
if (body.Length > 137)
body = body.Truncate(137).TrimEnd() + "...";
var priv = await metaService.GetVapidPrivateKey();
var pub = await metaService.GetVapidPublicKey();
if (priv == null || pub == null)
throw new Exception("Failed to fetch VAPID keys");
var priv = await meta.Get(MetaEntity.VapidPrivateKey);
var pub = await meta.Get(MetaEntity.VapidPublicKey);
var client = new WebPushClient(httpClient);
client.SetVapidDetails(new VapidDetails($"https://{config.Value.WebDomain}", pub, priv));