[backend/cron] Add cron task statistics & drastically improve cron logic (ISH-760)

Iceshrimp.Backend/Controllers/Web/AdminController.cs

@@ -303,10 +303,13 @@ public class AdminController(
 	[ProducesErrors(HttpStatusCode.NotFound)]
 	public void RunCronTask([FromServices] CronService cronSvc, string id)
 	{
-		var task = cronSvc.Tasks.FirstOrDefault(p => p.GetType().FullName == id)
+		var task = cronSvc.Tasks.FirstOrDefault(p => p.Task.GetType().FullName == id)
 		           ?? throw GracefulException.NotFound("Task not found");
 
-		_ = cronSvc.RunCronTaskAsync(task);
+		Task.Factory.StartNew(async () => await cronSvc.RunCronTaskAsync(task.Task, task.Trigger),
+		                      CancellationToken.None,
+		                      TaskCreationOptions.DenyChildAttach | TaskCreationOptions.LongRunning,
+		                      TaskScheduler.Default);
 	}
 
 	[HttpGet("policy")]

Iceshrimp.Backend/Core/Extensions/TimeSpanExtensions.cs

@@ -19,7 +19,7 @@ public static class TimeSpanExtensions
 		if (timeSpan.Ticks < Minutes)
 		{
 			var minutes = (int)timeSpan.TotalMinutes;
-			return minutes == 1 ? singleNumber ? "1 minute" : "minute" : $"{timeSpan.TotalMinutes} minutes";
+			return minutes == 1 ? singleNumber ? "1 minute" : "minute" : $"{minutes} minutes";
 		}
 
 		if (timeSpan.Ticks < Hours)
@@ -29,6 +29,6 @@ public static class TimeSpanExtensions
 		}
 
 		var days = (int)timeSpan.TotalDays;
-		return days == 1 ? singleNumber ? "1 day" : "day" : $"{timeSpan.TotalDays} days";
+		return days == 1 ? singleNumber ? "1 day" : "day" : $"{days} days";
 	}
 }

Iceshrimp.Backend/Core/Services/CronService.cs

@@ -6,27 +6,43 @@ namespace Iceshrimp.Backend.Core.Services;
 
 public class CronService(IServiceScopeFactory serviceScopeFactory) : BackgroundService
 {
-	public ICronTask[] Tasks { get; private set; } = [];
+	public CronTaskState[] Tasks { get; private set; } = [];
 
-	public async Task RunCronTaskAsync(ICronTask task)
+	public async Task RunCronTaskAsync(ICronTask task, ICronTrigger trigger)
 	{
-		await using var scope = serviceScopeFactory.CreateAsyncScope();
+		if (trigger.IsRunning) return;
-		await task.InvokeAsync(scope.ServiceProvider);
+		trigger.IsRunning = true;
+		trigger.Error     = null;
+
+		try
+		{
+			await using var scope = serviceScopeFactory.CreateAsyncScope();
+			await task.InvokeAsync(scope.ServiceProvider);
+		}
+		catch (Exception e)
+		{
+			trigger.Error = e;
+		}
+
+		trigger.LastRun   = DateTime.UtcNow;
+		trigger.IsRunning = false;
 	}
 
 	protected override Task ExecuteAsync(CancellationToken token)
 	{
-		Tasks = PluginLoader
+		var tasks = PluginLoader.Assemblies
-		            .Assemblies.Prepend(Assembly.GetExecutingAssembly())
+		                        .Prepend(Assembly.GetExecutingAssembly())
-		            .SelectMany(AssemblyLoader.GetImplementationsOfInterface<ICronTask>)
+		                        .SelectMany(AssemblyLoader.GetImplementationsOfInterface<ICronTask>)
-		            .OrderBy(p => p.AssemblyQualifiedName)
+		                        .OrderBy(p => p.AssemblyQualifiedName)
-		            .ThenBy(p => p.Name)
+		                        .ThenBy(p => p.Name)
-		            .Select(p => Activator.CreateInstance(p) as ICronTask)
+		                        .Select(p => Activator.CreateInstance(p) as ICronTask)
-		            .Where(p => p != null)
+		                        .Where(p => p != null)
-		            .Cast<ICronTask>()
+		                        .Cast<ICronTask>()
-		            .ToArray();
+		                        .ToArray();
 
-		foreach (var task in Tasks)
+		List<CronTaskState> stateObjs = [];
+
+		foreach (var task in tasks)
 		{
 			ICronTrigger trigger = task.Type switch
 			{
@@ -35,23 +51,33 @@ public class CronService(IServiceScopeFactory serviceScopeFactory) : BackgroundS
 				_                     => throw new ArgumentOutOfRangeException()
 			};
 
-			trigger.OnTrigger += async void () =>
+			trigger.OnTrigger += async void (state) =>
 			{
 				try
 				{
-					await RunCronTaskAsync(task);
+					await RunCronTaskAsync(task, state);
 				}
 				catch
 				{
 					// ignored (errors in the event handler crash the host process)
 				}
 			};
+
+			stateObjs.Add(new CronTaskState { Task = task, Trigger = trigger });
 		}
 
+		Tasks = stateObjs.ToArray();
+
 		return Task.CompletedTask;
 	}
 }
 
+public class CronTaskState
+{
+	public required ICronTask    Task;
+	public required ICronTrigger Trigger;
+}
+
 public interface ICronTask
 {
 	public TimeSpan     Trigger { get; }
@@ -65,36 +91,59 @@ public enum CronTaskType
 	Interval
 }
 
-file interface ICronTrigger
+public interface ICronTrigger
 {
-	public event Action? OnTrigger;
+	public event Action<ICronTrigger>? OnTrigger;
+
+	public DateTime   NextTrigger { get; }
+	public DateTime?  LastRun     { get; set; }
+	public bool       IsRunning   { get; set; }
+	public Exception? Error       { get; set; }
+
+	public TimeSpan UpdateNextTrigger();
 }
 
-file class DailyTrigger : ICronTrigger, IDisposable
+public class DailyTrigger : ICronTrigger, IDisposable
 {
 	public DailyTrigger(TimeSpan triggerTime, CancellationToken cancellationToken)
 	{
 		TriggerTime       = triggerTime;
 		CancellationToken = cancellationToken;
+		NextTrigger       = DateTime.UtcNow;
 
 		RunningTask = Task.Factory.StartNew(async () =>
 		{
 			while (!CancellationToken.IsCancellationRequested)
 			{
-				var nextTrigger = DateTime.Today + TriggerTime - DateTime.Now;
+				var nextTrigger = UpdateNextTrigger();
-				if (nextTrigger < TimeSpan.Zero)
-					nextTrigger = nextTrigger.Add(new TimeSpan(24, 0, 0));
 				await Task.Delay(nextTrigger, CancellationToken);
-				OnTrigger?.Invoke();
+				OnTrigger?.Invoke(this);
 			}
 		}, CancellationToken, TaskCreationOptions.DenyChildAttach | TaskCreationOptions.LongRunning, TaskScheduler.Default);
 	}
 
+	public TimeSpan UpdateNextTrigger()
+	{
+		var nextTrigger = DateTime.Today + TriggerTime - DateTime.Now;
+		if (nextTrigger < TimeSpan.Zero)
+			nextTrigger = nextTrigger.Add(new TimeSpan(24, 0, 0));
+
+		NextTrigger = DateTime.UtcNow + nextTrigger;
+
+		return nextTrigger;
+	}
+
 	private TimeSpan          TriggerTime       { get; }
 	private CancellationToken CancellationToken { get; }
 	private Task              RunningTask       { get; set; }
 
-	public event Action? OnTrigger;
+	public event Action<ICronTrigger>? OnTrigger;
+
+	public DateTime   NextTrigger { get; set; }
+	public DateTime?  LastRun     { get; set; }
+	public bool       IsRunning   { get; set; }
+
+	public Exception? Error { get; set; } = null;
 
 	public void Dispose()
 	{
@@ -106,28 +155,49 @@ file class DailyTrigger : ICronTrigger, IDisposable
 	~DailyTrigger() => Dispose();
 }
 
-file class IntervalTrigger : ICronTrigger, IDisposable
+public class IntervalTrigger : ICronTrigger, IDisposable
 {
 	public IntervalTrigger(TimeSpan triggerInterval, CancellationToken cancellationToken)
 	{
 		TriggerInterval   = triggerInterval;
 		CancellationToken = cancellationToken;
+		NextTrigger       = DateTime.UtcNow + TriggerInterval;
 
 		RunningTask = Task.Factory.StartNew(async () =>
 		{
 			while (!CancellationToken.IsCancellationRequested)
 			{
+				UpdateNextTrigger();
 				await Task.Delay(TriggerInterval, CancellationToken);
-				OnTrigger?.Invoke();
+
+				while (LastRun != null && LastRun + TriggerInterval + TimeSpan.FromMinutes(5) < DateTime.UtcNow)
+				{
+					NextTrigger = LastRun.Value + TriggerInterval;
+					await Task.Delay(NextTrigger - DateTime.UtcNow);
+				}
+
+				OnTrigger?.Invoke(this);
 			}
 		}, CancellationToken, TaskCreationOptions.DenyChildAttach | TaskCreationOptions.LongRunning, TaskScheduler.Default);
 	}
 
+	public TimeSpan UpdateNextTrigger()
+	{
+		NextTrigger = DateTime.UtcNow + TriggerInterval;
+		return TriggerInterval;
+	}
+
 	private TimeSpan          TriggerInterval   { get; }
 	private CancellationToken CancellationToken { get; }
 	private Task              RunningTask       { get; set; }
 
-	public event Action? OnTrigger;
+	public event Action<ICronTrigger>? OnTrigger;
+
+	public DateTime  NextTrigger { get; set; }
+	public DateTime? LastRun     { get; set; }
+	public bool      IsRunning   { get; set; }
+
+	public Exception? Error { get; set; } = null;
 
 	public void Dispose()
 	{

Iceshrimp.Backend/Pages/Admin/Tasks.razor

@@ -6,30 +6,56 @@
 @inject CronService CronSvc
 <AdminPageHeader Title="Cron tasks"/>
 
-<table>
+<table class="auto-table">
     <thead>
     <th>Name</th>
     <th>Assembly</th>
     <th>Schedule</th>
+    <th>Last run</th>
+    <th>Next run</th>
     <th>Actions</th>
     </thead>
     <tbody>
     @foreach (var task in CronSvc.Tasks)
     {
-        var type = task.GetType();
+        var type = task.Task.GetType();
-        var schedule = task.Type switch
+        var schedule = task.Task.Type switch
         {
-            CronTaskType.Daily when task.Trigger == TimeSpan.Zero => "daily at midnight",
+            CronTaskType.Daily when task.Task.Trigger == TimeSpan.Zero => "daily at midnight",
             //
-            CronTaskType.Daily    => $"daily at {((int)task.Trigger.TotalHours).ToString().PadLeft(2, '0')}:{((int)task.Trigger.TotalMinutes).ToString().PadLeft(2, '0')}",
+            CronTaskType.Daily    => $"daily at {((int)task.Task.Trigger.TotalHours).ToString().PadLeft(2, '0')}:{((int)task.Task.Trigger.TotalMinutes).ToString().PadLeft(2, '0')}",
-            CronTaskType.Interval => $"every {task.Trigger.ToDisplayString(singleNumber: false)}",
+            CronTaskType.Interval => $"every {task.Task.Trigger.ToDisplayString(singleNumber: false)}",
             _                     => throw new ArgumentOutOfRangeException()
         };
 
+        var last = task.Trigger.IsRunning
+            ? "now"
+            : task.Trigger.LastRun.HasValue
+                ? (DateTime.UtcNow - task.Trigger.LastRun.Value).ToDisplayString() + " ago"
+                : "never";
+
+        if (task.Trigger.LastRun.HasValue)
+        {
+            last += task.Trigger.Error is null ? " (OK)" : " (Error)";
+        }
+
+        var next = task.Trigger.IsRunning
+            ? "now"
+            : "in " + (task.Trigger.NextTrigger - DateTime.UtcNow).ToDisplayString();
+
         <tr>
             <td>@type.Name</td>
             <td>@type.Assembly.GetName().Name</td>
             <td>@schedule</td>
+            @if (task.Trigger is { LastRun: not null, Error: { } e })
+            {
+                <td title="@e.ToString()">@last</td>
+            }
+            else
+            {
+                <td>@last</td>
+            }
+            <td>@next</td>
             <td><a class="fake-link" onclick="runCronTask('@type.FullName', event.target)">Run now</a></td>
         </tr>
     }