From d1721d71c280859d55b789552da807cc9c6e4dbd Mon Sep 17 00:00:00 2001
From: Laura Hausmann <laura@hausmann.dev>
Date: Tue, 24 Sep 2024 00:36:13 +0200
Subject: [PATCH] [backend/federation] Retry inbox jobs with exponential
 backoff (ISH-499)

This should cover most transient failures. If one is missing, we can add it later.
---
 Iceshrimp.Backend/Core/Queues/InboxQueue.cs | 24 +++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/Iceshrimp.Backend/Core/Queues/InboxQueue.cs b/Iceshrimp.Backend/Core/Queues/InboxQueue.cs
index 3e2decfd..acf777b9 100644
--- a/Iceshrimp.Backend/Core/Queues/InboxQueue.cs
+++ b/Iceshrimp.Backend/Core/Queues/InboxQueue.cs
@@ -37,6 +37,30 @@ public class InboxQueue(int parallelism)
 			else
 				logger.LogDebug("Refusing to process activity {id}: Instance is blocked ({uri})", job.Id, e.Uri);
 		}
+		catch (Exception e) when (e is not GracefulException)
+		{
+			if (job.RetryCount++ < 4)
+			{
+				var jitter     = TimeSpan.FromSeconds(new Random().Next(0, 60));
+				var baseDelay  = TimeSpan.FromMinutes(1);
+				var maxBackoff = TimeSpan.FromHours(8);
+				var backoff    = (Math.Pow(2, job.RetryCount) - 1) * baseDelay;
+				if (backoff > maxBackoff)
+					backoff = maxBackoff;
+				backoff += jitter;
+
+				job.ExceptionMessage = e.Message;
+				job.ExceptionSource  = e.Source;
+				job.StackTrace       = e.StackTrace;
+				job.Exception        = e.ToString();
+				job.DelayedUntil     = DateTime.UtcNow + backoff;
+				job.Status           = Job.JobStatus.Delayed;
+			}
+			else
+			{
+				throw;
+			}
+		}
 	}
 }