From 657bc43761de11af8c9a96575863d1afdca9a922 Mon Sep 17 00:00:00 2001 From: Laura Hausmann Date: Thu, 7 Mar 2024 02:22:57 +0100 Subject: [PATCH] [backend/core] Finish the polls implementation (ISH-130, ISH-131) --- .../Controllers/Mastodon/PollController.cs | 8 --- .../Controllers/Mastodon/StatusController.cs | 17 +++-- .../Federation/ActivityPub/NoteRenderer.cs | 59 +++++++++++++++-- .../Core/Queues/BackgroundTaskQueue.cs | 28 +++++++- .../Core/Services/NoteService.cs | 66 ++++++++++++++++--- .../Core/Services/PollService.cs | 5 ++ .../Core/Services/QueueService.cs | 21 +++--- 7 files changed, 166 insertions(+), 38 deletions(-) diff --git a/Iceshrimp.Backend/Controllers/Mastodon/PollController.cs b/Iceshrimp.Backend/Controllers/Mastodon/PollController.cs index f38e0af9..86885a97 100644 --- a/Iceshrimp.Backend/Controllers/Mastodon/PollController.cs +++ b/Iceshrimp.Backend/Controllers/Mastodon/PollController.cs @@ -60,7 +60,6 @@ public class PollController(DatabaseContext db, PollRenderer pollRenderer, PollS throw GracefulException.BadRequest("This poll is expired"); var existingVotes = await db.PollVotes.Where(p => p.User == user && p.Note == note).ToListAsync(); - List incr = []; List votes = []; if (!poll.Multiple) { @@ -81,7 +80,6 @@ public class PollController(DatabaseContext db, PollRenderer pollRenderer, PollS }; await db.AddAsync(vote); - incr.Add(choice); votes.Add(vote); } else @@ -101,17 +99,11 @@ public class PollController(DatabaseContext db, PollRenderer pollRenderer, PollS }; await db.AddAsync(vote); - incr.Add(choice); votes.Add(vote); } } await db.SaveChangesAsync(); - foreach (var c in incr) - { - await db.Database - .ExecuteSqlAsync($"""UPDATE "poll" SET "votes"[{c + 1}] = "votes"[{c + 1}] + 1 WHERE "noteId" = {note.Id}"""); - } foreach (var vote in votes) await pollSvc.RegisterPollVote(vote, poll, note); diff --git a/Iceshrimp.Backend/Controllers/Mastodon/StatusController.cs b/Iceshrimp.Backend/Controllers/Mastodon/StatusController.cs index 4ce659a5..d83cbd86 100644 --- a/Iceshrimp.Backend/Controllers/Mastodon/StatusController.cs +++ b/Iceshrimp.Backend/Controllers/Mastodon/StatusController.cs @@ -268,11 +268,17 @@ public class StatusController( } } - if (request.Text == null && request.MediaIds is not { Count: > 0 } && request.Poll == null) + if (string.IsNullOrWhiteSpace(request.Text) && request.MediaIds is not { Count: > 0 } && request.Poll == null) throw GracefulException.BadRequest("Posts must have text, media or poll"); - if (request.Poll != null) - throw GracefulException.BadRequest("Polls haven't been implemented yet"); + var poll = request.Poll != null + ? new Poll + { + Choices = request.Poll.Options, + Multiple = request.Poll.Multiple, + ExpiresAt = DateTime.UtcNow + TimeSpan.FromSeconds(request.Poll.ExpiresIn), + } + : null; var visibility = StatusEntity.DecodeVisibility(request.Visibility); var reply = request.ReplyId != null @@ -302,7 +308,8 @@ public class StatusController( if (quote != null && quoteUri != null && request.Text != null) request.Text = request.Text[..(request.Text.Length - quoteUri.Length - 1)]; - var note = await noteSvc.CreateNoteAsync(user, visibility, request.Text, request.Cw, reply, quote, attachments); + var note = await noteSvc.CreateNoteAsync(user, visibility, request.Text, request.Cw, reply, quote, attachments, + poll); if (idempotencyKey != null) await cache.SetAsync($"idempotency:{idempotencyKey}", note.Id, TimeSpan.FromHours(24)); @@ -327,7 +334,7 @@ public class StatusController( throw GracefulException.BadRequest("Posts must have text, media or poll"); if (request.Poll != null) - throw GracefulException.BadRequest("Polls haven't been implemented yet"); + throw GracefulException.BadRequest("Poll edits haven't been implemented yet, please delete & redraft instead"); var attachments = request.MediaIds != null ? await db.DriveFiles.Where(p => request.MediaIds.Contains(p.Id)).ToListAsync() diff --git a/Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs b/Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs index 7ad6effc..f13483d7 100644 --- a/Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs +++ b/Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs @@ -89,11 +89,61 @@ public class NoteRenderer(IOptions config, MfmConverter var quoteUri = note.IsQuote ? note.Renote?.Uri ?? note.Renote?.GetPublicUriOrNull(config.Value) : null; var text = quoteUri != null ? note.Text + $"\n\nRE: {quoteUri}" : note.Text; + if (note.HasPoll) + { + var poll = await db.Polls.FirstOrDefaultAsync(p => p.Note == note); + if (poll != null) + { + var closed = poll.ExpiresAt != null && poll.ExpiresAt < DateTime.UtcNow ? poll.ExpiresAt : null; + var endTime = poll.ExpiresAt != null && poll.ExpiresAt > DateTime.UtcNow ? poll.ExpiresAt : null; + + var choices = poll.Choices + .Select(p => new ASQuestion.ASQuestionOption + { + Name = p, + Replies = new ASCollectionBase + { + TotalItems = + (ulong)poll.Votes[poll.Choices.IndexOf(p)] + } + }) + .ToList(); + + var anyOf = poll.Multiple ? choices : null; + var oneOf = !poll.Multiple ? choices : null; + + return new ASQuestion + { + Id = id, + AttributedTo = [new ASObjectBase(userId)], + MkContent = note.Text, + PublishedAt = note.CreatedAt, + Sensitive = note.Cw != null, + InReplyTo = replyId, + Cc = cc, + To = to, + Tags = tags, + Attachments = attachments, + Content = text != null ? await mfmConverter.ToHtmlAsync(text, mentions, note.UserHost) : null, + Summary = note.Cw, + Source = text != null + ? new ASNoteSource { Content = text, MediaType = "text/x.misskeymarkdown" } + : null, + MkQuote = quoteUri, + QuoteUri = quoteUri, + QuoteUrl = quoteUri, + EndTime = endTime, + Closed = closed, + AnyOf = anyOf, + OneOf = oneOf + }; + } + } + return new ASNote { Id = id, AttributedTo = [new ASObjectBase(userId)], - Type = $"{Constants.ActivityStreamsNs}#Note", MkContent = note.Text, PublishedAt = note.CreatedAt, Sensitive = note.Cw != null, @@ -104,10 +154,9 @@ public class NoteRenderer(IOptions config, MfmConverter Attachments = attachments, Content = text != null ? await mfmConverter.ToHtmlAsync(text, mentions, note.UserHost) : null, Summary = note.Cw, - Source = - text != null - ? new ASNoteSource { Content = text, MediaType = "text/x.misskeymarkdown" } - : null, + Source = text != null + ? new ASNoteSource { Content = text, MediaType = "text/x.misskeymarkdown" } + : null, MkQuote = quoteUri, QuoteUri = quoteUri, QuoteUrl = quoteUri diff --git a/Iceshrimp.Backend/Core/Queues/BackgroundTaskQueue.cs b/Iceshrimp.Backend/Core/Queues/BackgroundTaskQueue.cs index 297e3b9b..bc62a496 100644 --- a/Iceshrimp.Backend/Core/Queues/BackgroundTaskQueue.cs +++ b/Iceshrimp.Backend/Core/Queues/BackgroundTaskQueue.cs @@ -1,3 +1,4 @@ +using System.Diagnostics.CodeAnalysis; using Iceshrimp.Backend.Core.Configuration; using Iceshrimp.Backend.Core.Database; using Iceshrimp.Backend.Core.Extensions; @@ -131,6 +132,9 @@ public abstract class BackgroundTaskQueue } } + [SuppressMessage("ReSharper", "EntityFramework.NPlusOne.IncompleteDataQuery", + Justification = "IncludeCommonProperties()")] + [SuppressMessage("ReSharper", "EntityFramework.NPlusOne.IncompleteDataUsage", Justification = "Same as above")] private static async Task ProcessPollExpiry( PollExpiryJob job, IServiceProvider scope, @@ -141,11 +145,31 @@ public abstract class BackgroundTaskQueue var poll = await db.Polls.FirstOrDefaultAsync(p => p.NoteId == job.NoteId, cancellationToken: token); if (poll == null) return; if (poll.ExpiresAt > DateTime.UtcNow + TimeSpan.FromMinutes(5)) return; - var note = await db.Notes.IncludeCommonProperties().FirstOrDefaultAsync(p => p.Id == poll.NoteId, cancellationToken: token); + var note = await db.Notes.IncludeCommonProperties() + .FirstOrDefaultAsync(p => p.Id == poll.NoteId, cancellationToken: token); if (note == null) return; - //TODO: try to update poll before completing it + var notificationSvc = scope.GetRequiredService(); await notificationSvc.GeneratePollEndedNotifications(note); + if (note.User.Host == null) + { + var voters = await db.PollVotes.Where(p => p.Note == note && p.User.Host != null) + .Select(p => p.User) + .ToListAsync(cancellationToken: token); + + if (voters.Count == 0) return; + + var activityRenderer = scope.GetRequiredService(); + var userRenderer = scope.GetRequiredService(); + var noteRenderer = scope.GetRequiredService(); + var deliverSvc = scope.GetRequiredService(); + + var actor = userRenderer.RenderLite(note.User); + var rendered = await noteRenderer.RenderAsync(note); + var activity = activityRenderer.RenderUpdate(rendered, actor); + + await deliverSvc.DeliverToAsync(activity, note.User, voters.ToArray()); + } } } diff --git a/Iceshrimp.Backend/Core/Services/NoteService.cs b/Iceshrimp.Backend/Core/Services/NoteService.cs index e00ecd85..bfa55630 100644 --- a/Iceshrimp.Backend/Core/Services/NoteService.cs +++ b/Iceshrimp.Backend/Core/Services/NoteService.cs @@ -41,7 +41,8 @@ public class NoteService( EmojiService emojiSvc, FollowupTaskService followupTaskSvc, ActivityPub.ObjectResolver objectResolver, - QueueService queueSvc + QueueService queueSvc, + PollService pollSvc ) { private readonly List _resolverHistory = []; @@ -49,7 +50,7 @@ public class NoteService( public async Task CreateNoteAsync( User user, Note.NoteVisibility visibility, string? text = null, string? cw = null, Note? reply = null, - Note? renote = null, IReadOnlyCollection? attachments = null + Note? renote = null, IReadOnlyCollection? attachments = null, Poll? poll = null ) { if (text?.Length > config.Value.CharacterLimit) @@ -65,6 +66,8 @@ public class NoteService( throw GracefulException.BadRequest("Cannot renote or quote a pure renote"); if (reply?.IsPureRenote ?? false) throw GracefulException.BadRequest("Cannot reply to a pure renote"); + if (poll is { Choices.Count: < 2 }) + throw GracefulException.BadRequest("Polls must have at least two options"); var (mentionedUserIds, mentionedLocalUserIds, mentions, remoteMentions, splitDomainMapping) = await ResolveNoteMentionsAsync(text); @@ -107,6 +110,18 @@ public class NoteService( Tags = tags }; + if (poll != null) + { + poll.Note = note; + poll.UserId = note.User.Id; + poll.UserHost = note.UserHost; + poll.Votes = poll.Choices.Select(_ => 0).ToList(); + poll.NoteVisibility = note.Visibility; + await db.AddAsync(poll); + note.HasPoll = true; + await EnqueuePollExpiryTask(poll); + } + await UpdateNoteCountersAsync(note, true); await db.AddAsync(note); @@ -371,7 +386,7 @@ public class NoteService( eventSvc.RaiseNoteDeleted(this, hit); } - public async Task ProcessNoteAsync(ASNote note, User actor, User? user = null) + public async Task ProcessNoteAsync(ASNote note, User actor, User? user = null) { var dbHit = await db.Notes.IncludeCommonProperties().FirstOrDefaultAsync(p => p.Uri == note.Id); @@ -398,11 +413,46 @@ public class NoteService( throw GracefulException.UnprocessableEntity("Note.Id schema is invalid"); if (note.Url?.Link != null && !note.Url.Link.StartsWith("https://")) throw GracefulException.UnprocessableEntity("Note.Url schema is invalid"); - if (note.PublishedAt is null or { Year: < 2007 } || note.PublishedAt > DateTime.Now + TimeSpan.FromDays(3)) - throw GracefulException.UnprocessableEntity("Note.PublishedAt is nonsensical"); if (actor.IsSuspended) throw GracefulException.Forbidden("User is suspended"); + var reply = note.InReplyTo?.Id != null ? await ResolveNoteAsync(note.InReplyTo.Id, user: user) : null; + + if (reply is { HasPoll: true } && note.Name != null) + { + if (reply.UserHost != null) + throw GracefulException.UnprocessableEntity("Poll vote not destined for this instance"); + + var poll = await db.Polls.FirstOrDefaultAsync(p => p.Note == reply) ?? + throw GracefulException.UnprocessableEntity("Poll does not exist"); + + if (poll.Choices.All(p => p != note.Name)) + throw GracefulException.UnprocessableEntity("Unknown poll option"); + + var existingVotes = await db.PollVotes.Where(p => p.User == actor && p.Note == reply).ToListAsync(); + if (existingVotes.Any(p => poll.Choices[p.Choice] == note.Name)) + throw GracefulException.UnprocessableEntity("Actor has already voted for this option"); + if (!poll.Multiple && existingVotes.Count != 0) + throw GracefulException.UnprocessableEntity("Actor has already voted in this poll"); + + var vote = new PollVote + { + Id = IdHelpers.GenerateSlowflakeId(), + CreatedAt = DateTime.UtcNow, + User = actor, + Note = reply, + Choice = poll.Choices.IndexOf(note.Name) + }; + await db.AddAsync(vote); + await db.SaveChangesAsync(); + await pollSvc.RegisterPollVote(vote, poll, reply); + + return null; + } + + if (note.PublishedAt is null or { Year: < 2007 } || note.PublishedAt > DateTime.Now + TimeSpan.FromDays(3)) + throw GracefulException.UnprocessableEntity("Note.PublishedAt is nonsensical"); + var (mentionedUserIds, mentionedLocalUserIds, mentions, remoteMentions, splitDomainMapping) = await ResolveNoteMentionsAsync(note); @@ -422,7 +472,7 @@ public class NoteService( CreatedAt = createdAt, UserHost = actor.Host, Visibility = note.GetVisibility(actor), - Reply = note.InReplyTo?.Id != null ? await ResolveNoteAsync(note.InReplyTo.Id, user: user) : null, + Reply = reply, Renote = quoteUrl != null ? await ResolveNoteAsync(quoteUrl, user: user) : null }; @@ -524,7 +574,7 @@ public class NoteService( [SuppressMessage("ReSharper", "EntityFramework.NPlusOne.IncompleteDataUsage", Justification = "Inspection doesn't understand IncludeCommonProperties()")] [SuppressMessage("ReSharper", "EntityFramework.NPlusOne.IncompleteDataQuery", Justification = "See above")] - public async Task ProcessNoteUpdateAsync(ASNote note, User actor) + public async Task ProcessNoteUpdateAsync(ASNote note, User actor) { var dbNote = await db.Notes.IncludeCommonProperties() .Include(p => p.Poll) @@ -1016,7 +1066,7 @@ public class NoteService( private async Task EnqueuePollExpiryTask(Poll poll) { if (!poll.ExpiresAt.HasValue) return; - var job = new PollExpiryJob { NoteId = poll.NoteId }; + var job = new PollExpiryJob { NoteId = poll.Note.Id }; await queueSvc.BackgroundTaskQueue.ScheduleAsync(job, poll.ExpiresAt.Value); } } \ No newline at end of file diff --git a/Iceshrimp.Backend/Core/Services/PollService.cs b/Iceshrimp.Backend/Core/Services/PollService.cs index dcdeb81e..df9baf6d 100644 --- a/Iceshrimp.Backend/Core/Services/PollService.cs +++ b/Iceshrimp.Backend/Core/Services/PollService.cs @@ -1,8 +1,11 @@ +using Iceshrimp.Backend.Core.Database; using Iceshrimp.Backend.Core.Database.Tables; +using Microsoft.EntityFrameworkCore; namespace Iceshrimp.Backend.Core.Services; public class PollService( + DatabaseContext db, ActivityPub.ActivityRenderer activityRenderer, ActivityPub.UserRenderer userRenderer, ActivityPub.ActivityDeliverService deliverSvc @@ -10,6 +13,8 @@ public class PollService( { public async Task RegisterPollVote(PollVote pollVote, Poll poll, Note note) { + await db.Database + .ExecuteSqlAsync($"""UPDATE "poll" SET "votes"[{pollVote.Choice + 1}] = "votes"[{pollVote.Choice + 1}] + 1 WHERE "noteId" = {note.Id}"""); if (poll.UserHost == null) return; var vote = activityRenderer.RenderVote(pollVote, poll, note); diff --git a/Iceshrimp.Backend/Core/Services/QueueService.cs b/Iceshrimp.Backend/Core/Services/QueueService.cs index ad364e7c..cfd2e647 100644 --- a/Iceshrimp.Backend/Core/Services/QueueService.cs +++ b/Iceshrimp.Backend/Core/Services/QueueService.cs @@ -129,7 +129,7 @@ public class JobQueue( { try { - var timestamp = (long)DateTime.Now.Subtract(DateTime.UnixEpoch).TotalSeconds; + var timestamp = (long)DateTime.UtcNow.Subtract(DateTime.UnixEpoch).TotalSeconds; var res = await _redisDb.SortedSetRangeByScoreAsync("delayed", 0, timestamp, take: 10); if (res.Length == 0) @@ -233,13 +233,14 @@ public class JobQueue( await _redisDb.ListRemoveAsync("running", res, 1); if (targetQueue == "delayed") { - job.DelayedUntil ??= DateTime.Now; + job.DelayedUntil = (job.DelayedUntil ?? DateTime.Now).ToLocalTime(); + var logger = scope.ServiceProvider.GetRequiredService>(); logger.LogTrace("Job in queue {queue} was delayed to {time} after {duration} ms, has been queued since {time}", name, job.DelayedUntil.Value.ToStringIso8601Like(), job.Duration, job.QueuedAt.ToStringIso8601Like()); - var timestamp = (long)job.DelayedUntil.Value.Subtract(DateTime.UnixEpoch).TotalSeconds; + var timestamp = (long)job.DelayedUntil.Value.ToUniversalTime().Subtract(DateTime.UnixEpoch).TotalSeconds; await _redisDb.SortedSetAddAsync(targetQueue, RedisValue.Unbox(RedisHelpers.Serialize(job)), timestamp); await _subscriber.PublishAsync(_delayedChannel, ""); } @@ -257,15 +258,15 @@ public class JobQueue( await _redisDb.ListRightPushAsync("queued", RedisValue.Unbox(RedisHelpers.Serialize(job))); await _subscriber.PublishAsync(_queuedChannel, ""); } - + public async Task ScheduleAsync(T job, DateTime triggerAt) { - job.Status = Job.JobStatus.Delayed; - job.DelayedUntil = triggerAt; - var timestamp = (long)job.DelayedUntil.Value.Subtract(DateTime.UnixEpoch).TotalSeconds; - await _redisDb.SortedSetAddAsync("delayed", RedisValue.Unbox(RedisHelpers.Serialize(job)), timestamp); - await _subscriber.PublishAsync(_delayedChannel, ""); - } + job.Status = Job.JobStatus.Delayed; + job.DelayedUntil = triggerAt.ToLocalTime(); + var timestamp = (long)job.DelayedUntil.Value.ToUniversalTime().Subtract(DateTime.UnixEpoch).TotalSeconds; + await _redisDb.SortedSetAddAsync("delayed", RedisValue.Unbox(RedisHelpers.Serialize(job)), timestamp); + await _subscriber.PublishAsync(_delayedChannel, ""); + } } [ProtoContract]