notfire/Iceshrimp.NET
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[backend/federation] Properly handle hashtags in notes & user profiles (ISH-114, ISH-125)

Browse source
This commit is contained in:
Laura Hausmann 2024-03-04 21:19:10 +01:00
parent 810c21a275
commit 92d229ad63
No known key found for this signature in database
GPG key ID: D044E84C5BE01605
9 changed files with 6416 additions and 148 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
Iceshrimp.Backend/Core/Database/DatabaseContext.cs
View file

@ -461,15 +461,7 @@ public class DatabaseContext(DbContextOptions<DatabaseContext> options)
entity.HasOne(d => d.User).WithMany(p => p.GalleryPosts);
});
modelBuilder.Entity<Hashtag>(entity =>
{
entity.Property(e => e.AttachedLocalUsersCount).HasDefaultValue(0);
entity.Property(e => e.AttachedRemoteUsersCount).HasDefaultValue(0);
entity.Property(e => e.AttachedUsersCount).HasDefaultValue(0);
entity.Property(e => e.MentionedLocalUsersCount).HasDefaultValue(0);
entity.Property(e => e.MentionedRemoteUsersCount).HasDefaultValue(0);
entity.Property(e => e.MentionedUsersCount).HasDefaultValue(0);
});
modelBuilder.Entity<Hashtag>();
modelBuilder.Entity<Instance>(entity =>
{

6092
Iceshrimp.Backend/Core/Database/Migrations/20240304201225_RemoveHashtagStatisticsColumns.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

199
Iceshrimp.Backend/Core/Database/Migrations/20240304201225_RemoveHashtagStatisticsColumns.cs Normal file
View file

@ -0,0 +1,199 @@
using System.Collections.Generic;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Iceshrimp.Backend.Core.Database.Migrations
{
/// <inheritdoc />
public partial class RemoveHashtagStatisticsColumns : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_hashtag_attachedLocalUsersCount",
table: "hashtag");
migrationBuilder.DropIndex(
name: "IX_hashtag_attachedRemoteUsersCount",
table: "hashtag");
migrationBuilder.DropIndex(
name: "IX_hashtag_attachedUsersCount",
table: "hashtag");
migrationBuilder.DropIndex(
name: "IX_hashtag_mentionedLocalUsersCount",
table: "hashtag");
migrationBuilder.DropIndex(
name: "IX_hashtag_mentionedRemoteUsersCount",
table: "hashtag");
migrationBuilder.DropIndex(
name: "IX_hashtag_mentionedUsersCount",
table: "hashtag");
migrationBuilder.DropColumn(
name: "attachedLocalUserIds",
table: "hashtag");
migrationBuilder.DropColumn(
name: "attachedLocalUsersCount",
table: "hashtag");
migrationBuilder.DropColumn(
name: "attachedRemoteUserIds",
table: "hashtag");
migrationBuilder.DropColumn(
name: "attachedRemoteUsersCount",
table: "hashtag");
migrationBuilder.DropColumn(
name: "attachedUserIds",
table: "hashtag");
migrationBuilder.DropColumn(
name: "attachedUsersCount",
table: "hashtag");
migrationBuilder.DropColumn(
name: "mentionedLocalUserIds",
table: "hashtag");
migrationBuilder.DropColumn(
name: "mentionedLocalUsersCount",
table: "hashtag");
migrationBuilder.DropColumn(
name: "mentionedRemoteUserIds",
table: "hashtag");
migrationBuilder.DropColumn(
name: "mentionedRemoteUsersCount",
table: "hashtag");
migrationBuilder.DropColumn(
name: "mentionedUserIds",
table: "hashtag");
migrationBuilder.DropColumn(
name: "mentionedUsersCount",
table: "hashtag");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<List<string>>(
name: "attachedLocalUserIds",
table: "hashtag",
type: "character varying(32)[]",
nullable: false);
migrationBuilder.AddColumn<int>(
name: "attachedLocalUsersCount",
table: "hashtag",
type: "integer",
nullable: false,
defaultValue: 0);
migrationBuilder.AddColumn<List<string>>(
name: "attachedRemoteUserIds",
table: "hashtag",
type: "character varying(32)[]",
nullable: false);
migrationBuilder.AddColumn<int>(
name: "attachedRemoteUsersCount",
table: "hashtag",
type: "integer",
nullable: false,
defaultValue: 0);
migrationBuilder.AddColumn<List<string>>(
name: "attachedUserIds",
table: "hashtag",
type: "character varying(32)[]",
nullable: false);
migrationBuilder.AddColumn<int>(
name: "attachedUsersCount",
table: "hashtag",
type: "integer",
nullable: false,
defaultValue: 0);
migrationBuilder.AddColumn<List<string>>(
name: "mentionedLocalUserIds",
table: "hashtag",
type: "character varying(32)[]",
nullable: false);
migrationBuilder.AddColumn<int>(
name: "mentionedLocalUsersCount",
table: "hashtag",
type: "integer",
nullable: false,
defaultValue: 0);
migrationBuilder.AddColumn<List<string>>(
name: "mentionedRemoteUserIds",
table: "hashtag",
type: "character varying(32)[]",
nullable: false);
migrationBuilder.AddColumn<int>(
name: "mentionedRemoteUsersCount",
table: "hashtag",
type: "integer",
nullable: false,
defaultValue: 0);
migrationBuilder.AddColumn<List<string>>(
name: "mentionedUserIds",
table: "hashtag",
type: "character varying(32)[]",
nullable: false);
migrationBuilder.AddColumn<int>(
name: "mentionedUsersCount",
table: "hashtag",
type: "integer",
nullable: false,
defaultValue: 0);
migrationBuilder.CreateIndex(
name: "IX_hashtag_attachedLocalUsersCount",
table: "hashtag",
column: "attachedLocalUsersCount");
migrationBuilder.CreateIndex(
name: "IX_hashtag_attachedRemoteUsersCount",
table: "hashtag",
column: "attachedRemoteUsersCount");
migrationBuilder.CreateIndex(
name: "IX_hashtag_attachedUsersCount",
table: "hashtag",
column: "attachedUsersCount");
migrationBuilder.CreateIndex(
name: "IX_hashtag_mentionedLocalUsersCount",
table: "hashtag",
column: "mentionedLocalUsersCount");
migrationBuilder.CreateIndex(
name: "IX_hashtag_mentionedRemoteUsersCount",
table: "hashtag",
column: "mentionedRemoteUsersCount");
migrationBuilder.CreateIndex(
name: "IX_hashtag_mentionedUsersCount",
table: "hashtag",
column: "mentionedUsersCount");
}
}
}

78
Iceshrimp.Backend/Core/Database/Migrations/DatabaseContextModelSnapshot.cs
View file

@ -1488,72 +1488,6 @@ namespace Iceshrimp.Backend.Core.Database.Migrations
.HasColumnType("character varying(32)")
.HasColumnName("id");
b.Property<List<string>>("AttachedLocalUserIds")
.IsRequired()
.HasColumnType("character varying(32)[]")
.HasColumnName("attachedLocalUserIds");
b.Property<int>("AttachedLocalUsersCount")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasDefaultValue(0)
.HasColumnName("attachedLocalUsersCount");
b.Property<List<string>>("AttachedRemoteUserIds")
.IsRequired()
.HasColumnType("character varying(32)[]")
.HasColumnName("attachedRemoteUserIds");
b.Property<int>("AttachedRemoteUsersCount")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasDefaultValue(0)
.HasColumnName("attachedRemoteUsersCount");
b.Property<List<string>>("AttachedUserIds")
.IsRequired()
.HasColumnType("character varying(32)[]")
.HasColumnName("attachedUserIds");
b.Property<int>("AttachedUsersCount")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasDefaultValue(0)
.HasColumnName("attachedUsersCount");
b.Property<List<string>>("MentionedLocalUserIds")
.IsRequired()
.HasColumnType("character varying(32)[]")
.HasColumnName("mentionedLocalUserIds");
b.Property<int>("MentionedLocalUsersCount")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasDefaultValue(0)
.HasColumnName("mentionedLocalUsersCount");
b.Property<List<string>>("MentionedRemoteUserIds")
.IsRequired()
.HasColumnType("character varying(32)[]")
.HasColumnName("mentionedRemoteUserIds");
b.Property<int>("MentionedRemoteUsersCount")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasDefaultValue(0)
.HasColumnName("mentionedRemoteUsersCount");
b.Property<List<string>>("MentionedUserIds")
.IsRequired()
.HasColumnType("character varying(32)[]")
.HasColumnName("mentionedUserIds");
b.Property<int>("MentionedUsersCount")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasDefaultValue(0)
.HasColumnName("mentionedUsersCount");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(128)
@ -1562,18 +1496,6 @@ namespace Iceshrimp.Backend.Core.Database.Migrations
b.HasKey("Id");
b.HasIndex("AttachedLocalUsersCount");
b.HasIndex("AttachedRemoteUsersCount");
b.HasIndex("AttachedUsersCount");
b.HasIndex("MentionedLocalUsersCount");
b.HasIndex("MentionedRemoteUsersCount");
b.HasIndex("MentionedUsersCount");
b.HasIndex("Name")
.IsUnique();

36
Iceshrimp.Backend/Core/Database/Tables/Hashtag.cs
View file

@ -5,13 +5,7 @@ using Microsoft.EntityFrameworkCore;
namespace Iceshrimp.Backend.Core.Database.Tables;
[Table("hashtag")]
[Index("AttachedRemoteUsersCount")]
[Index("AttachedLocalUsersCount")]
[Index("MentionedLocalUsersCount")]
[Index("MentionedUsersCount")]
[Index("Name", IsUnique = true)]
[Index("MentionedRemoteUsersCount")]
[Index("AttachedUsersCount")]
public class Hashtag : IEntity
{
[Key]
@ -20,34 +14,4 @@ public class Hashtag : IEntity
public string Id { get; set; } = null!;
[Column("name")] [StringLength(128)] public string Name { get; set; } = null!;
[Column("mentionedUserIds", TypeName = "character varying(32)[]")]
public List<string> MentionedUserIds { get; set; } = null!;
[Column("mentionedUsersCount")] public int MentionedUsersCount { get; set; }
[Column("mentionedLocalUserIds", TypeName = "character varying(32)[]")]
public List<string> MentionedLocalUserIds { get; set; } = null!;
[Column("mentionedLocalUsersCount")] public int MentionedLocalUsersCount { get; set; }
[Column("mentionedRemoteUserIds", TypeName = "character varying(32)[]")]
public List<string> MentionedRemoteUserIds { get; set; } = null!;
[Column("mentionedRemoteUsersCount")] public int MentionedRemoteUsersCount { get; set; }
[Column("attachedUserIds", TypeName = "character varying(32)[]")]
public List<string> AttachedUserIds { get; set; } = null!;
[Column("attachedUsersCount")] public int AttachedUsersCount { get; set; }
[Column("attachedLocalUserIds", TypeName = "character varying(32)[]")]
public List<string> AttachedLocalUserIds { get; set; } = null!;
[Column("attachedLocalUsersCount")] public int AttachedLocalUsersCount { get; set; }
[Column("attachedRemoteUserIds", TypeName = "character varying(32)[]")]
public List<string> AttachedRemoteUserIds { get; set; } = null!;
[Column("attachedRemoteUsersCount")] public int AttachedRemoteUsersCount { get; set; }
}

20
Iceshrimp.Backend/Core/Federation/ActivityPub/NoteRenderer.cs
View file

@ -59,14 +59,18 @@ public class NoteRenderer(IOptions<Config.InstanceSection> config, MfmConverter
_ => []
};
var tags = mentions
.Select(mention => new ASMention
{
Name = $"@{mention.Username}@{mention.Host}",
Href = new ASObjectBase(mention.Uri)
})
.Cast<ASTag>()
.ToList();
var tags = note.Tags.Select(tag => new ASHashtag
{
Name = $"#{tag}",
Href = new ASObjectBase($"https://{config.Value.WebDomain}/tags/{tag}")
})
.Concat<ASTag>(mentions.Select(mention => new ASMention
{
Name = $"@{mention.Username}@{mention.Host}",
Href = new ASObjectBase(mention.Uri)
}))
.ToList();
var attachments = note.FileIds.Count > 0
? await db.DriveFiles

12
Iceshrimp.Backend/Core/Federation/ActivityPub/UserRenderer.cs
View file

@ -46,6 +46,15 @@ public class UserRenderer(IOptions<Config.InstanceSection> config, DatabaseConte
? ASActor.Types.Service
: ASActor.Types.Person;
var tags = user.Tags
.Select(tag => new ASHashtag
{
Name = $"#{tag}",
Href = new ASObjectBase($"https://{config.Value.WebDomain}/tags/{tag}")
})
.Cast<ASTag>()
.ToList();
return new ASActor
{
Id = id,
@ -74,7 +83,8 @@ public class UserRenderer(IOptions<Config.InstanceSection> config, DatabaseConte
PublicKey = new ASPublicKey
{
Id = $"{id}#main-key", Owner = new ASObjectBase(id), PublicKey = keypair.PublicKey
}
},
Tags = tags
};
}
}

54
Iceshrimp.Backend/Core/Services/NoteService.cs
View file

@ -77,6 +77,8 @@ public class NoteService(
if ((user.UserSettings?.PrivateMode ?? false) && visibility < Note.NoteVisibility.Followers)
visibility = Note.NoteVisibility.Followers;
var tags = ResolveHashtags(text);
var note = new Note
{
Id = IdHelpers.GenerateSlowflakeId(),
@ -97,7 +99,8 @@ public class NoteService(
Mentions = mentionedUserIds,
VisibleUserIds = visibility == Note.NoteVisibility.Specified ? mentionedUserIds : [],
MentionedRemoteUsers = remoteMentions,
ThreadId = reply?.ThreadId ?? reply?.Id
ThreadId = reply?.ThreadId ?? reply?.Id,
Tags = tags
};
await UpdateNoteCountersAsync(note, true);
@ -213,7 +216,7 @@ public class NoteService(
mentionedLocalUserIds = mentionedLocalUserIds.Except(previousMentionedLocalUserIds).ToList();
note.Text = text;
note.Tags = ResolveHashtags(text);
if (text is not null)
{
@ -396,8 +399,6 @@ public class NoteService(
if (actor.IsSuspended)
throw GracefulException.Forbidden("User is suspended");
//TODO: resolve emoji
var (mentionedUserIds, mentionedLocalUserIds, mentions, remoteMentions, splitDomainMapping) =
await ResolveNoteMentionsAsync(note);
@ -461,6 +462,7 @@ public class NoteService(
}
dbNote.Text = mentionsResolver.ResolveMentions(dbNote.Text, dbNote.UserHost, mentions, splitDomainMapping);
dbNote.Tags = ResolveHashtags(dbNote.Text, note);
}
var sensitive = (note.Sensitive ?? false) || dbNote.Cw != null;
@ -544,6 +546,7 @@ public class NoteService(
}
dbNote.Text = mentionsResolver.ResolveMentions(dbNote.Text, dbNote.UserHost, mentions, splitDomainMapping);
dbNote.Tags = ResolveHashtags(dbNote.Text, note);
}
//TODO: handle updated alt text et al
@ -608,6 +611,49 @@ public class NoteService(
return ResolveNoteMentions(users.Where(p => p != null).Select(p => p!).ToList());
}
private List<string> ResolveHashtags(string? text, ASNote? note = null)
{
List<string> tags = [];
if (text != null)
{
tags = MfmParser.Parse(text)
.SelectMany(p => p.Children.Append(p))
.OfType<MfmHashtagNode>()
.Select(p => p.Hashtag.ToLowerInvariant())
.Select(p => p.Trim('#'))
.Distinct()
.ToList();
}
var extracted = note?.Tags?.OfType<ASHashtag>()
.Select(p => p.Name?.ToLowerInvariant())
.Where(p => p != null)
.Cast<string>()
.Select(p => p.Trim('#'))
.Distinct()
.ToList();
if (extracted != null)
tags.AddRange(extracted);
if (tags.Count == 0) return [];
tags = tags.Distinct().ToList();
_ = followupTaskSvc.ExecuteTask("UpdateHashtagsTable", async provider =>
{
var bgDb = provider.GetRequiredService<DatabaseContext>();
var existing = await bgDb.Hashtags.Where(p => tags.Contains(p.Name)).Select(p => p.Name).ToListAsync();
var dbTags = tags.Except(existing)
.Select(p => new Hashtag { Id = IdHelpers.GenerateSlowflakeId(), Name = p });
await bgDb.AddRangeAsync(dbTags);
await bgDb.SaveChangesAsync();
});
return tags;
}
private MentionQuintuple ResolveNoteMentions(IReadOnlyCollection<User> users)
{
var userIds = users.Select(p => p.Id).Distinct().ToList();

63
Iceshrimp.Backend/Core/Services/UserService.cs
View file

@ -10,6 +10,8 @@ using Iceshrimp.Backend.Core.Extensions;
using Iceshrimp.Backend.Core.Federation.ActivityStreams.Types;
using Iceshrimp.Backend.Core.Helpers;
using Iceshrimp.Backend.Core.Helpers.LibMfm.Conversion;
using Iceshrimp.Backend.Core.Helpers.LibMfm.Parsing;
using Iceshrimp.Backend.Core.Helpers.LibMfm.Types;
using Iceshrimp.Backend.Core.Middleware;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
@ -115,11 +117,6 @@ public class UserService(
var host = AcctToTuple(acct).Host ?? throw new Exception("Host must not be null at this stage");
var emoji = await emojiSvc.ProcessEmojiAsync(actor.Tags?.OfType<ASEmoji>().ToList(), host);
var tags = actor.Tags?.OfType<ASHashtag>()
.Select(p => p.Name?.ToLowerInvariant())
.Where(p => p != null)
.Cast<string>()
.ToList();
var fields = actor.Attachments != null
? await actor.Attachments
@ -134,6 +131,8 @@ public class UserService(
var bio = actor.MkSummary ?? await MfmConverter.FromHtmlAsync(actor.Summary);
var tags = ResolveHashtags(bio, actor);
user = new User
{
Id = IdHelpers.GenerateSlowflakeId(),
@ -261,12 +260,6 @@ public class UserService(
user.Host ??
throw new Exception("User host must not be null at this stage"));
var tags = actor.Tags?.OfType<ASHashtag>()
.Select(p => p.Name?.ToLowerInvariant())
.Where(p => p != null)
.Cast<string>()
.ToList();
var fields = actor.Attachments != null
? await actor.Attachments
.OfType<ASField>()
@ -279,7 +272,6 @@ public class UserService(
: null;
user.Emojis = emoji.Select(p => p.Id).ToList();
user.Tags = tags ?? [];
//TODO: FollowersCount
//TODO: FollowingCount
@ -296,6 +288,8 @@ public class UserService(
user.UserProfile.MentionsResolved = false;
user.Tags = ResolveHashtags(user.UserProfile.Description, actor);
db.Update(user);
await db.SaveChangesAsync();
await processPendingDeletes();
@ -308,6 +302,8 @@ public class UserService(
{
if (user.Host != null) throw new Exception("This method is only valid for local users");
if (user.UserProfile == null) throw new Exception("user.UserProfile must not be null at this stage");
user.Tags = ResolveHashtags(user.UserProfile.Description);
db.Update(user);
db.Update(user.UserProfile);
@ -786,4 +782,47 @@ public class UserService(
return user;
}
private List<string> ResolveHashtags(string? text, ASActor? actor = null)
{
List<string> tags = [];
if (text != null)
{
tags = MfmParser.Parse(text)
.SelectMany(p => p.Children.Append(p))
.OfType<MfmHashtagNode>()
.Select(p => p.Hashtag.ToLowerInvariant())
.Select(p => p.Trim('#'))
.Distinct()
.ToList();
}
var extracted = actor?.Tags?.OfType<ASHashtag>()
.Select(p => p.Name?.ToLowerInvariant())
.Where(p => p != null)
.Cast<string>()
.Select(p => p.Trim('#'))
.Distinct()
.ToList();
if (extracted != null)
tags.AddRange(extracted);
if (tags.Count == 0) return [];
tags = tags.Distinct().ToList();
_ = followupTaskSvc.ExecuteTask("UpdateHashtagsTable", async provider =>
{
var bgDb = provider.GetRequiredService<DatabaseContext>();
var existing = await bgDb.Hashtags.Where(p => tags.Contains(p.Name)).Select(p => p.Name).ToListAsync();
var dbTags = tags.Except(existing)
.Select(p => new Hashtag { Id = IdHelpers.GenerateSlowflakeId(), Name = p });
await bgDb.AddRangeAsync(dbTags);
await bgDb.SaveChangesAsync();
});
return tags;
}
}