Skip to content

Commit 7bc47d6

Browse files
ankitsmt211ankitsmt211
authored
Feat/meta data v2 (#990)
* alter help_thread table to add more meta data * update write operation with extra fields * listener that updates meta data based on thread status * fix error in log messages * method to update tagName on change in db * more suitable name for class and java doc * change name to avoid confusion & update java doc * minor bug that would change status back to active again and log message update * more columns added for metadata * update listener to record added metadata columns * spotless fix * replacing get(0) with getFirst() for list * add routine to settle thread status if was left open * spotless fix * refactor MarkHelpThreadCloseInDBRoutine and changes * update constructor with HelpThreadLifecycleListener instance * using HelpThreadLifecycleListener method to clean up left over threads * increasing scope of handleArchiveStatus to package level * adding logger to MarkHelpThreadCloseInDBRoutine class * document updated param in MarkHelpThreadCloseInDBRoutine constructor * use time of creation/modfication from thread and tags as csv * rely on discord for getting thread status instead of DB * changes * requested changes * update config to include tagsToIgnore * refactor tag update logic * update sql script * add tagsToIgnore list to config * change list * adding data fields that are being collected in privacy policy * updating duration of data storage in privacy policy doc * HelpThreadMetadataPurger will now purge help threads data post 180 days * instead of appending new tags, only new tags are stored now * spotless fix * tags from config sanitized * requested changes * spotless * requested changes * remove unnecessary map * increase routine schedule to 24 hrs
1 parent 9122c16 commit 7bc47d6

File tree

8 files changed

+267
-9
lines changed

8 files changed

+267
-9
lines changed

‎PP.md

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,14 +48,21 @@ The databases may store
4848
* `guild_id` of guilds the **bot** is member of (the unique id of a Discord guild),
4949
* `channel_id` of channels belonging to guilds the **bot** is member of (the unique id of a Discord channel),
5050
* `message_id` of messages send by users in guilds the **bot** is member of (the unique id of a Discord message),
51+
* `participant_count` of no of people who participated in help thread discussions,
52+
* `tags` aka categories to which these help threads belong to,
53+
* `timestamp`s for both when thread was created and closed,
54+
* `message_count` the no of messages that were sent in lifecycle of any help thread
55+
56+
_Note: Help threads are just threads that are created via forum channels, used for anyone to ask questions and get help
57+
in certain problems._
5158

5259
and any combination of those.
5360

5461
For example, **TJ-Bot** may associate your `user_id` with a `message_id` and a `timestamp` for any message that you send in a channel belonging to guilds the **bot** is member of.
5562

5663
**TJ-Bot** may further store data that you explicitly provided for **TJ-Bot** to offer its services. For example the reason of a moderative action when using its moderation commands.
5764

58-
Furthermore, upon utilization of our help service, `user_id`s and `channel_id`s are stored to track when/how many questions a user asks. The data may be stored for up to **30** days.
65+
Furthermore, upon utilization of our help service, `user_id`s and `channel_id`s are stored to track when/how many questions a user asks. The data may be stored for up to **180** days.
5966

6067
The stored data is not linked to any information that is personally identifiable.
6168

‎application/src/main/java/org/togetherjava/tjbot/features/Features.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,9 @@
2929
import org.togetherjava.tjbot.features.help.HelpThreadAutoArchiver;
3030
import org.togetherjava.tjbot.features.help.HelpThreadCommand;
3131
import org.togetherjava.tjbot.features.help.HelpThreadCreatedListener;
32+
import org.togetherjava.tjbot.features.help.HelpThreadLifecycleListener;
3233
import org.togetherjava.tjbot.features.help.HelpThreadMetadataPurger;
34+
import org.togetherjava.tjbot.features.help.MarkHelpThreadCloseInDBRoutine;
3335
import org.togetherjava.tjbot.features.help.PinnedNotificationRemover;
3436
import org.togetherjava.tjbot.features.javamail.RSSHandlerRoutine;
3537
import org.togetherjava.tjbot.features.jshell.JShellCommand;
@@ -113,6 +115,8 @@ public static Collection<Feature> createFeatures(JDA jda, Database database, Con
113115
new CodeMessageHandler(blacklistConfig.special(), jshellEval);
114116
ChatGptService chatGptService = new ChatGptService(config);
115117
HelpSystemHelper helpSystemHelper = new HelpSystemHelper(config, database, chatGptService);
118+
HelpThreadLifecycleListener helpThreadLifecycleListener =
119+
new HelpThreadLifecycleListener(helpSystemHelper, database);
116120

117121
// NOTE The system can add special system relevant commands also by itself,
118122
// hence this list may not necessarily represent the full list of all commands actually
@@ -129,6 +133,7 @@ public static Collection<Feature> createFeatures(JDA jda, Database database, Con
129133
features.add(new HelpThreadActivityUpdater(helpSystemHelper));
130134
features.add(new HelpThreadAutoArchiver(helpSystemHelper));
131135
features.add(new LeftoverBookmarksCleanupRoutine(bookmarksSystem));
136+
features.add(new MarkHelpThreadCloseInDBRoutine(database, helpThreadLifecycleListener));
132137
features.add(new MemberCountDisplayRoutine(config));
133138
features.add(new RSSHandlerRoutine(config, database));
134139

@@ -151,6 +156,7 @@ public static Collection<Feature> createFeatures(JDA jda, Database database, Con
151156
features.add(new GuildLeaveCloseThreadListener(config));
152157
features.add(new LeftoverBookmarksListener(bookmarksSystem));
153158
features.add(new HelpThreadCreatedListener(helpSystemHelper));
159+
features.add(new HelpThreadLifecycleListener(helpSystemHelper, database));
154160

155161
// Message context commands
156162
features.add(new TransferQuestionCommand(config, chatGptService));

‎application/src/main/java/org/togetherjava/tjbot/features/help/HelpSystemHelper.java

Lines changed: 43 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,11 @@
2929
import org.togetherjava.tjbot.features.componentids.ComponentIdInteractor;
3030

3131
import java.awt.Color;
32+
import java.time.Instant;
3233
import java.util.ArrayList;
3334
import java.util.Arrays;
3435
import java.util.Collection;
3536
import java.util.Comparator;
36-
import java.util.HashSet;
3737
import java.util.List;
3838
import java.util.Map;
3939
import java.util.Optional;
@@ -66,6 +66,7 @@ public final class HelpSystemHelper {
6666
private final Set<String> categories;
6767
private final Set<String> threadActivityTagNames;
6868
private final String categoryRoleSuffix;
69+
6970
private final Database database;
7071
private final ChatGptService chatGptService;
7172
private static final int MAX_QUESTION_LENGTH = 200;
@@ -90,7 +91,10 @@ public HelpSystemHelper(Config config, Database database, ChatGptService chatGpt
9091
isHelpForumName = Pattern.compile(helpForumPattern).asMatchPredicate();
9192

9293
List<String> categoriesList = helpConfig.getCategories();
93-
categories = new HashSet<>(categoriesList);
94+
categories = categoriesList.stream()
95+
.map(String::strip)
96+
.map(String::toLowerCase)
97+
.collect(Collectors.toSet());
9498
categoryRoleSuffix = helpConfig.getCategoryRoleSuffix();
9599

96100
Map<String, Integer> categoryToCommonDesc = IntStream.range(0, categoriesList.size())
@@ -104,6 +108,8 @@ public HelpSystemHelper(Config config, Database database, ChatGptService chatGpt
104108
threadActivityTagNames = Arrays.stream(ThreadActivity.values())
105109
.map(ThreadActivity::getTagName)
106110
.collect(Collectors.toSet());
111+
112+
107113
}
108114

109115
/**
@@ -221,11 +227,22 @@ private RestAction<Message> useChatGptFallbackMessage(ThreadChannel threadChanne
221227
}
222228

223229
void writeHelpThreadToDatabase(long authorId, ThreadChannel threadChannel) {
230+
231+
Instant createdAt = threadChannel.getTimeCreated().toInstant();
232+
233+
String appliedTags = threadChannel.getAppliedTags()
234+
.stream()
235+
.filter(this::shouldIgnoreTag)
236+
.map(ForumTag::getName)
237+
.collect(Collectors.joining(","));
238+
224239
database.write(content -> {
225240
HelpThreadsRecord helpThreadsRecord = content.newRecord(HelpThreads.HELP_THREADS)
226241
.setAuthorId(authorId)
227242
.setChannelId(threadChannel.getIdLong())
228-
.setCreatedAt(threadChannel.getTimeCreated().toInstant());
243+
.setCreatedAt(createdAt)
244+
.setTags(appliedTags)
245+
.setTicketStatus(TicketStatus.ACTIVE.val);
229246
if (helpThreadsRecord.update() == 0) {
230247
helpThreadsRecord.insert();
231248
}
@@ -265,7 +282,7 @@ private Optional<ForumTag> getFirstMatchingTagOfChannel(Set<String> tagNamesToMa
265282
ThreadChannel channel) {
266283
return channel.getAppliedTags()
267284
.stream()
268-
.filter(tag -> tagNamesToMatch.contains(tag.getName()))
285+
.filter(tag -> tagNamesToMatch.contains(tag.getName().toLowerCase()))
269286
.min(byCategoryCommonnessAsc);
270287
}
271288

@@ -375,6 +392,17 @@ public String getTagName() {
375392
}
376393
}
377394

395+
enum TicketStatus {
396+
ARCHIVED(0),
397+
ACTIVE(1);
398+
399+
final int val;
400+
401+
TicketStatus(int val) {
402+
this.val = val;
403+
}
404+
}
405+
378406
Optional<Long> getAuthorByHelpThreadId(final long channelId) {
379407

380408
logger.debug("Looking for thread-record using channel ID: {}", channelId);
@@ -384,4 +412,15 @@ Optional<Long> getAuthorByHelpThreadId(final long channelId) {
384412
.where(HelpThreads.HELP_THREADS.CHANNEL_ID.eq(channelId))
385413
.fetchOptional(HelpThreads.HELP_THREADS.AUTHOR_ID));
386414
}
415+
416+
417+
/**
418+
* will be used to filter a tag based on categories config
419+
*
420+
* @param tag applied tag
421+
* @return boolean result whether to ignore this tag or not
422+
*/
423+
boolean shouldIgnoreTag(ForumTag tag) {
424+
return this.categories.contains(tag.getName().toLowerCase());
425+
}
387426
}

‎application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import net.dv8tion.jda.api.events.message.MessageReceivedEvent;
1515
import net.dv8tion.jda.api.hooks.ListenerAdapter;
1616
import net.dv8tion.jda.api.requests.RestAction;
17-
import org.jetbrains.annotations.NotNull;
1817

1918
import org.togetherjava.tjbot.features.EventReceiver;
2019
import org.togetherjava.tjbot.features.UserInteractionType;
@@ -58,7 +57,7 @@ public HelpThreadCreatedListener(HelpSystemHelper helper) {
5857
}
5958

6059
@Override
61-
public void onMessageReceived(@NotNull MessageReceivedEvent event) {
60+
public void onMessageReceived(MessageReceivedEvent event) {
6261
if (event.isFromThread()) {
6362
ThreadChannel threadChannel = event.getChannel().asThreadChannel();
6463
Channel parentChannel = threadChannel.getParentChannel();

‎application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadLifecycleListener.java

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
package org.togetherjava.tjbot.features.help;
2+
3+
import net.dv8tion.jda.api.entities.channel.concrete.ThreadChannel;
4+
import net.dv8tion.jda.api.entities.channel.forums.ForumTag;
5+
import net.dv8tion.jda.api.events.channel.update.ChannelUpdateAppliedTagsEvent;
6+
import net.dv8tion.jda.api.events.channel.update.ChannelUpdateArchivedEvent;
7+
import net.dv8tion.jda.api.hooks.ListenerAdapter;
8+
import org.slf4j.Logger;
9+
import org.slf4j.LoggerFactory;
10+
11+
import org.togetherjava.tjbot.db.Database;
12+
import org.togetherjava.tjbot.features.EventReceiver;
13+
14+
import java.time.Instant;
15+
import java.util.List;
16+
import java.util.stream.Collectors;
17+
18+
import static org.togetherjava.tjbot.db.generated.tables.HelpThreads.HELP_THREADS;
19+
20+
/**
21+
* Listens for help thread events after creation of thread. Updates metadata based on those events
22+
* in database.
23+
*/
24+
public final class HelpThreadLifecycleListener extends ListenerAdapter implements EventReceiver {
25+
private static final Logger logger = LoggerFactory.getLogger(HelpThreadLifecycleListener.class);
26+
private final HelpSystemHelper helper;
27+
private final Database database;
28+
29+
/**
30+
* Creates a new instance.
31+
*
32+
* @param helper to work with the help threads
33+
* @param database the database to store help thread metadata in
34+
*/
35+
public HelpThreadLifecycleListener(HelpSystemHelper helper, Database database) {
36+
this.helper = helper;
37+
this.database = database;
38+
}
39+
40+
@Override
41+
public void onChannelUpdateArchived(ChannelUpdateArchivedEvent event) {
42+
ThreadChannel threadChannel = event.getChannel().asThreadChannel();
43+
44+
if (!helper.isHelpForumName(threadChannel.getParentChannel().getName())) {
45+
return;
46+
}
47+
handleThreadStatus(threadChannel);
48+
}
49+
50+
@Override
51+
public void onChannelUpdateAppliedTags(ChannelUpdateAppliedTagsEvent event) {
52+
ThreadChannel threadChannel = event.getChannel().asThreadChannel();
53+
54+
if (!helper.isHelpForumName(threadChannel.getParentChannel().getName())
55+
|| shouldIgnoreUpdatedTagEvent(event)) {
56+
return;
57+
}
58+
59+
60+
String newlyAppliedTagsOnly = event.getNewTags()
61+
.stream()
62+
.filter(helper::shouldIgnoreTag)
63+
.map(ForumTag::getName)
64+
.collect(Collectors.joining(","));
65+
66+
67+
long threadId = threadChannel.getIdLong();
68+
69+
handleTagsUpdate(threadId, newlyAppliedTagsOnly);
70+
}
71+
72+
private void handleThreadStatus(ThreadChannel threadChannel) {
73+
Instant closedAt = threadChannel.getTimeArchiveInfoLastModified().toInstant();
74+
long threadId = threadChannel.getIdLong();
75+
boolean isArchived = threadChannel.isArchived();
76+
77+
if (isArchived) {
78+
handleArchiveStatus(closedAt, threadChannel);
79+
return;
80+
}
81+
82+
updateThreadStatusToActive(threadId);
83+
}
84+
85+
void handleArchiveStatus(Instant closedAt, ThreadChannel threadChannel) {
86+
long threadId = threadChannel.getIdLong();
87+
int messageCount = threadChannel.getMessageCount();
88+
int participantsExceptAuthor = threadChannel.getMemberCount() - 1;
89+
90+
database.write(context -> context.update(HELP_THREADS)
91+
.set(HELP_THREADS.CLOSED_AT, closedAt)
92+
.set(HELP_THREADS.TICKET_STATUS, HelpSystemHelper.TicketStatus.ARCHIVED.val)
93+
.set(HELP_THREADS.MESSAGE_COUNT, messageCount)
94+
.set(HELP_THREADS.PARTICIPANTS, participantsExceptAuthor)
95+
.where(HELP_THREADS.CHANNEL_ID.eq(threadId))
96+
.execute());
97+
98+
logger.info("Thread with id: {}, updated to archived status in database", threadId);
99+
}
100+
101+
private void updateThreadStatusToActive(long threadId) {
102+
database.write(context -> context.update(HELP_THREADS)
103+
.set(HELP_THREADS.TICKET_STATUS, HelpSystemHelper.TicketStatus.ACTIVE.val)
104+
.where(HELP_THREADS.CHANNEL_ID.eq(threadId))
105+
.execute());
106+
107+
logger.info("Thread with id: {}, updated to active status in database", threadId);
108+
}
109+
110+
private void handleTagsUpdate(long threadId, String updatedTag) {
111+
database.write(context -> context.update(HELP_THREADS)
112+
.set(HELP_THREADS.TAGS, updatedTag)
113+
.where(HELP_THREADS.CHANNEL_ID.eq(threadId))
114+
.execute());
115+
116+
logger.info("Updated tag for thread with id: {} in database", threadId);
117+
}
118+
119+
/**
120+
* will ignore updated tag event if all new tags belong to the categories config
121+
*
122+
* @param event updated tags event
123+
* @return boolean
124+
*/
125+
private boolean shouldIgnoreUpdatedTagEvent(ChannelUpdateAppliedTagsEvent event) {
126+
List<ForumTag> newTags =
127+
event.getNewTags().stream().filter(helper::shouldIgnoreTag).toList();
128+
return newTags.isEmpty();
129+
}
130+
}

‎application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadMetadataPurger.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
public class HelpThreadMetadataPurger implements Routine {
1919
private final Database database;
2020
private static final Logger logger = LoggerFactory.getLogger(HelpThreadMetadataPurger.class);
21-
private static final Period DELETE_MESSAGE_RECORDS_AFTER = Period.ofDays(30);
21+
private static final Period DELETE_MESSAGE_RECORDS_AFTER = Period.ofDays(180);
2222

2323
/**
2424
* Creates a new instance.
@@ -31,7 +31,7 @@ public HelpThreadMetadataPurger(Database database) {
3131

3232
@Override
3333
public Schedule createSchedule() {
34-
return new Schedule(ScheduleMode.FIXED_RATE, 0, 4, TimeUnit.HOURS);
34+
return new Schedule(ScheduleMode.FIXED_RATE, 0, 1, TimeUnit.DAYS);
3535
}
3636

3737
@Override

0 commit comments

Comments
 (0)