[pull] master from mattermost:master

e2e-tests/cypress/tests/support/api/on_prem_default_config.json

@@ -484,6 +484,7 @@
         "Password": "changeme",
         "EnableIndexing": false,
         "EnableSearching": false,
+        "EnableCJKAnalyzers": false,
         "EnableAutocomplete": false,
         "Sniff": false,
         "PostIndexReplicas": 1,

e2e-tests/playwright/lib/src/server/default_config.ts

@@ -627,6 +627,7 @@ const defaultServerConfig: AdminConfig = {
         Password: 'changeme',
         EnableIndexing: false,
         EnableSearching: false,
+        EnableCJKAnalyzers: false,
         EnableAutocomplete: false,
         Sniff: true,
         PostIndexReplicas: 1,

server/.golangci.yml

@@ -112,6 +112,7 @@ linters:
           channels/store/localcachelayer/webhook_layer_test.go|\
           channels/store/retrylayer/retrylayer_test.go|\
           channels/store/searchtest/channel_layer.go|\
+          channels/store/searchtest/cjk_plugins.go|\
           channels/store/searchtest/file_info_layer.go|\
           channels/store/searchtest/helper.go|\
           channels/store/searchtest/post_layer.go|\

server/build/Dockerfile.elasticsearch

@@ -0,0 +1,4 @@
+ARG ELASTICSEARCH_VERSION=8.9.0
+FROM mattermostdevelopment/mattermost-elasticsearch:${ELASTICSEARCH_VERSION}
+
+RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch analysis-nori analysis-kuromoji analysis-smartcn

server/build/Dockerfile.opensearch

@@ -1,4 +1,4 @@
 ARG OPENSEARCH_VERSION=2.7.0
 FROM opensearchproject/opensearch:$OPENSEARCH_VERSION
 
-RUN /usr/share/opensearch/bin/opensearch-plugin install analysis-icu
+RUN /usr/share/opensearch/bin/opensearch-plugin install analysis-icu analysis-nori analysis-kuromoji analysis-smartcn

server/build/docker-compose.common.yml

@@ -47,7 +47,9 @@ services:
       LDAP_DOMAIN: "mm.test.com"
       LDAP_ADMIN_PASSWORD: "mostest"
   elasticsearch:
-    image: "mattermostdevelopment/mattermost-elasticsearch:8.9.0"
+    build:
+      context: .
+      dockerfile: ./Dockerfile.elasticsearch
     networks:
       - mm-test
     environment:

server/channels/app/content_flagging.go

@@ -551,7 +551,7 @@ func (a *App) PermanentDeleteFlaggedPost(rctx request.CTX, actionRequest *model.
 	if jsonErr != nil {
 		return model.NewAppError("PermanentlyRemoveFlaggedPost", "app.content_flagging.permanently_delete.marshal_comment.app_error", nil, "", http.StatusInternalServerError).Wrap(jsonErr)
 	}
-	// Storing marshalled content into RawMessage to ensure proper escaping of special characters and prevent
+	// Storing marshaled content into RawMessage to ensure proper escaping of special characters and prevent
 	// generating unsafe JSON values
 	commentJsonValue := json.RawMessage(commentBytes)
 
@@ -565,61 +565,11 @@ func (a *App) PermanentDeleteFlaggedPost(rctx request.CTX, actionRequest *model.
 		return model.NewAppError("PermanentlyRemoveFlaggedPost", "api.content_flagging.error.post_not_in_progress", nil, "", http.StatusBadRequest)
 	}
 
-	editHistories, appErr := a.GetEditHistoryForPost(flaggedPost.Id)
-	if appErr != nil {
-		if appErr.StatusCode != http.StatusNotFound {
-			rctx.Logger().Error("PermanentlyRemoveFlaggedPost: Failed to get edit history for flaggedPost", mlog.Err(appErr), mlog.String("post_id", flaggedPost.Id))
-		}
-	}
-
-	for _, editHistory := range editHistories {
-		if filesDeleteAppErr := a.PermanentDeleteFilesByPost(rctx, editHistory.Id); filesDeleteAppErr != nil {
-			rctx.Logger().Error("PermanentlyRemoveFlaggedPost: Failed to permanently delete files for one of the edit history posts", mlog.Err(filesDeleteAppErr), mlog.String("post_id", editHistory.Id))
-		}
-
-		if deletePostAppErr := a.PermanentDeletePost(rctx, editHistory.Id, reviewerId); deletePostAppErr != nil {
-			rctx.Logger().Error("PermanentlyRemoveFlaggedPost: Failed to permanently delete one of the edit history posts", mlog.Err(deletePostAppErr), mlog.String("post_id", editHistory.Id))
-		}
-	}
-
-	if filesDeleteAppErr := a.PermanentDeleteFilesByPost(rctx, flaggedPost.Id); filesDeleteAppErr != nil {
-		rctx.Logger().Error("PermanentlyRemoveFlaggedPost: Failed to permanently delete files for the flaggedPost", mlog.Err(filesDeleteAppErr), mlog.String("post_id", flaggedPost.Id))
-	}
-
-	if err := a.DeletePriorityForPost(flaggedPost.Id); err != nil {
-		rctx.Logger().Error("PermanentlyRemoveFlaggedPost: Failed to delete flaggedPost priority for the flaggedPost", mlog.Err(err), mlog.String("post_id", flaggedPost.Id))
-	}
-
-	if err := a.Srv().Store().PostAcknowledgement().DeleteAllForPost(flaggedPost.Id); err != nil {
-		rctx.Logger().Error("PermanentlyRemoveFlaggedPost: Failed to delete flaggedPost acknowledgements for the flaggedPost", mlog.Err(err), mlog.String("post_id", flaggedPost.Id))
-	}
-
-	if err := a.Srv().Store().Post().DeleteAllPostRemindersForPost(flaggedPost.Id); err != nil {
-		rctx.Logger().Error("PermanentlyRemoveFlaggedPost: Failed to delete flaggedPost reminders for the flaggedPost", mlog.Err(err), mlog.String("post_id", flaggedPost.Id))
-	}
-
-	scrubPost(flaggedPost)
-	_, err := a.Srv().Store().Post().Overwrite(rctx, flaggedPost)
-	if err != nil {
-		return model.NewAppError("PermanentlyRemoveFlaggedPost", "app.content_flagging.permanently_delete.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
-	}
-
-	contentReviewBot, appErr := a.getContentReviewBot(rctx)
+	appErr = a.PermanentDeletePostDataRetainStub(rctx, flaggedPost, reviewerId)
 	if appErr != nil {
 		return appErr
 	}
 
-	// If the post is not already deleted, delete it now.
-	// This handles the case when "Hide message from channel while it is being reviewed" setting is set to false when the post was flagged.
-	if flaggedPost.DeleteAt == 0 {
-		// DeletePost is called to care of WebSocket events, cache invalidation, search index removal,
-		// persistent notification removal and other cleanup tasks that need to happen on post deletion.
-		_, appErr = a.DeletePost(rctx, flaggedPost.Id, contentReviewBot.UserId)
-		if appErr != nil {
-			return appErr
-		}
-	}
-
 	groupId, appErr := a.ContentFlaggingGroupId()
 	if appErr != nil {
 		return appErr
@@ -654,7 +604,7 @@ func (a *App) PermanentDeleteFlaggedPost(rctx request.CTX, actionRequest *model.
 		},
 	}
 
-	_, err = a.Srv().propertyAccessService.CreatePropertyValues(anonymousCallerId, propertyValues)
+	_, err := a.Srv().propertyAccessService.CreatePropertyValues(anonymousCallerId, propertyValues)
 	if err != nil {
 		return model.NewAppError("PermanentlyRemoveFlaggedPost", "app.content_flagging.create_property_values.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
 	}
@@ -685,6 +635,69 @@ func (a *App) PermanentDeleteFlaggedPost(rctx request.CTX, actionRequest *model.
 	return nil
 }
 
+func (a *App) PermanentDeletePostDataRetainStub(rctx request.CTX, post *model.Post, deleteByID string) *model.AppError {
+	// when a post is removed, the following things need to be done
+	// 1. Hard delete corresponding file infos - covered
+	// 2. Hard delete file infos associated to post's edit history - NA
+	// 3. Hard delete post's edit history - NA
+	// 4. Hard delete the files from file storage - covered
+	// 5. Hard delete post's priority data - missing
+	// 6. Hard delete post's post acknowledgements - missing
+	// 7. Hard delete post reminders - missing
+	// 8. Scrub the post's content - message, props - missing
+
+	editHistories, appErr := a.GetEditHistoryForPost(post.Id)
+	if appErr != nil {
+		if appErr.StatusCode != http.StatusNotFound {
+			rctx.Logger().Error("PermanentDeletePostDataRetainStub: Failed to get edit history for post", mlog.Err(appErr), mlog.String("post_id", post.Id))
+		}
+	}
+
+	for _, editHistory := range editHistories {
+		if deletePostAppErr := a.PermanentDeletePost(rctx, editHistory.Id, deleteByID); deletePostAppErr != nil {
+			rctx.Logger().Error("PermanentDeletePostDataRetainStub: Failed to permanently delete one of the edit history posts", mlog.Err(deletePostAppErr), mlog.String("post_id", editHistory.Id))
+		}
+	}
+
+	if filesDeleteAppErr := a.PermanentDeleteFilesByPost(rctx, post.Id); filesDeleteAppErr != nil {
+		rctx.Logger().Error("PermanentDeletePostDataRetainStub: Failed to permanently delete files for the post", mlog.Err(filesDeleteAppErr), mlog.String("post_id", post.Id))
+	}
+
+	if err := a.DeletePriorityForPost(post.Id); err != nil {
+		rctx.Logger().Error("PermanentDeletePostDataRetainStub: Failed to delete post priority for the post", mlog.Err(err), mlog.String("post_id", post.Id))
+	}
+
+	if err := a.Srv().Store().PostAcknowledgement().DeleteAllForPost(post.Id); err != nil {
+		rctx.Logger().Error("PermanentDeletePostDataRetainStub: Failed to delete post acknowledgements for the post", mlog.Err(err), mlog.String("post_id", post.Id))
+	}
+
+	if err := a.Srv().Store().Post().DeleteAllPostRemindersForPost(post.Id); err != nil {
+		rctx.Logger().Error("PermanentDeletePostDataRetainStub: Failed to delete post reminders for the post", mlog.Err(err), mlog.String("post_id", post.Id))
+	}
+
+	if err := a.Srv().Store().Post().PermanentDeleteAssociatedData([]string{post.Id}); err != nil {
+		rctx.Logger().Error("PermanentDeletePostDataRetainStub: Failed to permanently delete associated data for the post", mlog.Err(err), mlog.String("post_id", post.Id))
+	}
+
+	scrubPost(post)
+	_, err := a.Srv().Store().Post().Overwrite(rctx, post)
+	if err != nil {
+		rctx.Logger().Error("PermanentDeletePostDataRetainStub: Failed to scrub post content", mlog.Err(err), mlog.String("post_id", post.Id))
+	}
+
+	// If the post is not already deleted, delete it now.
+	if post.DeleteAt == 0 {
+		// DeletePost is called to care of WebSocket events, cache invalidation, search index removal,
+		// persistent notification removal and other cleanup tasks that need to happen on post deletion.
+		_, appErr = a.DeletePost(rctx, post.Id, deleteByID)
+		if appErr != nil {
+			return appErr
+		}
+	}
+
+	return nil
+}
+
 func (a *App) KeepFlaggedPost(rctx request.CTX, actionRequest *model.FlagContentActionRequest, reviewerId string, flaggedPost *model.Post) *model.AppError {
 	// for keeping a flagged flaggedPost we need to-
 	// 1. Undelete the flaggedPost if it was deleted, that's it
@@ -808,11 +821,17 @@ func (a *App) KeepFlaggedPost(rctx request.CTX, actionRequest *model.FlagContent
 }
 
 func scrubPost(post *model.Post) {
-	post.Message = "*Content deleted as part of Content Flagging review process*"
+	if post.Type == model.PostTypeBurnOnRead {
+		post.Message = "*Content deleted as part of burning the post*"
+	} else {
+		post.Message = "*Content deleted as part of Content Flagging review process*"
+	}
+
 	post.MessageSource = post.Message
 	post.Hashtags = ""
 	post.Metadata = nil
 	post.FileIds = []string{}
+	post.UpdateAt = model.GetMillis()
 	post.SetProps(make(map[string]any))
 }
 

server/channels/app/post.go

@@ -1694,10 +1694,6 @@ func (a *App) DeletePost(rctx request.CTX, postID, deleteByID string) (*model.Po
 		return nil, model.NewAppError("DeletePost", "app.post.get.app_error", nil, "", http.StatusBadRequest).Wrap(err)
 	}
 
-	if post.Type == model.PostTypeBurnOnRead {
-		return nil, a.PermanentDeletePost(rctx, postID, deleteByID)
-	}
-
 	channel, appErr := a.GetChannel(rctx, post.ChannelId)
 	if appErr != nil {
 		return nil, appErr
@@ -3703,7 +3699,7 @@ func (a *App) BurnPost(rctx request.CTX, post *model.Post, userID string, connec
 
 	// If user is the author, permanently delete the post
 	if post.UserId == userID {
-		return a.PermanentDeletePost(rctx, post.Id, userID)
+		return a.PermanentDeletePostDataRetainStub(rctx, post, userID)
 	}
 
 	// If not the author, check read receipt

server/channels/app/post_helpers.go

@@ -372,6 +372,10 @@ func (a *App) revealBurnOnReadPostsForUser(rctx request.CTX, postList *model.Pos
 	}
 
 	for _, post := range postList.BurnOnReadPosts {
+		if post.DeleteAt > 0 {
+			continue
+		}
+
 		// If user is the author, reveal the post with recipients
 		if post.UserId == userID {
 			if err := a.revealPostForAuthor(rctx, postList, post); err != nil {

server/channels/app/post_helpers_test.go

@@ -425,3 +425,39 @@ func Test_getInaccessibleRange(t *testing.T) {
 		})
 	}
 }
+
+func TestRevealBurnOnReadPostsForUser(t *testing.T) {
+	th := Setup(t).InitBasic(t)
+
+	// Enable BurnOnRead feature
+	th.App.Srv().SetLicense(model.NewTestLicenseSKU(model.LicenseShortSkuEnterpriseAdvanced))
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		cfg.FeatureFlags.BurnOnRead = true
+		cfg.ServiceSettings.EnableBurnOnRead = model.NewPointer(true)
+	})
+
+	t.Run("skips deleted burn-on-read post", func(t *testing.T) {
+		deletedPost := &model.Post{
+			Id:        model.NewId(),
+			UserId:    th.BasicUser.Id,
+			ChannelId: th.BasicChannel.Id,
+			Message:   "deleted burn on read message",
+			Type:      model.PostTypeBurnOnRead,
+			DeleteAt:  model.GetMillis(),
+			CreateAt:  model.GetMillis(),
+		}
+
+		postList := model.NewPostList()
+		postList.AddPost(deletedPost)
+
+		resultList, appErr := th.App.revealBurnOnReadPostsForUser(th.Context, postList, th.BasicUser2.Id)
+
+		require.Nil(t, appErr)
+		require.NotNil(t, resultList)
+		// The deleted post should remain in BurnOnReadPosts but not be processed
+		assert.Contains(t, resultList.BurnOnReadPosts, deletedPost.Id)
+		// Verify the post was not modified (still has DeleteAt set)
+		assert.Equal(t, deletedPost.DeleteAt, resultList.BurnOnReadPosts[deletedPost.Id].DeleteAt)
+		assert.Equal(t, deletedPost.Message, resultList.BurnOnReadPosts[deletedPost.Id].Message)
+	})
+}

server/channels/app/post_test.go

@@ -5367,3 +5367,58 @@ func TestGetFlaggedPostsWithExpiredBurnOnRead(t *testing.T) {
 		require.Greater(t, post.Metadata.ExpireAt, model.GetMillis())
 	})
 }
+
+func TestGetBurnOnReadPost(t *testing.T) {
+	t.Run("success - temporary post found", func(t *testing.T) {
+		th := Setup(t).InitBasic(t)
+
+		post := &model.Post{
+			Id:        model.NewId(),
+			ChannelId: th.BasicChannel.Id,
+			UserId:    th.BasicUser.Id,
+			Message:   "placeholder message",
+			FileIds:   model.StringArray{"file1"},
+			Type:      model.PostTypeBurnOnRead,
+		}
+
+		temporaryPost := &model.TemporaryPost{
+			ID:       post.Id,
+			Type:     model.PostTypeBurnOnRead,
+			ExpireAt: model.GetMillis() + 3600000,
+			Message:  "actual secret message",
+			FileIDs:  model.StringArray{"file2", "file3"},
+		}
+
+		_, err := th.App.Srv().Store().TemporaryPost().Save(th.Context, temporaryPost)
+		require.NoError(t, err)
+
+		resultPost, appErr := th.App.getBurnOnReadPost(th.Context, post)
+
+		require.Nil(t, appErr)
+		require.NotNil(t, resultPost)
+		assert.Equal(t, temporaryPost.Message, resultPost.Message)
+		assert.Equal(t, temporaryPost.FileIDs, resultPost.FileIds)
+		// Ensure original post is not modified
+		assert.Equal(t, "placeholder message", post.Message)
+		assert.Equal(t, model.StringArray{"file1"}, post.FileIds)
+	})
+
+	t.Run("temporary post not found - returns app error", func(t *testing.T) {
+		th := Setup(t).InitBasic(t)
+
+		post := &model.Post{
+			Id:        model.NewId(),
+			ChannelId: th.BasicChannel.Id,
+			UserId:    th.BasicUser.Id,
+			Message:   "placeholder message",
+			Type:      model.PostTypeBurnOnRead,
+		}
+
+		resultPost, appErr := th.App.getBurnOnReadPost(th.Context, post)
+
+		require.NotNil(t, appErr)
+		require.Nil(t, resultPost)
+		assert.Equal(t, "app.post.get_post.app_error", appErr.Id)
+		assert.Equal(t, http.StatusInternalServerError, appErr.StatusCode)
+	})
+}

server/channels/app/scheduled_post_job.go

@@ -19,7 +19,7 @@ import (
 
 const (
 	getPendingScheduledPostsPageSize = 100
-	scheduledPostBatchWaitTime       = 1 * time.Second
+	scheduledPostBatchWaitTime       = 100 * time.Millisecond
 )
 
 func (a *App) ProcessScheduledPosts(rctx request.CTX) {