feat(send_queue): Implement sending of MSC4274 galleries

Signed-off-by: Johannes Marbach <[email protected]>

Author: Johennes

crates/matrix-sdk-base/src/store/mod.rs

@@ -72,7 +72,7 @@ mod send_queue;
 #[cfg(any(test, feature = "testing"))]
 pub use self::integration_tests::StateStoreIntegrationTests;
 #[cfg(feature = "unstable-msc4274")]
-pub use self::send_queue::AccumulatedSentMediaInfo;
+pub use self::send_queue::{AccumulatedSentMediaInfo, FinishGalleryItemInfo};
 pub use self::{
     memory_store::MemoryStore,
     send_queue::{

crates/matrix-sdk-base/src/store/send_queue.rs

@@ -258,6 +258,19 @@ pub enum DependentQueuedRequestKind {
         /// Information about the thumbnail, if present.
         thumbnail_info: Option<FinishUploadThumbnailInfo>,
     },
+
+    /// Finish a gallery upload by updating references to the media cache and
+    /// sending the final gallery event with the remote MXC URIs.
+    #[cfg(feature = "unstable-msc4274")]
+    FinishGallery {
+        /// Local echo for the event (containing the local MXC URIs).
+        ///
+        /// `Box` the local echo so that it reduces the size of the whole enum.
+        local_echo: Box<RoomMessageEventContent>,
+
+        /// Metadata about the gallery items
+        item_infos: Vec<FinishGalleryItemInfo>,
+    },
 }
 
 /// If parent_is_thumbnail_upload is missing, we assume the request is for a
@@ -285,6 +298,17 @@ pub struct FinishUploadThumbnailInfo {
     pub height: Option<UInt>,
 }
 
+#[cfg(feature = "unstable-msc4274")]
+#[derive(Clone, Debug, Serialize, Deserialize)]
+/// Detailed record about a file and thumbnail used when finishing a gallery
+/// upload.
+pub struct FinishGalleryItemInfo {
+    /// Transaction id for the file upload.
+    pub file_upload: OwnedTransactionId,
+    /// Information about the thumbnail, if present.
+    pub thumbnail_info: Option<FinishUploadThumbnailInfo>,
+}
+
 /// A transaction id identifying a [`DependentQueuedRequest`] rather than its
 /// parent [`QueuedRequest`].
 ///
@@ -442,6 +466,11 @@ impl DependentQueuedRequest {
                 // This one graduates into a new media event.
                 true
             }
+            #[cfg(feature = "unstable-msc4274")]
+            DependentQueuedRequestKind::FinishGallery { .. } => {
+                // This one graduates into a new gallery event.
+                true
+            }
         }
     }
 }

crates/matrix-sdk/CHANGELOG.md

@@ -8,6 +8,10 @@ All notable changes to this project will be documented in this file.
 
 ### Features
 
+- `RoomSendQueue::send_gallery` has been added to allow sending MSC4274-style media galleries
+  via the send queue under the `unstable-msc4274` feature.
+  ([#4977](https://github.com/matrix-org/matrix-rust-sdk/pull/4977))
+
 ### Bug fixes
 
 ### Refactor

crates/matrix-sdk/Cargo.toml

@@ -51,7 +51,7 @@ docsrs = ["e2e-encryption", "sqlite", "indexeddb", "sso-login", "qrcode"]
 experimental-share-history-on-invite = []
 
 # Add support for inline media galleries via msgtypes
-unstable-msc4274 = ["matrix-sdk-base/unstable-msc4274"]
+unstable-msc4274 = ["ruma/unstable-msc4274", "matrix-sdk-base/unstable-msc4274"]
 
 [dependencies]
 anyhow = { workspace = true, optional = true }

crates/matrix-sdk/src/attachment.rs

@@ -276,3 +276,95 @@ impl AttachmentConfig {
         self
     }
 }
+
+/// Configuration for sending a gallery.
+#[cfg(feature = "unstable-msc4274")]
+#[derive(Debug, Default)]
+pub struct GalleryConfig {
+    pub(crate) txn_id: Option<OwnedTransactionId>,
+    pub(crate) caption: Option<String>,
+    pub(crate) formatted_caption: Option<FormattedBody>,
+    pub(crate) mentions: Option<Mentions>,
+    pub(crate) reply: Option<Reply>,
+}
+
+#[cfg(feature = "unstable-msc4274")]
+impl GalleryConfig {
+    /// Create a new empty `GalleryConfig`.
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Set the transaction ID to send.
+    ///
+    /// # Arguments
+    ///
+    /// * `txn_id` - A unique ID that can be attached to a `MessageEvent` held
+    ///   in its unsigned field as `transaction_id`. If not given, one is
+    ///   created for the message.
+    #[must_use]
+    pub fn txn_id(mut self, txn_id: &TransactionId) -> Self {
+        self.txn_id = Some(txn_id.to_owned());
+        self
+    }
+
+    /// Set the optional caption
+    ///
+    /// # Arguments
+    ///
+    /// * `caption` - The optional caption
+    pub fn caption(mut self, caption: Option<String>) -> Self {
+        self.caption = caption;
+        self
+    }
+
+    /// Set the optional formatted caption
+    ///
+    /// # Arguments
+    ///
+    /// * `formatted_caption` - The optional formatted caption
+    pub fn formatted_caption(mut self, formatted_caption: Option<FormattedBody>) -> Self {
+        self.formatted_caption = formatted_caption;
+        self
+    }
+
+    /// Set the mentions of the message.
+    ///
+    /// # Arguments
+    ///
+    /// * `mentions` - The mentions of the message
+    pub fn mentions(mut self, mentions: Option<Mentions>) -> Self {
+        self.mentions = mentions;
+        self
+    }
+
+    /// Set the reply information of the message.
+    ///
+    /// # Arguments
+    ///
+    /// * `reply` - The reply information of the message
+    pub fn reply(mut self, reply: Option<Reply>) -> Self {
+        self.reply = reply;
+        self
+    }
+}
+
+#[cfg(feature = "unstable-msc4274")]
+#[derive(Debug)]
+/// Metadata for a gallery item
+pub struct GalleryItemInfo {
+    /// The filename
+    pub filename: String,
+    /// The mime type
+    pub content_type: mime::Mime,
+    /// The binary data
+    pub data: Vec<u8>,
+    /// The attachment info
+    pub attachment_info: AttachmentInfo,
+    /// The caption
+    pub caption: Option<String>,
+    /// The formatted caption
+    pub formatted_caption: Option<FormattedBody>,
+    /// The thumbnail
+    pub thumbnail: Option<Thumbnail>,
+}

crates/matrix-sdk/src/room/edit.rs

@@ -269,6 +269,12 @@ pub(crate) fn update_media_caption(
             event.formatted = formatted_caption;
             true
         }
+        #[cfg(feature = "unstable-msc4274")]
+        MessageType::Gallery(event) => {
+            event.body = caption.unwrap_or_default();
+            event.formatted = formatted_caption;
+            true
+        }
         MessageType::Image(event) => {
             set_caption!(event, caption);
             event.formatted = formatted_caption;

crates/matrix-sdk/src/room/mod.rs

@@ -56,6 +56,8 @@ use matrix_sdk_common::{
 };
 use mime::Mime;
 use reply::Reply;
+#[cfg(feature = "unstable-msc4274")]
+use ruma::events::room::message::GalleryItemType;
 #[cfg(feature = "e2e-encryption")]
 use ruma::events::{
     room::encrypted::OriginalSyncRoomEncryptedEvent, AnySyncMessageLikeEvent, AnySyncTimelineEvent,
@@ -2160,7 +2162,7 @@ impl Room {
         }
 
         let content = self
-            .make_attachment_event(
+            .make_message_event(
                 self.make_attachment_type(
                     content_type,
                     filename,
@@ -2276,7 +2278,7 @@ impl Room {
 
     /// Creates the [`RoomMessageEventContent`] based on the message type,
     /// mentions and reply information.
-    pub(crate) async fn make_attachment_event(
+    pub(crate) async fn make_message_event(
         &self,
         msg_type: MessageType,
         mentions: Option<Mentions>,
@@ -2294,6 +2296,99 @@ impl Room {
         Ok(content)
     }
 
+    /// Creates the inner [`GalleryItemType`] for an already-uploaded media file
+    /// provided by its source.
+    #[cfg(feature = "unstable-msc4274")]
+    #[allow(clippy::too_many_arguments)]
+    pub(crate) fn make_gallery_item_type(
+        &self,
+        content_type: &Mime,
+        filename: String,
+        source: MediaSource,
+        caption: Option<String>,
+        formatted_caption: Option<FormattedBody>,
+        info: Option<AttachmentInfo>,
+        thumbnail: Option<(MediaSource, Box<ThumbnailInfo>)>,
+    ) -> GalleryItemType {
+        // If caption is set, use it as body, and filename as the file name; otherwise,
+        // body is the filename, and the filename is not set.
+        // https://github.com/matrix-org/matrix-spec-proposals/blob/main/proposals/2530-body-as-caption.md
+        let (body, filename) = match caption {
+            Some(caption) => (caption, Some(filename)),
+            None => (filename, None),
+        };
+
+        let (thumbnail_source, thumbnail_info) = thumbnail.unzip();
+
+        match content_type.type_() {
+            mime::IMAGE => {
+                let info = assign!(info.map(ImageInfo::from).unwrap_or_default(), {
+                    mimetype: Some(content_type.as_ref().to_owned()),
+                    thumbnail_source,
+                    thumbnail_info
+                });
+                let content = assign!(ImageMessageEventContent::new(body, source), {
+                    info: Some(Box::new(info)),
+                    formatted: formatted_caption,
+                    filename
+                });
+                GalleryItemType::Image(content)
+            }
+
+            mime::AUDIO => {
+                let mut content = assign!(AudioMessageEventContent::new(body, source), {
+                    formatted: formatted_caption,
+                    filename
+                });
+
+                if let Some(AttachmentInfo::Voice { audio_info, waveform: Some(waveform_vec) }) =
+                    &info
+                {
+                    if let Some(duration) = audio_info.duration {
+                        let waveform = waveform_vec.iter().map(|v| (*v).into()).collect();
+                        content.audio =
+                            Some(UnstableAudioDetailsContentBlock::new(duration, waveform));
+                    }
+                    content.voice = Some(UnstableVoiceContentBlock::new());
+                }
+
+                let mut audio_info = info.map(AudioInfo::from).unwrap_or_default();
+                audio_info.mimetype = Some(content_type.as_ref().to_owned());
+                let content = content.info(Box::new(audio_info));
+
+                GalleryItemType::Audio(content)
+            }
+
+            mime::VIDEO => {
+                let info = assign!(info.map(VideoInfo::from).unwrap_or_default(), {
+                    mimetype: Some(content_type.as_ref().to_owned()),
+                    thumbnail_source,
+                    thumbnail_info
+                });
+                let content = assign!(VideoMessageEventContent::new(body, source), {
+                    info: Some(Box::new(info)),
+                    formatted: formatted_caption,
+                    filename
+                });
+                GalleryItemType::Video(content)
+            }
+
+            _ => {
+                let info = assign!(info.map(FileInfo::from).unwrap_or_default(), {
+                    mimetype: Some(content_type.as_ref().to_owned()),
+                    thumbnail_source,
+                    thumbnail_info
+                });
+                let content = assign!(FileMessageEventContent::new(body, source), {
+                    info: Some(Box::new(info)),
+                    formatted: formatted_caption,
+                    filename,
+                });
+                GalleryItemType::File(content)
+            }
+        }
+    }
+
     /// Update the power levels of a select set of users of this room.
     ///
     /// Issue a `power_levels` state event request to the server, changing the