feat(frontend-canister): create_chunks (#3898)

Author: sesi200

CHANGELOG.md

@@ -36,6 +36,14 @@ This applies to the following:
  - tech stack value computation
  - packtool (vessel, mops etc)
 
+## Dependencies
+
+### Frontend canister
+
+Added `create_chunks`. It has the same behavior as `create_chunk`, except that it takes a `vec blob` and returns a `vec BatchId` instead of non-`vec` variants.
+
+Module hash: 3a533f511b3960b4186e76cf9abfbd8222a2c507456a66ec55671204ee70cae3
+
 # 0.23.0
 
 ### feat: Add canister snapshots

docs/design/asset-canister-interface.md

@@ -267,7 +267,7 @@ Required Permission: [Prepare](#permission-prepare)
   create_chunk: (
     record { 
       batch_id: BatchId;
-      content: blob 
+      content: blob
     }
   ) -> (record { 
     chunk_id: ChunkId
@@ -286,6 +286,31 @@ Preconditions:
 
 Required Permission: [Prepare](#permission-prepare)
 
+### Method: `create_chunks`
+
+```candid
+  create_chunks: (
+    record {
+      batch_id: BatchId;
+      content: vec blob;
+    }
+  ) -> (
+    chunk_ids: vec ChunkId;
+  );
+```
+
+This method stores a number of chunks and extends the batch expiry time.
+
+When creating chunks for a given content encoding, the size of each chunk except the last must be the same.
+
+The asset canister must retain all data related to a batch for at least the [Minimum Batch Retention Duration](#constant-minimum-batch-retention-duration) after creating a chunk in a batch.
+
+Preconditions:
+- The batch exists.
+- Creation of the chunk would not exceed chunk creation limits.
+
+Required Permission: [Prepare](#permission-prepare)
+
 ### Method: `commit_batch`
 
 ```candid

src/canisters/frontend/ic-certified-assets/assets.did

@@ -199,6 +199,7 @@ service: (asset_canister_args: opt AssetCanisterArgs) -> {
   create_batch : (record {}) -> (record { batch_id: BatchId });
 
   create_chunk: (record { batch_id: BatchId; content: blob }) -> (record { chunk_id: ChunkId });
+  create_chunks: (record { batch_id: BatchId; content: vec blob }) -> (record { chunk_ids: vec ChunkId });
 
   // Perform all operations successfully, or reject
   commit_batch: (CommitBatchArguments) -> ();

src/canisters/frontend/ic-certified-assets/src/lib.rs

@@ -167,6 +167,15 @@ fn create_chunk(arg: CreateChunkArg) -> CreateChunkResponse {
     })
 }
 
+#[update(guard = "can_prepare")]
+#[candid_method(update)]
+fn create_chunks(arg: CreateChunksArg) -> CreateChunksResponse {
+    STATE.with(|s| match s.borrow_mut().create_chunks(arg, time()) {
+        Ok(chunk_ids) => CreateChunksResponse { chunk_ids },
+        Err(msg) => trap(&msg),
+    })
+}
+
 #[update(guard = "can_commit")]
 #[candid_method(update)]
 fn create_asset(arg: CreateAssetArguments) {

src/canisters/frontend/ic-certified-assets/src/state_machine.rs

@@ -582,43 +582,73 @@ impl State {
     }
 
     pub fn create_chunk(&mut self, arg: CreateChunkArg, now: u64) -> Result<ChunkId, String> {
+        let ids = self.create_chunks_helper(arg.batch_id, vec![arg.content], now)?;
+        ids.into_iter()
+            .next()
+            .ok_or_else(|| "Bug: created chunk did not return a chunk id.".to_string())
+    }
+
+    pub fn create_chunks(
+        &mut self,
+        CreateChunksArg {
+            batch_id,
+            content: chunks,
+        }: CreateChunksArg,
+        now: u64,
+    ) -> Result<Vec<ChunkId>, String> {
+        self.create_chunks_helper(batch_id, chunks, now)
+    }
+
+    /// Post-condition: `chunks.len() == output_chunk_ids.len()`
+    fn create_chunks_helper(
+        &mut self,
+        batch_id: Nat,
+        chunks: Vec<ByteBuf>,
+        now: u64,
+    ) -> Result<Vec<ChunkId>, String> {
         if let Some(max_chunks) = self.configuration.max_chunks {
-            if self.chunks.len() + 1 > max_chunks as usize {
+            if self.chunks.len() + chunks.len() > max_chunks as usize {
                 return Err("chunk limit exceeded".to_string());
             }
         }
         if let Some(max_bytes) = self.configuration.max_bytes {
             let current_total_bytes = &self.batches.iter().fold(0, |acc, (_batch_id, batch)| {
                 acc + batch.chunk_content_total_size
             });
-
-            if current_total_bytes + arg.content.as_ref().len() > max_bytes as usize {
+            let new_bytes: usize = chunks.iter().map(|chunk| chunk.len()).sum();
+            if current_total_bytes + new_bytes > max_bytes as usize {
                 return Err("byte limit exceeded".to_string());
             }
         }
         let batch = self
             .batches
-            .get_mut(&arg.batch_id)
+            .get_mut(&batch_id)
             .ok_or_else(|| "batch not found".to_string())?;
         if batch.commit_batch_arguments.is_some() {
             return Err("batch has been proposed".to_string());
         }
 
         batch.expires_at = Int::from(now + BATCH_EXPIRY_NANOS);
 
-        let chunk_id = self.next_chunk_id.clone();
-        self.next_chunk_id += 1_u8;
-        batch.chunk_content_total_size += arg.content.as_ref().len();
-
-        self.chunks.insert(
-            chunk_id.clone(),
-            Chunk {
-                batch_id: arg.batch_id,
-                content: RcBytes::from(arg.content),
-            },
-        );
+        let chunks_len = chunks.len();
+
+        let mut chunk_ids = Vec::with_capacity(chunks.len());
+        for chunk in chunks {
+            let chunk_id = self.next_chunk_id.clone();
+            self.next_chunk_id += 1_u8;
+            batch.chunk_content_total_size += chunk.len();
+            self.chunks.insert(
+                chunk_id.clone(),
+                Chunk {
+                    batch_id: batch_id.clone(),
+                    content: RcBytes::from(chunk),
+                },
+            );
+            chunk_ids.push(chunk_id);
+        }
 
-        Ok(chunk_id)
+        debug_assert!(chunks_len == chunk_ids.len());
+        Ok(chunk_ids)
     }
 
     pub fn commit_batch(&mut self, arg: CommitBatchArguments, now: u64) -> Result<(), String> {

src/canisters/frontend/ic-certified-assets/src/tests.rs

@@ -9,6 +9,7 @@ use crate::types::{
     SetAssetPropertiesArguments,
 };
 use crate::url_decode::{url_decode, UrlDecodeError};
+use crate::CreateChunksArg;
 use candid::{Nat, Principal};
 use ic_certification_testing::CertificateBuilder;
 use ic_crypto_tree_hash::Digest;
@@ -724,14 +725,24 @@ fn cannot_create_chunk_in_proposed_batch_() {
     const BODY: &[u8] = b"<!DOCTYPE html><html></html>";
     match state.create_chunk(
         CreateChunkArg {
-            batch_id: batch_1,
+            batch_id: batch_1.clone(),
             content: ByteBuf::from(BODY.to_vec()),
         },
         time_now,
     ) {
         Err(err) if err == *"batch has been proposed" => {}
         other => panic!("expected batch already proposed error, got: {:?}", other),
     }
+    match state.create_chunks(
+        CreateChunksArg {
+            batch_id: batch_1,
+            content: vec![ByteBuf::from(BODY.to_vec())],
+        },
+        time_now,
+    ) {
+        Err(err) if err == *"batch has been proposed" => {}
+        other => panic!("expected batch already proposed error, got: {:?}", other),
+    }
 }
 
 #[test]
@@ -3765,6 +3776,18 @@ mod enforce_limits {
                 time_now,
             )
             .unwrap();
+        assert_eq!(
+            state
+                .create_chunks(
+                    CreateChunksArg {
+                        batch_id: batch_2.clone(),
+                        content: vec![ByteBuf::new(), ByteBuf::new()]
+                    },
+                    time_now
+                )
+                .unwrap_err(),
+            "chunk limit exceeded"
+        );
         state
             .create_chunk(
                 CreateChunkArg {
@@ -3818,20 +3841,27 @@ mod enforce_limits {
 
         let batch_1 = state.create_batch(time_now).unwrap();
         let batch_2 = state.create_batch(time_now).unwrap();
+        assert_eq!(
+            state
+                .create_chunks(
+                    CreateChunksArg {
+                        batch_id: batch_1.clone(),
+                        content: vec![
+                            ByteBuf::from(c0.clone()),
+                            ByteBuf::from(c1.clone()),
+                            ByteBuf::from(c2.clone())
+                        ]
+                    },
+                    time_now
+                )
+                .unwrap_err(),
+            "byte limit exceeded"
+        );
         state
-            .create_chunk(
-                CreateChunkArg {
+            .create_chunks(
+                CreateChunksArg {
                     batch_id: batch_1.clone(),
-                    content: ByteBuf::from(c0),
-                },
-                time_now,
-            )
-            .unwrap();
-        state
-            .create_chunk(
-                CreateChunkArg {
-                    batch_id: batch_2.clone(),
-                    content: ByteBuf::from(c1),
+                    content: vec![ByteBuf::from(c0), ByteBuf::from(c1)],
                 },
                 time_now,
             )

src/canisters/frontend/ic-certified-assets/src/types.rs

@@ -134,6 +134,17 @@ pub struct CreateChunkResponse {
     pub chunk_id: ChunkId,
 }
 
+#[derive(Clone, Debug, CandidType, Deserialize)]
+pub struct CreateChunksArg {
+    pub batch_id: BatchId,
+    pub content: Vec<ByteBuf>,
+}
+
+#[derive(Clone, Debug, CandidType, Deserialize)]
+pub struct CreateChunksResponse {
+    pub chunk_ids: Vec<ChunkId>,
+}
+
 #[derive(Clone, Debug, CandidType, Deserialize, PartialEq, Eq)]
 pub struct AssetProperties {
     pub max_age: Option<u64>,

src/distributed/assetstorage.did

@@ -199,6 +199,7 @@ service: (asset_canister_args: opt AssetCanisterArgs) -> {
   create_batch : (record {}) -> (record { batch_id: BatchId });
 
   create_chunk: (record { batch_id: BatchId; content: blob }) -> (record { chunk_id: ChunkId });
+  create_chunks: (record { batch_id: BatchId; content: vec blob }) -> (record { chunk_ids: vec ChunkId });
 
   // Perform all operations successfully, or reject
   commit_batch: (CommitBatchArguments) -> ();