batch submitting chunks to redis

Author: ebarajas

ruby/lib/ci/queue/redis/worker.rb

@@ -276,19 +276,25 @@ def register
         private
 
         def store_chunk_metadata(chunks)
-          redis.multi do |transaction|
-            chunks.each do |chunk|
-              # Store chunk metadata with TTL
-              transaction.set(
-                key('chunk', chunk.id),
-                chunk.to_json
-              )
-              transaction.expire(key('chunk', chunk.id), config.redis_ttl)
-
-              # Track all chunks for cleanup
-              transaction.sadd(key('chunks'), chunk.id)
+          # Batch operations to avoid exceeding Redis multi operation limits
+          # Each chunk requires 3 commands (set, expire, sadd), so batch conservatively
+          batch_size = 7 # 7 chunks = 21 commands + 1 expire = 22 commands per batch
+
+          chunks.each_slice(batch_size) do |chunk_batch|
+            redis.multi do |transaction|
+              chunk_batch.each do |chunk|
+                # Store chunk metadata with TTL
+                transaction.set(
+                  key('chunk', chunk.id),
+                  chunk.to_json
+                )
+                transaction.expire(key('chunk', chunk.id), config.redis_ttl)
+
+                # Track all chunks for cleanup
+                transaction.sadd(key('chunks'), chunk.id)
+              end
+              transaction.expire(key('chunks'), config.redis_ttl)
             end
-            transaction.expire(key('chunks'), config.redis_ttl)
           end
         end
 

ruby/test/ci/queue/redis/worker_chunk_test.rb

@@ -202,6 +202,34 @@ def test_chunk_metadata_has_ttl
     assert ttl > 0, 'Chunk metadata should have TTL set'
   end
 
+  def test_populate_with_many_chunks_uses_batching
+    # Create 20 test chunks to verify batching works (batch size is 7)
+    tests = (1..20).map { |i| MockTest.new("TestSuite#{i}#test_1") }
+    chunks = (1..20).map do |i|
+      CI::Queue::TestChunk.new("TestSuite#{i}:full_suite", "TestSuite#{i}", :full_suite, [], 1000.0)
+    end
+
+    CI::Queue.stub(:shuffle, chunks) do
+      @worker.populate(tests)
+    end
+
+    # Verify all chunks were stored despite batching
+    chunks.each do |chunk|
+      chunk_data = @redis.get("build:42:chunk:#{chunk.id}")
+      refute_nil chunk_data, "Chunk #{chunk.id} should be stored"
+
+      parsed = JSON.parse(chunk_data)
+      assert_equal 'full_suite', parsed['type']
+      assert_equal chunk.suite_name, parsed['suite_name']
+    end
+
+    # Verify all chunk IDs are in the chunks set
+    stored_chunks = @redis.smembers('build:42:chunks')
+    chunks.each do |chunk|
+      assert_includes stored_chunks, chunk.id
+    end
+  end
+
   private
 
   def create_mock_tests(test_ids)