[SPARK-59624][PYTHON] Move element-wise UDF conversion behind Arrow batch interfaces

python/pyspark/sql/conversion.py

@@ -24,6 +24,7 @@
     TYPE_CHECKING,
     Any,
     Callable,
+    Iterable,
     Iterator,
     List,
     Optional,
@@ -180,6 +181,26 @@ def select_columns(cls, batch: "pa.RecordBatch", column_indices: list[int]) -> "
             [batch.schema.names[i] for i in column_indices],
         )
 
+    @classmethod
+    def concat_batches(cls, batches: Iterable["pa.RecordBatch"]) -> "pa.RecordBatch":
+        """Concatenate same-schema RecordBatches by row.
+
+        A single batch is returned unchanged. PyArrow before 19.0.0 has no ``concat_batches``;
+        the fallback concatenates the equivalent StructArrays and converts the result back to a
+        RecordBatch.
+        """
+        import pyarrow as pa
+
+        batches = tuple(batches)
+        assert batches
+        if len(batches) == 1:
+            return batches[0]
+        if hasattr(pa, "concat_batches"):
+            return pa.concat_batches(batches)
+        return pa.RecordBatch.from_struct_array(
+            pa.concat_arrays([batch.to_struct_array() for batch in batches])
+        )
+
     @staticmethod
     def wrap_struct(batch: "pa.RecordBatch") -> "pa.RecordBatch":
         """

python/pyspark/sql/tests/test_conversion.py

@@ -115,6 +115,17 @@ def test_flatten_struct_empty_batch(self):
         self.assertEqual(flattened.num_rows, 0)
         self.assertEqual(flattened.num_columns, 2)
 
+    def test_concat_batches(self):
+        import pyarrow as pa
+
+        batches = [
+            pa.RecordBatch.from_arrays([pa.array([1, 2])], ["x"]),
+            pa.RecordBatch.from_arrays([pa.array([3])], ["x"]),
+        ]
+        result = ArrowBatchTransformer.concat_batches(iter(batches))
+        self.assertEqual(result.column(0).to_pylist(), [1, 2, 3])
+        self.assertIs(ArrowBatchTransformer.concat_batches(iter(batches[:1])), batches[0])
+
     def test_wrap_struct_basic(self):
         """Test wrapping columns into a struct."""
         import pyarrow as pa