examples: Add C and Rust examples of handling large uploads

ac000 · ac000 · commit e4a868078ab4 · 2023-09-25T17:39:51.000+01:00
The programs demonstrate handling requests with payloads larger than
4GiB which means they need to be written out to disk and so also
demonstrates the use of the file-system access mechanism.

Signed-off-by: Andrew Clayton &lt;a.clayton@nginx.com&gt;
diff --git a/README.md b/README.md
@@ -264,7 +264,7 @@ repository root for more details) but will instead assume you already have a
 Unit with the WebAssembly language module already running, perhaps installed
 via a package.
 
-Create the following Unit config
+Create the following Unit config (editing the module paths as appropriate)
 
 ```JSON
 {
@@ -276,7 +276,7 @@ Create the following Unit config
 
     "settings": {
         "http": {
-            "max_body_size": 1073741824
+            "max_body_size": 8589934592
         }
     },
 
@@ -297,6 +297,14 @@ Create the following Unit config
                 "pass": "applications/luw-upload-reflector"
             }
         },
+        {
+            "match": {
+                "uri": "/large-upload*"
+            },
+            "action": {
+                "pass": "applications/large-upload"
+            }
+        },
         {
             "match": {
                 "uri": "/rust-echo*"
@@ -315,7 +323,15 @@ Create the following Unit config
         },
         {
             "match": {
-                "uri": "/hello-world*"
+                "uri": "/rust-large-upload*"
+            },
+            "action": {
+                "pass": "applications/rust-large-upload"
+            }
+        },
+        {
+            "match": {
+                "uri": "/rust-hello-world*"
             },
             "action": {
                 "pass": "applications/rust-hello-world"
@@ -342,6 +358,21 @@ Create the following Unit config
             "request_end_handler": "luw_request_end_handler",
             "response_end_handler": "luw_response_end_handler"
         },
+        "large-upload": {
+            "type": "wasm",
+            "module": "/path/to/unit-wasm/examples/c/large-upload.wasm",
+            "request_handler": "luw_request_handler",
+            "malloc_handler": "luw_malloc_handler",
+            "free_handler": "luw_free_handler",
+            "module_init_handler": "luw_module_init_handler",
+            "module_end_handler": "luw_module_end_handler",
+            "response_end_handler": "luw_response_end_handler",
+            "access": {
+                "filesystem": [
+                    "/var/tmp"
+                ]
+            }
+        },
         "rust-echo-request": {
             "type": "wasm",
             "module": "/path/to/unit-wasm/examples/rust/echo-request/target/wasm32-wasi/debug/rust_echo_request.wasm",
@@ -360,6 +391,21 @@ Create the following Unit config
             "request_end_handler": "uwr_request_end_handler",
             "response_end_handler": "uwr_response_end_handler"
         },
+        "rust-large-upload": {
+            "type": "wasm",
+            "module": "/path/to/src/unit-wasm/examples/rust/large-upload/target/wasm32-wasi/debug/rust_large_upload.wasm",
+            "request_handler": "uwr_request_handler",
+            "malloc_handler": "luw_malloc_handler",
+            "free_handler": "luw_free_handler",
+            "module_init_handler": "uwr_module_init_handler",
+            "module_end_handler": "uwr_module_end_handler",
+            "response_end_handler": "uwr_response_end_handler",
+            "access": {
+                "filesystem": [
+                    "/var/tmp"
+                ]
+            }
+        },
         "rust-hello-world": {
             "type": "wasm",
             "module": "/path/to/unit-wasm/examples/rust/hello-world/target/wasm32-wasi/debug/rust_hello_world.wasm",
diff --git a/examples/c/Makefile b/examples/c/Makefile
@@ -12,7 +12,9 @@ luw_deps = $(LUW_SRCDIR)/libunit-wasm.a \
 
 examples: examples-luw
 
-examples-luw: luw-echo-request.wasm luw-upload-reflector.wasm
+examples-luw: luw-echo-request.wasm \
+	      luw-upload-reflector.wasm \
+	      large-upload.wasm
 
 examples-raw: echo-request-raw.wasm upload-reflector-raw.wasm
 
@@ -36,5 +38,9 @@ upload-reflector-raw.wasm: upload-reflector-raw.c unit-wasm-raw.o
 	$(PP_CCLNK) $(SDIR)/$@
 	$(v)$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $< unit-wasm-raw.o
 
+large-upload.wasm: large-upload.c $(luw_deps)
+	$(PP_CCLNK) $(SDIR)/$@
+	$(v)$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $< $(LIBS)
+
 clean:
 	rm -f *.wasm *.o *.gch
diff --git a/examples/c/large-upload.c b/examples/c/large-upload.c
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+
+/* examples/c/large-upload.c - Example of handling request payload larger
+ *			       larger than the shared memory
+ *
+ * Copyright (C) Andrew Clayton
+ * Copyright (C) F5, Inc.
+ */
+
+#define _XOPEN_SOURCE	500
+
+#define _FILE_OFFSET_BITS 64
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "unit/unit-wasm.h"
+
+static luw_ctx_t ctx;
+static u8 *request_buf;
+static unsigned long long total_bytes_wrote;
+static int fd;
+
+__luw_export_name("luw_module_end_handler")
+void luw_module_end_handler(void)
+{
+        free(request_buf);
+}
+
+__luw_export_name("luw_module_init_handler")
+void luw_module_init_handler(void)
+{
+        request_buf = malloc(luw_mem_get_init_size());
+}
+
+__luw_export_name("luw_response_end_handler")
+void luw_response_end_handler(void)
+{
+	close(fd);
+	total_bytes_wrote = 0;
+}
+
+__luw_export_name("luw_request_handler")
+int luw_request_handler(u8 *addr)
+{
+	ssize_t bytes_wrote;
+
+	if (total_bytes_wrote == 0) {
+		luw_init_ctx(&ctx, addr, 0);
+		luw_set_req_buf(&ctx, &request_buf, LUW_SRB_NONE);
+
+		fd = open("/var/tmp/large-file.dat", O_CREAT|O_TRUNC|O_WRONLY,
+			  0666);
+	}
+
+	bytes_wrote = luw_mem_splice_file(addr, fd);
+	if (bytes_wrote == -1)
+		return -1;
+
+	total_bytes_wrote += bytes_wrote;
+	if (total_bytes_wrote == luw_get_http_content_len(&ctx))
+		luw_http_response_end();
+
+	return 0;
+}
diff --git a/examples/rust/Makefile b/examples/rust/Makefile
@@ -2,7 +2,10 @@ include ../../shared.mk
 
 SDIR = examples/rust
 
-examples: rust-echo-request rust-upload-reflector rust-hello-world
+examples: rust-echo-request \
+	  rust-upload-reflector \
+	  rust-hello-world \
+	  rust-large-upload
 
 rust-echo-request: echo-request/Cargo.toml echo-request/src/lib.rs
 	$(PP_GEN) $(SDIR)/echo-request/target/wasm32-wasi/
@@ -16,6 +19,10 @@ rust-hello-world: hello-world/Cargo.toml hello-world/src/lib.rs
 	$(PP_GEN) $(SDIR)/hello-world/target/wasm32-wasi/
 	$(v)cd hello-world; cargo build --target=wasm32-wasi
 
+rust-large-upload: large-upload/Cargo.toml large-upload/src/lib.rs
+	$(PP_GEN) $(SDIR)/large-upload/target/wasm32-wasi/
+	$(v)cd large-upload; cargo build --target=wasm32-wasi
+
 clean:
 	rm -f */Cargo.lock
 	rm -rf */target
diff --git a/examples/rust/large-upload/Cargo.toml b/examples/rust/large-upload/Cargo.toml
@@ -0,0 +1,12 @@
+[package]
+name = "rust-large-upload"
+version = "0.2.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+unit-wasm = { path = "../../../src/rust", version = "0.2.0" }
+
+[lib]
+crate-type = ["cdylib"]
diff --git a/examples/rust/large-upload/src/lib.rs b/examples/rust/large-upload/src/lib.rs
@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+
+/*
+ * Copyright (C) Andrew Clayton
+ * Copyright (C) Timo Stark
+ * Copyright (C) F5, Inc.
+ */
+
+use unit_wasm::rusty::*;
+
+use std::fs::File;
+use std::ptr::null_mut;
+
+static mut CTX: luw_ctx_t = UWR_CTX_INITIALIZER();
+static mut REQUEST_BUF: *mut u8 = null_mut();
+static mut TOTAL_BYTES_WROTE: u64 = 0;
+
+#[no_mangle]
+pub extern "C" fn uwr_module_end_handler() {
+    unsafe { uwr_free(REQUEST_BUF); }
+}
+
+#[no_mangle]
+pub extern "C" fn uwr_module_init_handler() {
+    unsafe { REQUEST_BUF = uwr_malloc(uwr_mem_get_init_size()); }
+}
+
+#[no_mangle]
+pub extern "C" fn uwr_response_end_handler() {
+    unsafe { TOTAL_BYTES_WROTE = 0; }
+}
+
+#[no_mangle]
+pub extern "C" fn uwr_request_handler(addr: *mut u8) -> i32 {
+    let ctx: *mut luw_ctx_t = unsafe { &mut CTX };
+    let mut f;
+    let bytes_wrote: isize;
+    let mut total = unsafe { TOTAL_BYTES_WROTE };
+
+    if total == 0 {
+        uwr_init_ctx(ctx, addr, 0);
+        uwr_set_req_buf(ctx, unsafe { &mut REQUEST_BUF }, LUW_SRB_NONE);
+
+        f = File::create("/var/tmp/large-file.dat").unwrap();
+    } else {
+        f = File::options()
+            .append(true)
+            .open("/var/tmp/large-file.dat")
+            .unwrap();
+    }
+
+    bytes_wrote = uwr_mem_splice_file(addr, &mut f);
+    if bytes_wrote == -1 {
+        return -1;
+    }
+
+    total += bytes_wrote as u64;
+    if total == uwr_get_http_content_len(ctx) {
+        uwr_http_response_end();
+    } else {
+        unsafe { TOTAL_BYTES_WROTE = total };
+    }
+
+    return 0;
+}
diff --git a/unit-wasm-conf.json b/unit-wasm-conf.json
@@ -7,7 +7,7 @@
 
     "settings": {
         "http": {
-            "max_body_size": 1073741824
+            "max_body_size": 8589934592
         }
     },
 
@@ -28,6 +28,14 @@
                 "pass": "applications/luw-upload-reflector"
             }
         },
+        {
+            "match": {
+                "uri": "/large-upload*"
+            },
+            "action": {
+                "pass": "applications/large-upload"
+            }
+        },
         {
             "match": {
                 "uri": "/rust-echo*"
@@ -44,6 +52,14 @@
                 "pass": "applications/rust-upload-reflector"
             }
         },
+        {
+            "match": {
+                "uri": "/rust-large-upload*"
+            },
+            "action": {
+                "pass": "applications/rust-large-upload"
+            }
+        },
         {
             "match": {
                 "uri": "/rust-hello-world*"
@@ -73,6 +89,21 @@
             "request_end_handler": "luw_request_end_handler",
             "response_end_handler": "luw_response_end_handler"
         },
+        "large-upload": {
+            "type": "wasm",
+            "module": "/path/to/unit-wasm/examples/c/large-upload.wasm",
+            "request_handler": "luw_request_handler",
+            "malloc_handler": "luw_malloc_handler",
+            "free_handler": "luw_free_handler",
+            "module_init_handler": "luw_module_init_handler",
+            "module_end_handler": "luw_module_end_handler",
+            "response_end_handler": "luw_response_end_handler",
+            "access": {
+                "filesystem": [
+                    "/var/tmp"
+                ]
+            }
+        },
         "rust-echo-request": {
             "type": "wasm",
             "module": "/path/to/unit-wasm/examples/rust/echo-request/target/wasm32-wasi/debug/rust_echo_request.wasm",
@@ -91,6 +122,21 @@
             "request_end_handler": "uwr_request_end_handler",
             "response_end_handler": "uwr_response_end_handler"
         },
+        "rust-large-upload": {
+            "type": "wasm",
+            "module": "/path/to/src/unit-wasm/examples/rust/large-upload/target/wasm32-wasi/debug/rust_large_upload.wasm",
+            "request_handler": "uwr_request_handler",
+            "malloc_handler": "luw_malloc_handler",
+            "free_handler": "luw_free_handler",
+            "module_init_handler": "uwr_module_init_handler",
+            "module_end_handler": "uwr_module_end_handler",
+            "response_end_handler": "uwr_response_end_handler",
+            "access": {
+                "filesystem": [
+                    "/var/tmp"
+                ]
+            }
+        },
         "rust-hello-world": {
             "type": "wasm",
             "module": "/path/to/unit-wasm/examples/rust/hello-world/target/wasm32-wasi/debug/rust_hello_world.wasm",
