examples: Add C and Rust examples of handling large uploads

The programs demonstrate handling requests with payloads larger than
4GiB which means they need to be written out to disk and so also
demonstrates the use of the file-system access mechanism.

Signed-off-by: Andrew Clayton <[email protected]>

Author: ac000

README.md

@@ -264,7 +264,7 @@ repository root for more details) but will instead assume you already have a
 Unit with the WebAssembly language module already running, perhaps installed
 via a package.
 
-Create the following Unit config
+Create the following Unit config (editing the module paths as appropriate)
 
 ```JSON
 {
@@ -276,7 +276,7 @@ Create the following Unit config
 
     "settings": {
         "http": {
-            "max_body_size": 1073741824
+            "max_body_size": 8589934592
         }
     },
 
@@ -297,6 +297,14 @@ Create the following Unit config
                 "pass": "applications/luw-upload-reflector"
             }
         },
+        {
+            "match": {
+                "uri": "/large-upload*"
+            },
+            "action": {
+                "pass": "applications/large-upload"
+            }
+        },
         {
             "match": {
                 "uri": "/rust-echo*"
@@ -315,7 +323,15 @@ Create the following Unit config
         },
         {
             "match": {
-                "uri": "/hello-world*"
+                "uri": "/rust-large-upload*"
+            },
+            "action": {
+                "pass": "applications/rust-large-upload"
+            }
+        },
+        {
+            "match": {
+                "uri": "/rust-hello-world*"
             },
             "action": {
                 "pass": "applications/rust-hello-world"
@@ -342,6 +358,21 @@ Create the following Unit config
             "request_end_handler": "luw_request_end_handler",
             "response_end_handler": "luw_response_end_handler"
         },
+        "large-upload": {
+            "type": "wasm",
+            "module": "/path/to/unit-wasm/examples/c/large-upload.wasm",
+            "request_handler": "luw_request_handler",
+            "malloc_handler": "luw_malloc_handler",
+            "free_handler": "luw_free_handler",
+            "module_init_handler": "luw_module_init_handler",
+            "module_end_handler": "luw_module_end_handler",
+            "response_end_handler": "luw_response_end_handler",
+            "access": {
+                "filesystem": [
+                    "/var/tmp"
+                ]
+            }
+        },
         "rust-echo-request": {
             "type": "wasm",
             "module": "/path/to/unit-wasm/examples/rust/echo-request/target/wasm32-wasi/debug/rust_echo_request.wasm",
@@ -360,6 +391,21 @@ Create the following Unit config
             "request_end_handler": "uwr_request_end_handler",
             "response_end_handler": "uwr_response_end_handler"
         },
+        "rust-large-upload": {
+            "type": "wasm",
+            "module": "/path/to/src/unit-wasm/examples/rust/large-upload/target/wasm32-wasi/debug/rust_large_upload.wasm",
+            "request_handler": "uwr_request_handler",
+            "malloc_handler": "luw_malloc_handler",
+            "free_handler": "luw_free_handler",
+            "module_init_handler": "uwr_module_init_handler",
+            "module_end_handler": "uwr_module_end_handler",
+            "response_end_handler": "uwr_response_end_handler",
+            "access": {
+                "filesystem": [
+                    "/var/tmp"
+                ]
+            }
+        },
         "rust-hello-world": {
             "type": "wasm",
             "module": "/path/to/unit-wasm/examples/rust/hello-world/target/wasm32-wasi/debug/rust_hello_world.wasm",

examples/c/Makefile

@@ -12,7 +12,9 @@ luw_deps = $(LUW_SRCDIR)/libunit-wasm.a \
 
 examples: examples-luw
 
-examples-luw: luw-echo-request.wasm luw-upload-reflector.wasm
+examples-luw: luw-echo-request.wasm \
+	      luw-upload-reflector.wasm \
+	      large-upload.wasm
 
 examples-raw: echo-request-raw.wasm upload-reflector-raw.wasm
 
@@ -36,5 +38,9 @@ upload-reflector-raw.wasm: upload-reflector-raw.c unit-wasm-raw.o
 	$(PP_CCLNK) $(SDIR)/$@
 	$(v)$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $< unit-wasm-raw.o
 
+large-upload.wasm: large-upload.c $(luw_deps)
+	$(PP_CCLNK) $(SDIR)/$@
+	$(v)$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $< $(LIBS)
+
 clean:
 	rm -f *.wasm *.o *.gch

examples/c/large-upload.c

@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+
+/* examples/c/large-upload.c - Example of handling request payload larger
+ *			       larger than the shared memory
+ *
+ * Copyright (C) Andrew Clayton
+ * Copyright (C) F5, Inc.
+ */
+
+#define _XOPEN_SOURCE	500
+
+#define _FILE_OFFSET_BITS 64
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "unit/unit-wasm.h"
+
+static luw_ctx_t ctx;
+static u8 *request_buf;
+static unsigned long long total_bytes_wrote;
+static int fd;
+
+__luw_export_name("luw_module_end_handler")
+void luw_module_end_handler(void)
+{
+        free(request_buf);
+}
+
+__luw_export_name("luw_module_init_handler")
+void luw_module_init_handler(void)
+{
+        request_buf = malloc(luw_mem_get_init_size());
+}
+
+__luw_export_name("luw_response_end_handler")
+void luw_response_end_handler(void)
+{
+	close(fd);
+	total_bytes_wrote = 0;
+}
+
+__luw_export_name("luw_request_handler")
+int luw_request_handler(u8 *addr)
+{
+	ssize_t bytes_wrote;
+
+	if (total_bytes_wrote == 0) {
+		luw_init_ctx(&ctx, addr, 0);
+		luw_set_req_buf(&ctx, &request_buf, LUW_SRB_NONE);
+
+		fd = open("/var/tmp/large-file.dat", O_CREAT|O_TRUNC|O_WRONLY,
+			  0666);
+	}
+
+	bytes_wrote = luw_mem_splice_file(addr, fd);
+	if (bytes_wrote == -1)
+		return -1;
+
+	total_bytes_wrote += bytes_wrote;
+	if (total_bytes_wrote == luw_get_http_content_len(&ctx))
+		luw_http_response_end();
+
+	return 0;
+}

examples/rust/Makefile

@@ -2,7 +2,10 @@ include ../../shared.mk
 
 SDIR = examples/rust
 
-examples: rust-echo-request rust-upload-reflector rust-hello-world
+examples: rust-echo-request \
+	  rust-upload-reflector \
+	  rust-hello-world \
+	  rust-large-upload
 
 rust-echo-request: echo-request/Cargo.toml echo-request/src/lib.rs
 	$(PP_GEN) $(SDIR)/echo-request/target/wasm32-wasi/
@@ -16,6 +19,10 @@ rust-hello-world: hello-world/Cargo.toml hello-world/src/lib.rs
 	$(PP_GEN) $(SDIR)/hello-world/target/wasm32-wasi/
 	$(v)cd hello-world; cargo build --target=wasm32-wasi
 
+rust-large-upload: large-upload/Cargo.toml large-upload/src/lib.rs
+	$(PP_GEN) $(SDIR)/large-upload/target/wasm32-wasi/
+	$(v)cd large-upload; cargo build --target=wasm32-wasi
+
 clean:
 	rm -f */Cargo.lock
 	rm -rf */target

examples/rust/large-upload/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "rust-large-upload"
+version = "0.2.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+unit-wasm = { path = "../../../src/rust", version = "0.2.0" }
+
+[lib]
+crate-type = ["cdylib"]

examples/rust/large-upload/src/lib.rs

@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+
+/*
+ * Copyright (C) Andrew Clayton
+ * Copyright (C) Timo Stark
+ * Copyright (C) F5, Inc.
+ */
+
+use unit_wasm::rusty::*;
+
+use std::fs::File;
+use std::ptr::null_mut;
+
+static mut CTX: luw_ctx_t = UWR_CTX_INITIALIZER();
+static mut REQUEST_BUF: *mut u8 = null_mut();
+static mut TOTAL_BYTES_WROTE: u64 = 0;
+
+#[no_mangle]
+pub extern "C" fn uwr_module_end_handler() {
+    unsafe { uwr_free(REQUEST_BUF); }
+}
+
+#[no_mangle]
+pub extern "C" fn uwr_module_init_handler() {
+    unsafe { REQUEST_BUF = uwr_malloc(uwr_mem_get_init_size()); }
+}
+
+#[no_mangle]
+pub extern "C" fn uwr_response_end_handler() {
+    unsafe { TOTAL_BYTES_WROTE = 0; }
+}
+
+#[no_mangle]
+pub extern "C" fn uwr_request_handler(addr: *mut u8) -> i32 {
+    let ctx: *mut luw_ctx_t = unsafe { &mut CTX };
+    let mut f;
+    let bytes_wrote: isize;
+    let mut total = unsafe { TOTAL_BYTES_WROTE };
+
+    if total == 0 {
+        uwr_init_ctx(ctx, addr, 0);
+        uwr_set_req_buf(ctx, unsafe { &mut REQUEST_BUF }, LUW_SRB_NONE);
+
+        f = File::create("/var/tmp/large-file.dat").unwrap();
+    } else {
+        f = File::options()
+            .append(true)
+            .open("/var/tmp/large-file.dat")
+            .unwrap();
+    }
+
+    bytes_wrote = uwr_mem_splice_file(addr, &mut f);
+    if bytes_wrote == -1 {
+        return -1;
+    }
+
+    total += bytes_wrote as u64;
+    if total == uwr_get_http_content_len(ctx) {
+        uwr_http_response_end();
+    } else {
+        unsafe { TOTAL_BYTES_WROTE = total };
+    }
+
+    return 0;
+}

unit-wasm-conf.json

@@ -7,7 +7,7 @@
 
     "settings": {
         "http": {
-            "max_body_size": 1073741824
+            "max_body_size": 8589934592
         }
     },
 
@@ -28,6 +28,14 @@
                 "pass": "applications/luw-upload-reflector"
             }
         },
+        {
+            "match": {
+                "uri": "/large-upload*"
+            },
+            "action": {
+                "pass": "applications/large-upload"
+            }
+        },
         {
             "match": {
                 "uri": "/rust-echo*"
@@ -44,6 +52,14 @@
                 "pass": "applications/rust-upload-reflector"
             }
         },
+        {
+            "match": {
+                "uri": "/rust-large-upload*"
+            },
+            "action": {
+                "pass": "applications/rust-large-upload"
+            }
+        },
         {
             "match": {
                 "uri": "/rust-hello-world*"
@@ -73,6 +89,21 @@
             "request_end_handler": "luw_request_end_handler",
             "response_end_handler": "luw_response_end_handler"
         },
+        "large-upload": {
+            "type": "wasm",
+            "module": "/path/to/unit-wasm/examples/c/large-upload.wasm",
+            "request_handler": "luw_request_handler",
+            "malloc_handler": "luw_malloc_handler",
+            "free_handler": "luw_free_handler",
+            "module_init_handler": "luw_module_init_handler",
+            "module_end_handler": "luw_module_end_handler",
+            "response_end_handler": "luw_response_end_handler",
+            "access": {
+                "filesystem": [
+                    "/var/tmp"
+                ]
+            }
+        },
         "rust-echo-request": {
             "type": "wasm",
             "module": "/path/to/unit-wasm/examples/rust/echo-request/target/wasm32-wasi/debug/rust_echo_request.wasm",
@@ -91,6 +122,21 @@
             "request_end_handler": "uwr_request_end_handler",
             "response_end_handler": "uwr_response_end_handler"
         },
+        "rust-large-upload": {
+            "type": "wasm",
+            "module": "/path/to/src/unit-wasm/examples/rust/large-upload/target/wasm32-wasi/debug/rust_large_upload.wasm",
+            "request_handler": "uwr_request_handler",
+            "malloc_handler": "luw_malloc_handler",
+            "free_handler": "luw_free_handler",
+            "module_init_handler": "uwr_module_init_handler",
+            "module_end_handler": "uwr_module_end_handler",
+            "response_end_handler": "uwr_response_end_handler",
+            "access": {
+                "filesystem": [
+                    "/var/tmp"
+                ]
+            }
+        },
         "rust-hello-world": {
             "type": "wasm",
             "module": "/path/to/unit-wasm/examples/rust/hello-world/target/wasm32-wasi/debug/rust_hello_world.wasm",