Downgrade MULTI sentinel in rev_host_clear

mk/tests.mk

@@ -5,6 +5,7 @@
 TEST_DIR   = tests/unit
 TEST_SRCS  = $(TEST_DIR)/test-runner.c \
              $(TEST_DIR)/test-fd-table.c \
+			 $(TEST_DIR)/test-fd-table-refcount.c \
              $(TEST_DIR)/test-path.c \
              $(TEST_DIR)/test-mount.c \
              $(TEST_DIR)/test-cli.c \

src/fd-table.c

@@ -101,6 +101,8 @@ static inline void rev_host_set(struct kbox_fd_table *t, long host_fd, long vfd)
 
     if (host_fd < 0 || (uint64_t) host_fd >= KBOX_HOST_FD_REVERSE_MAX)
         return;
+
+    t->host_fd_refs[host_fd]++;
     cur = t->host_to_vfd[host_fd];
     if (cur == KBOX_HOST_VFD_NONE || cur == (int32_t) vfd) {
         t->host_to_vfd[host_fd] = (int32_t) vfd;
@@ -116,18 +118,35 @@ static inline void rev_host_clear(struct kbox_fd_table *t,
                                   long host_fd,
                                   long vfd)
 {
-    if (host_fd < 0 || (uint64_t) host_fd >= KBOX_HOST_FD_REVERSE_MAX)
+    if (host_fd < 0 || (uint64_t) host_fd >= KBOX_HOST_FD_REVERSE_MAX ||
+        t->host_fd_refs[host_fd] == 0)
         return;
     /* Only the single-holder case can be cleared authoritatively. If
      * the slot is MULTI, leave it: we cannot prove this is the last
      * holder without scanning, and the slow path will handle later
      * lookups correctly. If the slot is NONE or claims a different
      * vfd, we were not the indexed holder; nothing to do.
      */
-    if (t->host_to_vfd[host_fd] == (int32_t) vfd)
+
+    t->host_fd_refs[host_fd]--;
+    if (t->host_fd_refs[host_fd] == 0) {
         t->host_to_vfd[host_fd] = KBOX_HOST_VFD_NONE;
-}
+    } else if (t->host_fd_refs[host_fd] == 1) {
+        /* 1. Try a normal search first */
+        long cur_vfd = kbox_fd_table_find_by_host_fd(t, host_fd);
+
+        /* 2. If the search tripped over the dying slot, hide it and search
+         * again */
+        if (cur_vfd == vfd) {
+            struct kbox_fd_entry *e = fd_lookup(t, vfd);
+            e->host_fd = -1;
+            cur_vfd = kbox_fd_table_find_by_host_fd(t, host_fd);
+            e->host_fd = host_fd;
+        }
 
+        t->host_to_vfd[host_fd] = cur_vfd;
+    }
+}
 static inline void lkl_ref_inc(struct kbox_fd_table *t, long lkl_fd)
 {
     if (lkl_fd >= 0 && (uint64_t) lkl_fd < KBOX_LKL_FD_REFMAX &&
@@ -154,8 +173,10 @@ void kbox_fd_table_init(struct kbox_fd_table *t)
         clear_fd_entry(&t->low_fds[i]);
     for (i = 0; i < KBOX_MID_FD_MAX; i++)
         clear_fd_entry(&t->mid_fds[i]);
-    for (i = 0; i < KBOX_HOST_FD_REVERSE_MAX; i++)
+    for (i = 0; i < KBOX_HOST_FD_REVERSE_MAX; i++) {
         t->host_to_vfd[i] = KBOX_HOST_VFD_NONE;
+        t->host_fd_refs[i] = 0;
+    }
     for (i = 0; i < KBOX_LKL_FD_REFMAX; i++)
         t->lkl_fd_refs[i] = 0;
     t->next_fd = KBOX_FD_BASE;

src/fd-table.h

@@ -66,6 +66,10 @@ struct kbox_fd_table {
      * kbox_fd_table_find_by_host_fd() on the close() hot path.
      */
     int32_t host_to_vfd[KBOX_HOST_FD_REVERSE_MAX];
+    /* Tracks how many VFDs point to a specific Host FD.
+     * This allows us to know when to downgrade from MULTI.
+     */
+    uint16_t host_fd_refs[KBOX_HOST_FD_REVERSE_MAX];
     /* Refcount: how many virtual fds currently reference each
      * lkl_fd. Replaces the O(n) lkl_fd_has_other_ref scan and the
      * still_ref loop in forward_close.

tests/unit/test-fd-table-refcount.c

@@ -0,0 +1,43 @@
+/* SPDX-License-Identifier: MIT */
+#include <string.h>
+
+#include "fd-table.h"
+#include "test-runner.h"
+#define KBOX_HOST_VFD_NONE ((int32_t) -1)
+#define KBOX_HOST_VFD_MULTI ((int32_t) -2)
+
+static void test_fd_table_multi_downgrade_regression(void)
+{
+    struct kbox_fd_table t;
+    long vfd1, vfd2;
+    long host_fd = 42;
+
+    kbox_fd_table_init(&t);
+
+    vfd1 = kbox_fd_table_insert(&t, 10, 0);
+    kbox_fd_table_set_host_fd(&t, vfd1, host_fd);
+
+    ASSERT_EQ(t.host_to_vfd[host_fd], vfd1);
+    ASSERT_EQ(t.host_fd_refs[host_fd], 1);
+
+    /* MULTI state*/
+    vfd2 = kbox_fd_table_insert(&t, 20, 0);
+    kbox_fd_table_set_host_fd(&t, vfd2, host_fd);
+
+    ASSERT_EQ(t.host_to_vfd[host_fd], KBOX_HOST_VFD_MULTI);
+    ASSERT_EQ(t.host_fd_refs[host_fd], 2);
+
+    /* Downgrade back to Single*/
+    kbox_fd_table_remove(&t, vfd2);
+
+    ASSERT_EQ(t.host_to_vfd[host_fd], vfd1);
+    ASSERT_EQ(t.host_fd_refs[host_fd], 1);
+
+    /* Should return the exact vfd */
+    ASSERT_EQ(kbox_fd_table_find_by_host_fd(&t, host_fd), vfd1);
+}
+
+void test_fd_table_refcount_init(void)
+{
+    TEST_REGISTER(test_fd_table_multi_downgrade_regression);
+}

tests/unit/test-runner.c

@@ -90,6 +90,7 @@ void test_pass(void)
 /* External init functions from each test file */
 /* Portable test suites (all hosts) */
 extern void test_fd_table_init(void);
+extern void test_fd_table_refcount_init(void);
 extern void test_path_init(void);
 extern void test_mount_init(void);
 extern void test_cli_init(void);
@@ -120,6 +121,7 @@ int main(int argc, char *argv[])
 
     /* Portable suites */
     test_fd_table_init();
+    test_fd_table_refcount_init();
     test_path_init();
     test_mount_init();
     test_cli_init();