Go/Core Refactore: Move FFI to a dedicated folder to be reusable for other wrappers

barshaul · barshaul · commit c67ad2a220db · 2025-03-16T16:13:40.000Z
Signed-off-by: barshaul &lt;barshaul@amazon.com&gt;
diff --git a/.github/workflows/ffi.yml b/.github/workflows/ffi.yml
@@ -0,0 +1,45 @@
+name: Glide FFI CI
+
+on:
+    push:
+        branches:
+            - main
+            - release-*
+            - v*
+        paths:
+            - glide-core/src/**
+            - glide-core/redis-rs/redis/src/**
+            - ffi/**
+    pull_request:
+        paths:
+            - glide-core/src/**
+            - glide-core/redis-rs/redis/src/**
+            - ffi/**
+    workflow_call:
+
+concurrency:
+    group: ffi-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }}
+    cancel-in-progress: true
+
+env:
+    CARGO_TERM_COLOR: always
+
+jobs:
+    build-and-gen-c:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+            - name: Install Rust and protoc
+              uses: ./.github/workflows/install-rust-and-protoc
+              with:
+                  github-token: ${{ secrets.GITHUB_TOKEN }}
+
+            - name: Build the FFI library
+              working-directory: ffi
+              run: |
+                  cargo build --release
+            - name: Generate the C header file
+              working-directory: ffi
+              run: |
+                  cargo install cbindgen
+                  cbindgen --config cbindgen.toml --crate glide-ffi --output lib.h --lang c
diff --git a/.github/workflows/go-cd.yml b/.github/workflows/go-cd.yml
@@ -119,16 +119,14 @@ jobs:
               working-directory: go
               run: |
                   make install-build-tools
-                  make build-glide-client
-                  # TODO: move generation of protobuf and header file to a non-matrix step
-                  make generate-protobuf
+                  make build
             - name: Upload artifacts
               continue-on-error: true
               uses: actions/upload-artifact@v4
               with:
                   name: ${{ matrix.host.TARGET }}
                   path: |
-                      go/target/release/libglide_rs.a
+                      ffi/target/release/libglide_ffi.a
                       go/lib.h
                       go/protobuf/
 
@@ -149,7 +147,7 @@ jobs:
                   for dir in */; do
                     target_name=${dir%/}
                     mkdir -p $GITHUB_WORKSPACE/go/rustbin/${target_name}
-                    cp ${target_name}/target/release/libglide_rs.a $GITHUB_WORKSPACE/go/rustbin/${target_name}/
+                    cp ${target_name}/target/release/libglide_ffi.a $GITHUB_WORKSPACE/go/rustbin/${target_name}/
                   done
                   # TODO: move generation of protobuf and header file to a non-matrix step
                   cd x86_64-unknown-linux-gnu
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -9,6 +9,7 @@ on:
         paths:
             - glide-core/src/**
             - glide-core/redis-rs/redis/src/**
+            - ffi/src/**
             - utils/cluster_manager.py
             - go/**
             - .github/workflows/go.yml
@@ -22,6 +23,7 @@ on:
         paths:
             - glide-core/src/**
             - glide-core/redis-rs/redis/src/**
+            - ffi/src/**
             - utils/cluster_manager.py
             - go/**
             - .github/workflows/go.yml
@@ -133,11 +135,6 @@ jobs:
         steps:
             - uses: actions/checkout@v4
 
-            - uses: ./.github/workflows/lint-rust
-              with:
-                  cargo-toml-folder: go
-                  github-token: ${{ secrets.GITHUB_TOKEN }}
-
             - name: Set up Go ${{ matrix.go }}
               uses: actions/setup-go@v5
               with:
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -10,6 +10,7 @@ on:
             - logger_core/**
             - glide-core/**
             - glide-core/redis-rs/redis/src/**
+            - ffi/**
             - utils/cluster_manager.py
             - .github/workflows/rust.yml
             - .github/workflows/install-shared-dependencies/action.yml
@@ -23,6 +24,7 @@ on:
             - logger_core/**
             - glide-core/**
             - glide-core/redis-rs/redis/src/**
+            - ffi/**
             - utils/cluster_manager.py
             - .github/workflows/rust.yml
             - .github/workflows/install-shared-dependencies/action.yml
@@ -127,6 +129,12 @@ jobs:
                   github-token: ${{ secrets.GITHUB_TOKEN }}
               name: lint glide-core
 
+            - uses: ./.github/workflows/lint-rust
+              with:
+                  cargo-toml-folder: ./ffi
+                  github-token: ${{ secrets.GITHUB_TOKEN }}
+              name: lint glide-ffi
+
             - uses: ./.github/workflows/lint-rust
               with:
                   cargo-toml-folder: ./logger_core
diff --git a/ffi/.cargo/config.toml b/ffi/.cargo/config.toml
@@ -1,5 +1,5 @@
 [env]
-GLIDE_NAME = { value = "GlideGo", force = true }
+GLIDE_NAME = { value = "GlideFFI", force = true }
 GLIDE_VERSION = "0.1.0"
 # Suppress error
 # > ... was built for newer 'macOS' version (14.5) than being linked (14.0)
diff --git a/ffi/.gitignore b/ffi/.gitignore
@@ -0,0 +1,2 @@
+# cbindgen generated header file
+lib.h
diff --git a/ffi/Cargo.toml b/ffi/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "glide-rs"
+name = "glide-ffi"
 version = "0.1.0"
 edition = "2021"
 license = "Apache-2.0"
@@ -9,10 +9,10 @@ authors = ["Valkey GLIDE Maintainers"]
 crate-type = ["staticlib"]
 
 [dependencies]
-redis = { path = "../glide-core/redis-rs/redis", features = ["aio", "tokio-comp", "connection-manager", "tokio-rustls-comp"] }
+protobuf = { version = "3", features = [] }
+redis = { path = "../glide-core/redis-rs/redis", features = ["aio", "tokio-comp", "tokio-rustls-comp"] }
 glide-core = { path = "../glide-core", features = ["proto"] }
 tokio = { version = "^1", features = ["rt", "macros", "rt-multi-thread", "time"] }
-protobuf = { version = "3.3.0", features = [] }
 
 [profile.release]
 lto = true
diff --git a/ffi/README.md b/ffi/README.md
@@ -0,0 +1,31 @@
+# glide_ffi
+
+## Overview
+
+The glide_ffi crate provides a C-compatible Foreign Function Interface (FFI) for interacting with `glide-core`. It serves as a bridge for internal wrappers written in other languages, such as Go, to integrate seamlessly with `glide-core`.
+
+## Structure
+	•	src/lib.rs: Defines the FFI interface.
+	•	cbindgen.toml: Configuration for generating the C header file (lib.h).
+	•	Cargo.toml: Manages dependencies, including glide-core.
+
+## Building the Library
+
+To build the FFI library:
+```
+cargo build --release
+```
+
+To generate the C header file:
+```
+cargo install cbindgen
+cbindgen --config cbindgen.toml --crate glide-ffi --output lib.h --lang c
+```
+
+## Running the linters
+
+```bash
+rustup component add clippy rustfmt
+cargo clippy --all-features --all-targets -- -D warnings
+cargo fmt --manifest-path ./Cargo.toml --all
+```
diff --git a/ffi/cbindgen.toml b/ffi/cbindgen.toml
diff --git a/ffi/src/lib.rs b/ffi/src/lib.rs
diff --git a/go/DEVELOPER.md b/go/DEVELOPER.md
@@ -202,33 +202,18 @@ Development on the Go wrapper may involve changes in either the Go or Rust code.
 - staticcheck
 - golines
 
-**Rust:**
-
-- clippy
-- fmt
-
 #### Running the linters
 
 Run from the main `/go` folder
 
-1. Go
-
-    ```bash
-    # For go1.22:
-    make install-dev-tools
-    # For go1.22:
-    make install-dev-tools-go1.22
-
-    make lint
-    ```
-
-2. Rust
+```bash
+# For go1.22:
+make install-dev-tools
+# For go1.22:
+make install-dev-tools-go1.22
 
-    ```bash
-    rustup component add clippy rustfmt
-    cargo clippy --all-features --all-targets -- -D warnings
-    cargo fmt --manifest-path ./Cargo.toml --all
-    ```
+make lint
+```
 
 #### Fixing lint formatting errors
 
diff --git a/go/Makefile b/go/Makefile
@@ -1,6 +1,10 @@
 # Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0
 
 SHELL:=/bin/bash
+GO_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+ROOT_FOLDER := $(abspath $(GO_DIR)/..)
+GLIDE_CORE_FOLDER := $(ROOT_FOLDER)/glide-core
+GLIDE_FFI_FOLDER := $(ROOT_FOLDER)/ffi
 
 install-build-tools:
 	go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.33.0
@@ -42,51 +46,58 @@ clean:
 	rm -rf target
 	rm -rf reports
 
-build-glide-client:
-	cargo build --release
-	cbindgen --config cbindgen.toml --crate glide-rs --output lib.h --lang c
+build-glide-ffi:
+	cd $(GLIDE_FFI_FOLDER) && cargo build $(BUILD_FLAGS)
+	$(MAKE) gen-c-bindings
+
+# Release build
+build-glide-client: BUILD_FLAGS=--release
+build-glide-client: build-glide-ffi
+
+# Debug build (no --release flag)
+build-glide-client-debug: BUILD_FLAGS=
+build-glide-client-debug: build-glide-ffi
+
+gen-c-bindings:
+	cd $(GLIDE_FFI_FOLDER) && \
+	cbindgen --config cbindgen.toml --crate glide-ffi --output $(GO_DIR)/lib.h --lang c
 	@if [ "$$(uname)" = "Darwin" ]; then \
-		strip -x target/release/libglide_rs.a; \
+		strip -x $(GLIDE_FFI_FOLDER)/target/release/libglide_ffi.a; \
 		if [ "$$(uname -m)" = "arm64" ]; then \
 			echo "Platform: aarch64-apple-darwin"; \
 			mkdir -p rustbin/aarch64-apple-darwin; \
-			cp target/release/libglide_rs.a rustbin/aarch64-apple-darwin/; \
+			cp $(GLIDE_FFI_FOLDER)/target/release/libglide_ffi.a rustbin/aarch64-apple-darwin/; \
 		elif [ "$$(uname -m)" = "x86_64" ]; then \
 			echo "Platform: x86_64-apple-darwin"; \
 			mkdir -p rustbin/x86_64-apple-darwin; \
-			cp target/release/libglide_rs.a rustbin/x86_64-apple-darwin/; \
+			cp $(GLIDE_FFI_FOLDER)/target/release/libglide_ffi.a rustbin/x86_64-apple-darwin/; \
 		fi; \
 	elif [ "$$(uname)" = "Linux" ]; then \
-		strip --strip-unneeded target/release/libglide_rs.a; \
+		strip --strip-unneeded $(GLIDE_FFI_FOLDER)/target/release/libglide_ffi.a; \
 		if [ "$$(uname -m)" = "arm64" ] || [ "$$(uname -m)" = "aarch64" ]; then \
 			echo "Platform: aarch64-unknown-linux-gnu"; \
 			mkdir -p rustbin/aarch64-unknown-linux-gnu; \
-			cp target/release/libglide_rs.a rustbin/aarch64-unknown-linux-gnu/; \
+			cp $(GLIDE_FFI_FOLDER)/target/release/libglide_ffi.a rustbin/aarch64-unknown-linux-gnu/; \
 		elif [ "$$(uname -m)" = "x86_64" ]; then \
 			echo "Platform: x86_64-unknown-linux-gnu"; \
 			mkdir -p rustbin/x86_64-unknown-linux-gnu; \
-			cp target/release/libglide_rs.a rustbin/x86_64-unknown-linux-gnu/; \
+			cp $(GLIDE_FFI_FOLDER)/target/release/libglide_ffi.a rustbin/x86_64-unknown-linux-gnu/; \
 		fi; \
 	else \
 		echo "Unsupported platform: $(shell uname), $(shell uname -m)"; \
 		exit 1; \
 	fi
 
-
-build-glide-client-debug:
-	cargo build
-	cbindgen --config cbindgen.toml --crate glide-rs --output lib.h --lang c
-
 generate-protobuf:
 	rm -rf protobuf
 	mkdir -p protobuf
-	protoc --proto_path=../glide-core/src/protobuf \
+	protoc --proto_path=$(GLIDE_CORE_FOLDER)/src/protobuf \
 		--go_opt=Mconnection_request.proto=github.com/valkey-io/valkey-glide/go/protobuf \
 		--go_opt=Mcommand_request.proto=github.com/valkey-io/valkey-glide/go/protobuf \
 		--go_opt=Mresponse.proto=github.com/valkey-io/valkey-glide/go/protobuf \
 		--go_out=./protobuf \
 		--go_opt=paths=source_relative \
-		../glide-core/src/protobuf/*.proto
+		$(GLIDE_CORE_FOLDER)/src/protobuf/*.proto
 
 lint:
 	go vet ./...
@@ -116,7 +127,7 @@ format:
 unit-test:
 	mkdir -p reports
 	set -o pipefail; \
-	LD_LIBRARY_PATH=$(shell find . -name libglide_rs.a|grep -w release|tail -1|xargs dirname|xargs readlink -f):${LD_LIBRARY_PATH} \
+	LD_LIBRARY_PATH=$(shell find . -name libglide_ffi.a|grep -w release|tail -1|xargs dirname|xargs readlink -f):${LD_LIBRARY_PATH} \
 	go test -v ./... -skip 'Example|TestGlideTestSuite' -run Test $(if $(test-filter), -run $(test-filter)) \
 	| tee >(go tool test2json -t -p github.com/valkey-io/valkey-glide/go/utils | go-test-report -o reports/unit-tests.html -t unit-test > /dev/null)
 
@@ -131,7 +142,7 @@ example-test:
 	CLUSTER_OUTPUT=$$(python3 ../utils/cluster_manager.py start --cluster-mode); \
 	GLIDE_CLUSTER_NODES=$$(echo "$$CLUSTER_OUTPUT" | grep 'CLUSTER_NODES=' | cut -d'=' -f2); \
 	echo "GLIDE_CLUSTER_NODES=$$GLIDE_CLUSTER_NODES"; \
-	LD_LIBRARY_PATH=$(shell find . -name libglide_rs.a|grep -w release|tail -1|xargs dirname|xargs readlink -f):${LD_LIBRARY_PATH} \
+	LD_LIBRARY_PATH=$(shell find . -name libglide_ffi.a|grep -w release|tail -1|xargs dirname|xargs readlink -f):${LD_LIBRARY_PATH} \
 	go test -v ./api -skip Test $(if $(test-filter), -run $(test-filter)) -clusternodes $$GLIDE_CLUSTER_NODES -standalonenode $$GLIDE_NODE \
 	| tee >(go tool test2json -t -p github.com/valkey-io/valkey-glide/go/api \
 	| go-test-report -o reports/example-tests.html -t example-test > /dev/null)
@@ -147,7 +158,7 @@ modules-test: __it
 __it:
 	mkdir -p reports
 	set -o pipefail; \
-	LD_LIBRARY_PATH=$(shell find . -name libglide_rs.a|grep -w release|tail -1|xargs dirname|xargs readlink -f):${LD_LIBRARY_PATH} \
+	LD_LIBRARY_PATH=$(shell find . -name libglide_ffi.a|grep -w release|tail -1|xargs dirname|xargs readlink -f):${LD_LIBRARY_PATH} \
 	go test -v ./integTest/... \
 	$(TEST_FILTER) \
 	$(if $(filter true, $(tls)), --tls,) \
diff --git a/go/api/base_client.go b/go/api/base_client.go
@@ -2,7 +2,7 @@
 
 package api
 
-// #cgo LDFLAGS: -lglide_rs
+// #cgo LDFLAGS: -lglide_ffi
 // #cgo !windows LDFLAGS: -lm
 // #cgo darwin LDFLAGS: -framework Security
 // #cgo darwin,amd64 LDFLAGS: -framework CoreFoundation

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+# cbindgen generated header file`
	`2`	`+lib.h`