folk2 (parallel Folk evaluator)

.gitignore

@@ -1,6 +1,10 @@
+folk
+folk.pid
+*.o
+folk.dSYM
+CFLAGS
 .DS_Store
-statements.pdf
-statements.dot
-trie.pdf
-trie.dot
-
+play
+.vscode/settings.json
+vendor/jimtcl/conftest__.dSYM/Contents/Info.plist
+vendor/jimtcl/conftest__.dSYM/Contents/Resources/Relocations/aarch64/conftest__.yml

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "live-build/config/includes.chroot_after_packages/etc/skel/apriltag"]
-	path = live-build/config/includes.chroot_after_packages/home/folk/apriltag
-	url = https://github.com/FolkComputer/apriltag
+[submodule "vendor/tracy"]
+	path = vendor/tracy
+	url = https://github.com/wolfpld/tracy.git

Makefile

@@ -1,61 +1,184 @@
-start:
-	tclsh8.6 main.tcl
-debug:
-	gdb --args tclsh8.6 main.tcl
-remote-debug: sync
-	ssh -tt folk@$(FOLK_SHARE_NODE) -- 'sudo systemctl stop folk && make -C /home/folk/folk debug'
-remote-valgrind: sync
-	ssh -tt folk@$(FOLK_SHARE_NODE) -- 'cd folk; sudo systemctl stop folk && valgrind --leak-check=yes tclsh8.6 main.tcl'
+ifeq ($(shell uname -s),Linux)
+	override BUILTIN_CFLAGS += -Wl,--export-dynamic
+endif
 
-FOLK_SHARE_NODE := $(shell tclsh8.6 hosts.tcl shareNode)
+ifneq (,$(filter -DTRACY_ENABLE,$(CFLAGS)))
+# Tracy is enabled
+	TRACY_TARGET = vendor/tracy/public/TracyClient.o
+	override CPPFLAGS += -std=c++20 -DTRACY_ENABLE
+	LINKER := c++
+else
+	LINKER := cc
+endif
 
-sync:
-	rsync --delete --timeout=5 -e "ssh -o StrictHostKeyChecking=no" -a --no-links . folk@$(FOLK_SHARE_NODE):/home/folk/folk
+folk: workqueue.o db.o trie.o sysmon.o epoch.o cache.o folk.o \
+	vendor/c11-queues/mpmc_queue.o vendor/c11-queues/memory.o \
+	vendor/jimtcl/libjim.a $(TRACY_TARGET) CFLAGS
+
+	$(LINKER) -g -fno-omit-frame-pointer $(if $(ASAN_ENABLE),-fsanitize=address -fsanitize-recover=address,) -o$@ \
+		$(CFLAGS) $(BUILTIN_CFLAGS) \
+		-L./vendor/jimtcl \
+		$(filter %.o %.a,$^) \
+		-ljim -lm -lssl -lcrypto -lz
+	if [ "$$(uname)" = "Darwin" ]; then \
+		dsymutil $@; \
+	fi
+	# Hack for the gadget trigger button.
+	if [ "$$(uname)" = "Linux" ]; then \
+		(sudo -n true 2>/dev/null && sudo setcap cap_sys_rawio+ep $@) || true; \
+	fi
 
-sync-restart: sync
-	ssh -tt folk@$(FOLK_SHARE_NODE) -- 'sudo systemctl restart folk'
+%.o: %.c trie.h workqueue.h CFLAGS
+	cc -c -O2 -g -fno-omit-frame-pointer $(if $(ASAN_ENABLE),-fsanitize=address -fsanitize-recover=address,) -o$@  \
+		-D_GNU_SOURCE $(CFLAGS) $(BUILTIN_CFLAGS) \
+		$< -I./vendor/jimtcl -I./vendor/tracy/public
 
-test:
-	for testfile in test/*.tcl; do echo; echo $${testfile}; echo --------; make FOLK_ENTRY=$${testfile}; done
+.PHONY: test clean deps
+test: folk
+	@count=1; \
+	total=$$(ls test/*.folk | wc -l | tr -d ' '); \
+	for test in test/*.folk; do \
+		echo "Running test: $$test ($${count}/$$total)"; \
+		echo "--------------------"; \
+		./folk $$test; \
+		result=$$?; \
+		if [ $$result -eq 0 ]; then \
+			echo "Ran test: $$test ($${count}/$$total): ✅ passed"; \
+		else \
+			echo "Ran test: $$test ($${count}/$$total): ❌ failed"; \
+		fi; \
+		echo ""; \
+		count=$$((count + 1)); \
+	done
+test/%: test/%.folk folk
+	./folk $<
+debug-test/%: test/%.folk folk
+	if [ "$$(uname)" = "Darwin" ]; then \
+		lldb -o "process handle -p true -s false SIGUSR1" -- ./folk $<; \
+	else \
+		gdb -ex "handle SIGUSR1 nostop" -ex "handle SIGPIPE nostop" --args ./folk $<; \
+	fi
 
-test/%.debug:
-	FOLK_ENTRY=test/$*.tcl lldb -- tclsh8.6 main.tcl
+debug: folk
+	if [ "$$(uname)" = "Darwin" ]; then \
+		lldb -o "process handle -p true -s false SIGUSR1" -- ./folk; \
+	else \
+		gdb -ex "handle SIGUSR1 nostop" -ex "handle SIGPIPE nostop" ./folk; \
+	fi
 
-test/%:
-	make [email protected]
+clean:
+	rm -f folk *.o vendor/tracy/public/TracyClient.o vendor/c11-queues/*.o
+distclean: clean
+	make -C vendor/jimtcl distclean
+	make -C vendor/apriltag clean
 
-repl:
-	tclsh8.6 replmain.tcl
+remote-clean: sync
+	ssh $(FOLK_REMOTE_NODE) -- 'cd folk; make clean'
+remote-distclean: sync
+	ssh $(FOLK_REMOTE_NODE) -- 'cd folk; make distclean'
+deps:
+	if [ ! -f vendor/jimtcl/Makefile ]; then \
+		cd vendor/jimtcl && ./configure CFLAGS='-g -fno-omit-frame-pointer'; \
+	fi
+	make -C vendor/jimtcl
+	make -C vendor/apriltag libapriltag.so
+	if [ ! -f vendor/wslay/Makefile ]; then \
+		cd vendor/wslay && autoreconf -i && automake && autoconf && ./configure; \
+	fi
+	make -C vendor/wslay
+	if [ "$$(uname)" = "Darwin" ]; then \
+		install_name_tool -id @executable_path/vendor/apriltag/libapriltag.so vendor/apriltag/libapriltag.so; \
+		install_name_tool -id @executable_path/vendor/wslay/lib/.libs/libwslay.0.dylib vendor/wslay/lib/.libs/libwslay.0.dylib; \
+	fi
 
-journal:
-	ssh folk@$(FOLK_SHARE_NODE) -- journalctl -f -n 100 -u folk
+kill-folk:
+	sudo systemctl stop folk
+	if [ -f folk.pid ]; then \
+		OLD_PID=`cat folk.pid`; \
+		sudo pkill -9 --pgroup $$OLD_PID; \
+		while sudo pkill -0 --pgroup $$OLD_PID; do sleep 0.2; done; \
+	fi
 
-ssh:
-	ssh folk@$(FOLK_SHARE_NODE)
+FOLK_REMOTE_NODE ?= folk-live
+
+sync:
+	ssh $(FOLK_REMOTE_NODE) -t \
+		'cd ~/folk && git init > /dev/null && git ls-files --exclude-standard -oi --directory' \
+		> .git/ignores.tmp || true
+	git ls-files --exclude-standard -oi --directory >> .git/ignores.tmp
+	rsync --timeout=15 -e "ssh -o StrictHostKeyChecking=no" \
+		--archive --delete --itemize-changes \
+		--exclude='/.git' \
+		--exclude-from='.git/ignores.tmp' \
+		--include='vendor/tracy/public/***' \
+		--exclude='vendor/tracy/*' \
+		./ $(FOLK_REMOTE_NODE):~/folk/
+
+remote-setup:
+	ssh-copy-id $(FOLK_REMOTE_NODE)
+	make sync
+	ssh $(FOLK_REMOTE_NODE) -- 'sudo usermod -a -G tty folk && chmod +rwx ~/folk-calibration-poses; sudo apt update && sudo apt install libssl-dev gdb libwslay-dev google-perftools libgoogle-perftools-dev linux-perf && cd folk/vendor/jimtcl && make distclean; ./configure CFLAGS="-g -fno-omit-frame-pointer"'
+
+remote: sync
+	ssh $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk; make deps && make start CFLAGS="$(CFLAGS)" ASAN_ENABLE=$(ASAN_ENABLE)'
+sudo-remote: sync
+	ssh -tt $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk; make deps && make CFLAGS="$(CFLAGS)" && sudo HOME=/home/folk TRACY_SAMPLING_HZ=31000 ./folk'
+debug-remote: sync
+	ssh $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk; make deps && make CFLAGS="$(CFLAGS)" && gdb -ex "handle SIGUSR1 nostop" -ex "handle SIGPIPE nostop" ./folk'
+debug-sudo-remote: sync
+	ssh -tt $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk; make deps && make CFLAGS="$(CFLAGS)" && sudo HOME=/home/folk TRACY_SAMPLING_HZ=31000 gdb -ex "handle SIGUSR1 nostop" -ex "handle SIGPIPE nostop"  ./folk'
+valgrind-remote: sync
+	ssh $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk; make deps && make && valgrind --leak-check=yes ./folk'
+heapprofile-remote: sync
+	ssh $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk; make deps && make CFLAGS="$(CFLAGS)" && env LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc.so HEAPPROFILE=/tmp/folk.hprof PERFTOOLS_VERBOSE=-1 ./folk'
+debug-heapprofile-remote: sync
+	ssh $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk; make deps && make CFLAGS="$(CFLAGS)" && env LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc.so HEAPPROFILE=/tmp/folk.hprof PERFTOOLS_VERBOSE=-1 gdb -ex "handle SIGUSR1 nostop" ./folk'
+heapprofile-remote-show:
+	ssh $(FOLK_REMOTE_NODE) -- 'cd folk; google-pprof --text folk $(HEAPPROFILE)'
+heapprofile-remote-svg:
+	ssh $(FOLK_REMOTE_NODE) -- 'cd folk; google-pprof --svg folk $(HEAPPROFILE)' > out.svg
 
-FLAMEGRAPH_TID := $(shell pgrep tclsh8.6 | head -1)
 flamegraph:
-	sudo perf record -F 997 --tid=$(FLAMEGRAPH_TID) -g -- sleep 30
+	sudo perf record --freq=997 --call-graph lbr --pid=$(shell cat folk.pid) -g -- sleep 30
 	sudo perf script -f > out.perf
 	~/FlameGraph/stackcollapse-perf.pl out.perf > out.folded
 	~/FlameGraph/flamegraph.pl out.folded > out.svg
 
-# You can use the Web server to check the pid of display.folk,
-# apriltags.folk, camera.folk, etc.
 remote-flamegraph:
-	ssh -t folk@$(FOLK_SHARE_NODE) -- make -C /home/folk/folk flamegraph $(if $(REMOTE_FLAMEGRAPH_TID),FLAMEGRAPH_TID=$(REMOTE_FLAMEGRAPH_TID),)
-	scp folk@$(FOLK_SHARE_NODE):~/folk/out.svg .
-	scp folk@$(FOLK_SHARE_NODE):~/folk/out.perf .
-
-backup-printed-programs:
-	cd ~/folk-printed-programs && timestamp=$$(date '+%Y-%m-%d_%H-%M-%S%z') && tar -zcvf ~/"folk-printed-programs_$$timestamp.tar.gz" . && echo "Saved to: ~/folk-printed-programs_$$timestamp.tar.gz"
+	ssh -t $(FOLK_REMOTE_NODE) -- make -C folk flamegraph
+	scp $(FOLK_REMOTE_NODE):~/folk/out.svg .
+	scp $(FOLK_REMOTE_NODE):~/folk/out.perf .
 
-.PHONY: test sync start journal repl enable-pubkey install-deps
+start: folk
+	@if [ -n "$$(systemctl list-unit-files | grep folk.service)" ] && \
+	   [ -n "$$(systemctl cat folk.service | grep "ExecStart.*$$(pwd)")" ] && \
+	   [ -z "$(ENABLE_ASAN)" ] && \
+	   [ -z "$$INVOCATION_ID" ] && \
+	   [ -z "$(CFLAGS)" ] ; then \
+		sudo systemctl start folk.service; \
+		journalctl -f -u folk.service; \
+	else \
+		$(if $(ENABLE_ASAN),ASAN_OPTIONS=detect_leaks=1:halt_on_error=0,) ./folk; \
+	fi
 
-enable-pubkey:
-	ssh folk-live -- 'sudo sed -i "s/.*PubkeyAuthentication.*/PubkeyAuthentication yes/g" /etc/ssh/sshd_config && sudo systemctl restart ssh'
-	sleep 1
-	ssh-copy-id folk-live
+run-tracy:
+	vendor/tracy/profiler/build/tracy-profiler
+tracy-remote:
+	ssh $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk'
+	vendor/tracy/profiler/build/tracy-profiler -a `ssh -G $(FOLK_REMOTE_NODE) | awk '$$1 == "hostname" { print $$2 }'` & \
+		make remote CFLAGS=-DTRACY_ENABLE FOLK_REMOTE_NODE=$(FOLK_REMOTE_NODE)
+sudo-tracy-remote:
+	ssh $(FOLK_REMOTE_NODE) -- 'cd folk; make kill-folk'
+	vendor/tracy/profiler/build/tracy-profiler -a `ssh -G $(FOLK_REMOTE_NODE) | awk '$$1 == "hostname" { print $$2 }'` & \
+		make sudo-remote CFLAGS=-DTRACY_ENABLE FOLK_REMOTE_NODE=$(FOLK_REMOTE_NODE)
 
-install-deps:
-	sudo apt install console-data
+# From https://stackoverflow.com/a/26147844 to force rebuild if CFLAGS
+# changes (in particular, so we rebuild if we want to use Tracy)
+define DEPENDABLE_VAR
+.PHONY: phony
+$1: phony
+	@if [ "`cat $1 2>&1`" != '$($1)' ]; then \
+		/bin/echo -n $($1) > $1 ; \
+	fi
+endef
+$(eval $(call DEPENDABLE_VAR,CFLAGS))