WIP libn00b

.github/workflows/tests.yml

@@ -11,7 +11,9 @@ on:
       - "*.nimble"
       # ignore docs not to waste CI minutes
       - "!src/docs/**"
-  pull_request:
+  pull_request_target:
+    branches:
+      - main
     types:
       - opened
       - synchronize

.github/workflows/unit-tests.yml

@@ -5,14 +5,16 @@ on:
     branches:
       - main
     paths-ignore:
-      - '**.md'
-      - 'server/**'
-      - 'tests/functional/**'
-  pull_request:
+      - "**.md"
+      - "server/**"
+      - "tests/functional/**"
+  pull_request_target:
+    branches:
+      - main
     paths-ignore:
-      - '**.md'
-      - 'server/**'
-      - 'tests/functional/**'
+      - "**.md"
+      - "server/**"
+      - "tests/functional/**"
   workflow_dispatch:
 
 permissions:

Dockerfile

@@ -1,11 +1,25 @@
-FROM ghcr.io/crashappsec/nim:ubuntu-2.0.8 as nim
-FROM ghcr.io/sigstore/cosign/cosign:v2.2.3 as cosign
+ARG BASE=alpine
+FROM ghcr.io/sigstore/cosign/cosign:v2.2.3 AS cosign
 
 # -------------------------------------------------------------------
 
-FROM nim as deps
+FROM ghcr.io/crashappsec/nim:alpine-2.0.8 AS alpine
+
+RUN apk add --no-cache \
+    bash \
+    curl \
+    make \
+    musl-dev \
+    openssl \
+    strace
+
+# add musl-gcc so its consistent CC with ubuntu
+RUN ln -s $(which gcc) /usr/bin/musl-gcc
+
+# -------------------------------------------------------------------
+
+FROM ghcr.io/crashappsec/nim:ubuntu-2.0.8 AS ubuntu
 
-# curl - chalk downloads some things directly with curl for the moment
 RUN apt-get update -y && \
     apt-get install -y \
         curl \
@@ -15,6 +29,10 @@ RUN apt-get update -y && \
         && \
     apt-get clean -y
 
+# -------------------------------------------------------------------
+
+FROM $BASE AS deps
+
 # XXX this is needed for the github worker
 # https://github.com/actions/runner/issues/2033
 RUN if which git; then git config --global --add safe.directory "*"; fi
@@ -31,12 +49,12 @@ COPY src/config_version.nim /chalk/src/
 RUN mkdir -p src/configs && \
     echo 'chalk_version := "0.0.0"' > src/configs/base_keyspecs.c4m && \
     touch src/chalk.nim && \
-    nimble build
+    nimble build --verbose
 
 # -------------------------------------------------------------------
 # build chalk binary to be copied into final release stage
 
-FROM deps as build
+FROM deps AS build
 
 ARG CHALK_BUILD="release"
 

Makefile

@@ -16,6 +16,8 @@ SOURCES+=$(shell find ../con4m -name '*.nim' 2> /dev/null)
 SOURCES+=$(shell find ../con4m -name '*.c4m' 2> /dev/null)
 SOURCES+=$(shell find ../nimutils -name '*.nim' 2> /dev/null)
 SOURCES+=$(shell find ../nimutils -name '*.c' 2> /dev/null)
+SOURCES+=$(shell find ../n00b -name '*.c' 2> /dev/null)
+SOURCES+=$(shell find ../n00b -name '*.h' 2> /dev/null)
 SOURCES+=src/docs/CHANGELOG.md
 
 VERSION=$(shell cat src/configs/base_keyspecs.c4m \
@@ -57,11 +59,11 @@ version:
 
 .PHONY: clean
 clean:
-	-rm -rf $(BINARY) $(BINARY).bck dist nimutils con4m nimble.develop
+	-$(DOCKER) rm -rf $(BINARY) $(BINARY).bck dist nimutils con4m nimble.develop
 
 .PHONY: chalk-docs
 chalk-docs: $(BINARY)
-	rm -rf $@
+	$(DOCKER) rm -rf $@
 	$(DOCKER) ./$(BINARY) docgen
 
 # devmode for local deps
@@ -74,15 +76,15 @@ nimutils con4m::
 	# It does not like dep structure but it does create the folder
 	# and nimble build does honor it :shrug:
 	-$(DOCKER) nimble develop --add https://github.com/crashappsec/$@
-	cp -r ../$@/* $@
+	$(DOCKER) cp -r ../$@/* $@
 
 nimutils::
-	rm -rf $@/nimutils
-	cd $@ && ln -fs ../../$@/nimutils .
+	$(DOCKER) rm -rf $@/nimutils
+	$(DOCKER) ln -fs ../../$@/nimutils nimutils/$@
 
 con4m::
-	rm -rf $@/files
-	cd $@ && ln -fs ../../$@/files .
+	$(DOCKER) rm -rf $@/files
+	$(DOCKER) ln -fs ../../$@/files con4m/$@
 
 # ----------------------------------------------------------------------------
 # TOOL MAKEFILES

config.nims

@@ -16,7 +16,23 @@ var
   default  = getEnv("HOME").joinPath(".local/c0")
   localDir = getEnv("LOCAL_INSTALL_DIR", default)
   libDir   = localdir.joinPath("libs")
-  libs     = ["pcre", "ssl", "crypto", "gumbo", "hatrack"]
+  libs     = [
+    "n00b",
+    "curl",
+    "ssl",
+    "crypto",
+    "pcre",
+    "backtrace",
+    "ffi",
+    "unibreak",
+    "utf8proc",
+    "gumbo",
+  ]
 
 applyCommonLinkOptions()
-staticLinkLibraries(libs, libDir, muslBase = localDir)
+staticLinkLibraries(
+  libs,
+  libDir,
+  muslBase = localDir,
+  useMusl = true,
+)

docker-compose.yml

@@ -15,15 +15,49 @@ services:
     build:
       context: .
       target: deps
+      args:
+        BASE: ${BASE:-alpine}
     command: nimble ${CHALK_BUILD:-release}
     working_dir: $PWD
+    environment:
+      DEBUG: ${DEBUG:-}
+      CHALK_PASSWORD: ${CHALK_PASSWORD:-}
     volumes:
       - $PWD:$PWD
       - $PWD/../nimutils:$PWD/../nimutils
       - $PWD/../con4m:$PWD/../con4m
-    environment:
-      DEBUG: ${DEBUG:-}
-      CHALK_PASSWORD: ${CHALK_PASSWORD:-}
+      # n00b libs
+      - ../n00b/build_dev/libn00b.a:/root/.local/c0/libs/libn00b.a
+      - ../n00b/build_dev/libn00b.a.p:/root/.local/c0/libs/libn00b.a.p
+      - ../n00b/build_dev/subprojects/utf8proc-2.9.0/libutf8proc.a:/root/.local/c0/libs/libutf8proc.a
+      - ../n00b/subprojects/libffi-3.4.6/x86_64-pc-linux-musl/.libs/libffi.a:/root/.local/c0/libs/libffi.a
+      - ../n00b/subprojects/libbacktrace-531aec7c52b66cd750a28a698f3c060f279b18b0/.libs/libbacktrace.a:/root/.local/c0/libs/libbacktrace.a
+      - ../n00b/subprojects/libunibreak-5.1/src/.libs/libunibreak.a:/root/.local/c0/libs/libunibreak.a
+      - ../n00b/subprojects/curl-8.9.1/lib/.libs/libcurl.a:/root/.local/c0/libs/libcurl.a
+      - ../n00b/subprojects/openssl-3.4.0/libssl.a:/root/.local/c0/libs/libssl.a
+      - ../n00b/subprojects/openssl-3.4.0/libcrypto.a:/root/.local/c0/libs/libcrypto.a
+      # n00b headers
+      - ../n00b/include/n00b.h:/root/.local/c0/include/n00b.h
+      - ../n00b/include/hatrack.h:/root/.local/c0/include/hatrack.h
+      - ../n00b/include/quark.h:/root/.local/c0/include/quark.h
+      - ../n00b/include/vendor.h:/root/.local/c0/include/vendor.h
+      - ../n00b/include/adts/:/root/.local/c0/include/adts/
+      - ../n00b/include/compiler/:/root/.local/c0/include/compiler/
+      - ../n00b/include/core/:/root/.local/c0/include/core/
+      - ../n00b/include/crypto/:/root/.local/c0/include/crypto/
+      - ../n00b/include/hatrack/:/root/.local/c0/include/hatrack/
+      - ../n00b/include/io/:/root/.local/c0/include/io/
+      - ../n00b/include/n00b/:/root/.local/c0/include/n00b/
+      - ../n00b/include/util/:/root/.local/c0/include/util/
+      - ../n00b/include/vendor/:/root/.local/c0/include/vendor/
+      # other headers
+      - ../n00b/subprojects/libbacktrace-531aec7c52b66cd750a28a698f3c060f279b18b0/backtrace.h:/root/.local/c0/include/backtrace.h
+      - ../n00b/subprojects/curl-8.9.1/include/curl/:/root/.local/c0/include/curl/
+      - ../n00b/subprojects/libunibreak-5.1/src/linebreak.h:/root/.local/c0/include/linebreak.h
+      - ../n00b/subprojects/libunibreak-5.1/src/unibreakbase.h:/root/.local/c0/include/unibreakbase.h
+      - ../n00b/subprojects/utf8proc-2.9.0/utf8proc.h:/root/.local/c0/include/utf8proc.h
+      # nimutils
+      - ../nimutils/nimutils/c/md4nim.h:/root/.local/c0/include/md4nim.h
 
   # --------------------------------------------------------------------------
   # SERVER

src/chalk.nim

@@ -12,9 +12,10 @@ when cprofiling:
 # Note that imports cause topics and plugins to register.
 {.warning[UnusedImport]: off.}
 import "."/[config, confload, commands, norecurse, sinks,
-            attestation_api, util]
+            attestation_api, util, n00b/init]
 
 when isMainModule:
+  setupN00b()
   setupSignalHandlers() # util.nim
   setupTerminal()       # util.nim
   ioSetup()             # sinks.nim

src/n00b/init.nim

@@ -0,0 +1,17 @@
+proc n00bInit(
+  argc: cint,
+  argv: pointer,
+  envp: pointer,
+) {.importc:"n00b_init".}
+
+proc n00bInstallDefaultStyles() {.importc:"n00b_install_default_styles".}
+
+proc setupN00b*() =
+  let argv = @[cstring("chalk"), cast[cstring](nil)]
+  let envp = @[cast[cstring](nil)]
+  n00bInit(
+    1,
+    addr(argv[0]),
+    addr(envp[0]),
+  )
+  n00bInstallDefaultStyles()