feat: add dynamic module registration

Author: gfyrag

README.md

@@ -1,23 +1,13 @@
-# Formance Ledger
+<!-- Code generated by gomarkdoc. DO NOT EDIT -->
 
-Formance Ledger (fka numary) is a programmable financial ledger that provides a foundation for money-moving applications. The ledger provides atomic multi-postings transactions and is programmable in [Numscript](doc:machine-instructions), a built-in language dedicated to money movements. It can be used either as a standalone micro-service or as part of the greater Formance Stack, and will shine for apps that require a lot of custom, money-moving code, e.g:
+# ledger
 
-- E-commerce with complex payments flows, payments splitting, such as marketplaces
-- Company-issued currencies systems, e.g. Twitch Bits
-- In-game currencies, inventories and trading systems, e.g. Fortnite V-Bucks
-- Payment gateways using non-standard assets, e.g. learning credits
-- Local currencies and complementary finance
+```go
+import "github.com/formancehq/ledger"
+```
 
-Formance Ledger works as a standalone binary, the latest of which can be downloaded from the [releases page](https://github.com/formancehq/ledger/releases). You can move the binary to any executable path, such as to `/usr/local/bin`. Installations using brew, apt, yum or docker are also [available](https://docs.formance.com/docs/installation-1).
+## Index
 
-## Documentation
 
-You can find the complete Numary documentation at [docs.formance.com](https://docs.formance.com)
 
-## Community
-
-If you need help, want to show us what you built or just hang out and chat about ledgers you are more than welcome on our [Slack](https://bit.ly/formance-slack) - looking forward to see you there!
-
-## Contributing
-
-See [CONTRIBUTING.md](./CONTRIBUTING.md)
+Generated by [gomarkdoc](<https://github.com/princjef/gomarkdoc>)

cmd/config.go

@@ -6,16 +6,24 @@ import (
 	"github.com/spf13/viper"
 )
 
-func LoadConfig[V any](cmd *cobra.Command) (*V, error){
+func LoadConfig[V any](cmd *cobra.Command) (*V, error) {
+	var cfg V
+	if err := MapConfig(cmd, &cfg); err != nil {
+		return nil, err
+	}
+
+	return &cfg, nil
+}
+
+func MapConfig(cmd *cobra.Command, cfg any) error {
 	v := viper.New()
 	if err := v.BindPFlags(cmd.Flags()); err != nil {
-		return nil, fmt.Errorf("binding flags: %w", err)
+		return fmt.Errorf("binding flags: %w", err)
 	}
 
-	var cfg V
-	if err := v.Unmarshal(&cfg); err != nil {
-		return nil, fmt.Errorf("unmarshalling config: %w", err)
+	if err := v.Unmarshal(cfg); err != nil {
+		return fmt.Errorf("unmarshalling config: %w", err)
 	}
 
-	return &cfg, nil
+	return nil
 }

cmd/serve.go

@@ -38,8 +38,6 @@ import (
 )
 
 type ServeConfig struct {
-	WorkerConfiguration `mapstructure:",squash"`
-
 	Bind                        string   `mapstructure:"bind"`
 	BallastSizeInBytes          uint     `mapstructure:"ballast-size"`
 	NumscriptCacheMaxCount      uint     `mapstructure:"numscript-cache-max-count"`
@@ -150,10 +148,13 @@ func NewServeCommand() *cobra.Command {
 			}
 
 			if cfg.WorkerEnabled {
-				options = append(options, worker.NewFXModule(worker.ModuleConfig{
-					Schedule:     cfg.HashLogsBlockCRONSpec,
-					MaxBlockSize: cfg.HashLogsBlockMaxSize,
-				}))
+				workerModule, err := worker.NewFXModule(func(v any) error {
+					return MapConfig(cmd, v)
+				})
+				if err != nil {
+					return fmt.Errorf("creating worker module: %w", err)
+				}
+				options = append(options, workerModule)
 			}
 
 			return service.New(cmd.OutOrStdout(), options...).Run(cmd)

cmd/worker.go

@@ -11,21 +11,37 @@ import (
 	"github.com/formancehq/ledger/internal/worker"
 	"github.com/spf13/cobra"
 	"go.uber.org/fx"
+	"reflect"
+	"strconv"
 )
 
-const (
-	WorkerAsyncBlockHasherMaxBlockSizeFlag = "worker-async-block-hasher-max-block-size"
-	WorkerAsyncBlockHasherScheduleFlag     = "worker-async-block-hasher-schedule"
-)
+func addWorkerFlags(cmd *cobra.Command) {
+	for _, runnerFactory := range worker.AllRunnerFactories() {
+		typeOfRunnerFactory := reflect.TypeOf(runnerFactory)
+		method, _ := typeOfRunnerFactory.MethodByName("CreateRunner")
+		configType := method.Type.In(1)
 
-type WorkerConfiguration struct {
-	HashLogsBlockMaxSize  int    `mapstructure:"worker-async-block-hasher-max-block-size"`
-	HashLogsBlockCRONSpec string `mapstructure:"worker-async-block-hasher-schedule"`
-}
+		for i := 0; i < configType.NumField(); i++ {
+			field := configType.Field(i)
+			fieldTag := field.Tag
+			flag := field.Tag.Get("mapstructure")
+			description := fieldTag.Get("description")
+			defaultValue := fieldTag.Get("default")
 
-func addWorkerFlags(cmd *cobra.Command) {
-	cmd.Flags().Int(WorkerAsyncBlockHasherMaxBlockSizeFlag, 1000, "Max block size")
-	cmd.Flags().String(WorkerAsyncBlockHasherScheduleFlag, "0 * * * * *", "Schedule")
+			switch field.Type.Kind() {
+			case reflect.Int, reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8:
+				defaultValue, err := strconv.ParseInt(defaultValue, 10, 64)
+				if err != nil {
+					panic(err)
+				}
+				cmd.Flags().Int(flag, int(defaultValue), description)
+			case reflect.String:
+				cmd.Flags().String(flag, defaultValue, description)
+			default:
+				panic(fmt.Sprintf("cannot config flag %s as type %T is not handled", flag, field.Type))
+			}
+		}
+	}
 }
 
 func NewWorkerCommand() *cobra.Command {
@@ -38,9 +54,11 @@ func NewWorkerCommand() *cobra.Command {
 				return err
 			}
 
-			cfg, err := LoadConfig[WorkerConfiguration](cmd)
+			workerModule, err := worker.NewFXModule(func(v any) error {
+				return MapConfig(cmd, v)
+			})
 			if err != nil {
-				return fmt.Errorf("loading config: %w", err)
+				return fmt.Errorf("creating worker module: %w", err)
 			}
 
 			return service.New(cmd.OutOrStdout(),
@@ -50,10 +68,7 @@ func NewWorkerCommand() *cobra.Command {
 				otlpmetrics.FXModuleFromFlags(cmd),
 				bunconnect.Module(*connectionOptions, service.IsDebug(cmd)),
 				storage.NewFXModule(storage.ModuleConfig{}),
-				worker.NewFXModule(worker.ModuleConfig{
-					MaxBlockSize: cfg.HashLogsBlockMaxSize,
-					Schedule:     cfg.HashLogsBlockCRONSpec,
-				}),
+				workerModule,
 			).Run(cmd)
 		},
 	}

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/bluele/gcache v0.0.2
 	github.com/dop251/goja v0.0.0-20241009100908-5f46f2705ca3
 	github.com/formancehq/ledger/pkg/client v0.0.0-00010101000000-000000000000
-	github.com/go-chi/chi/v5 v5.2.1
+	github.com/go-chi/chi/v5 v5.2.2
 	github.com/go-chi/cors v1.2.1
 	github.com/google/go-cmp v0.7.0
 	github.com/google/uuid v1.6.0
@@ -34,30 +34,32 @@ require (
 	github.com/spf13/pflag v1.0.6
 	github.com/stoewer/go-strcase v1.3.0
 	github.com/stretchr/testify v1.10.0
-	github.com/uptrace/bun v1.2.13
-	github.com/uptrace/bun/dialect/pgdialect v1.2.13
-	github.com/uptrace/bun/extra/bundebug v1.2.12
+	github.com/uptrace/bun v1.2.14
+	github.com/uptrace/bun/dialect/pgdialect v1.2.14
+	github.com/uptrace/bun/extra/bundebug v1.2.14
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/xo/dburl v0.23.8
-	go.opentelemetry.io/otel v1.36.0
-	go.opentelemetry.io/otel/metric v1.36.0
-	go.opentelemetry.io/otel/sdk/metric v1.36.0
-	go.opentelemetry.io/otel/trace v1.36.0
+	go.opentelemetry.io/otel v1.37.0
+	go.opentelemetry.io/otel/metric v1.37.0
+	go.opentelemetry.io/otel/sdk/metric v1.37.0
+	go.opentelemetry.io/otel/trace v1.37.0
 	go.uber.org/fx v1.24.0
 	go.uber.org/mock v0.5.2
 	golang.org/x/oauth2 v0.30.0
 	golang.org/x/sync v0.15.0
 )
 
 require (
-	github.com/formancehq/go-libs/v3 v3.0.0-20250610201819-3bfa67aab0e2
+	github.com/formancehq/go-libs/v3 v3.0.0-20250710143255-4481f63e09c3
 	github.com/iancoleman/strcase v0.3.0
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/spf13/viper v1.20.1
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
+	github.com/ThreeDotsLabs/watermill-aws v1.0.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.2 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/google/go-tpm v0.9.5 // indirect
@@ -87,20 +89,20 @@ require (
 	github.com/ajg/form v1.5.1 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
 	github.com/aws/aws-msk-iam-sasl-signer-go v1.0.4 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.36.4 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.29.16 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.69 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.31 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.5.12 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.36.5 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.29.17 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.70 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.5.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.16 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.25.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.33.21 // indirect
-	github.com/aws/smithy-go v1.22.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 // indirect
+	github.com/aws/smithy-go v1.22.4 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
@@ -136,11 +138,11 @@ require (
 	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/gorilla/schema v1.4.1 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
@@ -164,7 +166,7 @@ require (
 	github.com/muhlemmer/gu v0.3.1 // indirect
 	github.com/muhlemmer/httpforwarded v0.1.0 // indirect
 	github.com/nats-io/jwt/v2 v2.7.4 // indirect
-	github.com/nats-io/nats-server/v2 v2.11.4 // indirect
+	github.com/nats-io/nats-server/v2 v2.11.5 // indirect
 	github.com/nats-io/nkeys v0.4.11 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
@@ -184,7 +186,7 @@ require (
 	github.com/tklauser/go-sysconf v0.3.15 // indirect
 	github.com/tklauser/numcpus v0.10.0 // indirect
 	github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
-	github.com/uptrace/bun/extra/bunotel v1.2.13 // indirect
+	github.com/uptrace/bun/extra/bunotel v1.2.14 // indirect
 	github.com/uptrace/opentelemetry-go-extra/otellogrus v0.3.2 // indirect
 	github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 // indirect
 	github.com/uptrace/opentelemetry-go-extra/otelutil v0.3.2 // indirect
@@ -198,19 +200,19 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zitadel/oidc/v2 v2.12.2 // indirect
-	go.opentelemetry.io/contrib/instrumentation/host v0.61.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/runtime v0.61.0 // indirect
-	go.opentelemetry.io/contrib/propagators/b3 v1.36.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.36.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0 // indirect
-	go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 // indirect
-	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.36.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/host v0.62.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/runtime v0.62.0 // indirect
+	go.opentelemetry.io/contrib/propagators/b3 v1.37.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.37.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.37.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 // indirect
+	go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.37.0 // indirect
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.37.0 // indirect
 	go.opentelemetry.io/otel/log v0.12.2 // indirect
-	go.opentelemetry.io/otel/sdk v1.36.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.37.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.7.0 // indirect
 	go.uber.org/dig v1.19.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
@@ -220,7 +222,7 @@ require (
 	golang.org/x/net v0.41.0 // indirect
 	golang.org/x/sys v0.33.0 // indirect
 	golang.org/x/text v0.26.0 // indirect
-	golang.org/x/time v0.11.0 // indirect
+	golang.org/x/time v0.12.0 // indirect
 	golang.org/x/tools v0.34.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect