Merge branch 'master' into peter/refactor-access-models

Author: peterargue

.github/workflows/flaky-test-monitor.yml

@@ -55,7 +55,7 @@ jobs:
       matrix:
         targets: ${{ fromJSON(needs.create-dynamic-test-matrix.outputs.dynamic-matrix)}}
     # need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4

access/api.go

@@ -209,44 +209,21 @@ type API interface {
 	// If invalid parameters will be supplied SubscribeBlockDigestsFromLatest will return a failed subscription.
 	SubscribeBlockDigestsFromLatest(ctx context.Context, blockStatus flow.BlockStatus) subscription.Subscription
 
-	// SubscribeTransactionStatusesFromStartBlockID subscribes to transaction status updates for a given transaction ID.
-	// Monitoring begins from the specified block ID. The subscription streams status updates until the transaction
-	// reaches a final state (TransactionStatusSealed or TransactionStatusExpired). When the transaction reaches one of
-	// these final statuses, the subscription will automatically terminate.
+	// SubscribeTransactionStatuses subscribes to transaction status updates for a given transaction ID. Monitoring starts
+	// from the latest block to obtain the current transaction status. If the transaction is already in the final state
+	// ([flow.TransactionStatusSealed] or [flow.TransactionStatusExpired]), all statuses will be prepared and sent to the client
+	// sequentially. If the transaction is not in the final state, the subscription will stream status updates until the transaction
+	// reaches the final state. Once a final state is reached, the subscription will automatically terminate.
 	//
 	// Parameters:
-	//   - ctx: The context to manage the subscription's lifecycle, including cancellation.
-	//   - txID: The identifier of the transaction to monitor.
-	//   - startBlockID: The block ID from which to start monitoring.
-	//   - requiredEventEncodingVersion: The version of event encoding required for the subscription.
-	SubscribeTransactionStatusesFromStartBlockID(ctx context.Context, txID flow.Identifier, startBlockID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription
-
-	// SubscribeTransactionStatusesFromStartHeight subscribes to transaction status updates for a given transaction ID.
-	// Monitoring begins from the specified block height. The subscription streams status updates until the transaction
-	// reaches a final state (TransactionStatusSealed or TransactionStatusExpired). When the transaction reaches one of
-	// these final statuses, the subscription will automatically terminate.
-	//
-	// Parameters:
-	//   - ctx: The context to manage the subscription's lifecycle, including cancellation.
-	//   - txID: The unique identifier of the transaction to monitor.
-	//   - startHeight: The block height from which to start monitoring.
-	//   - requiredEventEncodingVersion: The version of event encoding required for the subscription.
-	SubscribeTransactionStatusesFromStartHeight(ctx context.Context, txID flow.Identifier, startHeight uint64, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription
-
-	// SubscribeTransactionStatusesFromLatest subscribes to transaction status updates for a given transaction ID.
-	// Monitoring begins from the latest block. The subscription streams status updates until the transaction
-	// reaches a final state (TransactionStatusSealed or TransactionStatusExpired). When the transaction reaches one of
-	// these final statuses, the subscription will automatically terminate.
-	//
-	// Parameters:
-	//   - ctx: The context to manage the subscription's lifecycle, including cancellation.
+	//   - ctx: Context to manage the subscription's lifecycle, including cancellation.
 	//   - txID: The unique identifier of the transaction to monitor.
 	//   - requiredEventEncodingVersion: The version of event encoding required for the subscription.
-	SubscribeTransactionStatusesFromLatest(ctx context.Context, txID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription
+	SubscribeTransactionStatuses(ctx context.Context, txID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription
 
-	// SendAndSubscribeTransactionStatuses sends a transaction to the network and subscribes to its status updates.
+	// SendAndSubscribeTransactionStatuses sends a transaction to the execution node and subscribes to its status updates.
 	// Monitoring begins from the reference block saved in the transaction itself and streams status updates until the transaction
-	// reaches a final state (TransactionStatusSealed or TransactionStatusExpired). Once a final status is reached, the subscription
+	// reaches the final state ([flow.TransactionStatusSealed] or [flow.TransactionStatusExpired]). Once the final status has been reached, the subscription
 	// automatically terminates.
 	//
 	// Parameters:

access/mock/api.go

@@ -120,3 +120,14 @@ curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"
 ```
 curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "backfill-tx-error-messages", "data": { "start-height": 340, "end-height": 343, "execution-node-ids":["ec7b934df29248d574ae1cc33ae77f22f0fcf96a79e009224c46374d1837824e", "8cbdc8d24a28899a33140cb68d4146cd6f2f6c18c57f54c299f26351d126919e"] }}'
 ```
+
+### Trigger chunk data pack pebble database checkpoint
+```
+curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "create-chunk-data-packs-checkpoint" }'
+```
+
+### Trigger pebble protocol database checkpoints
+Useful for reading protocol state data from the checkpoints using the read-badger util without stopping the node process.
+```
+curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "create-pebble-checkpoint" }'
+```

admin/README.md

@@ -0,0 +1,59 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"path"
+	"time"
+
+	"github.com/cockroachdb/pebble"
+	"github.com/rs/zerolog/log"
+
+	"github.com/onflow/flow-go/admin"
+	"github.com/onflow/flow-go/admin/commands"
+)
+
+var _ commands.AdminCommand = (*PebbleDBCheckpointCommand)(nil)
+
+// PebbleDBCheckpointCommand creates a checkpoint for pebble database for querying the data
+// while keeping the node alive.
+type PebbleDBCheckpointCommand struct {
+	checkpointDir string
+	dbname        string // dbname is for logging purposes only
+	pebbleDB      *pebble.DB
+}
+
+func NewPebbleDBCheckpointCommand(checkpointDir string, dbname string, pebbleDB *pebble.DB) *PebbleDBCheckpointCommand {
+	return &PebbleDBCheckpointCommand{
+		checkpointDir: checkpointDir,
+		dbname:        dbname,
+		pebbleDB:      pebbleDB,
+	}
+}
+
+func (c *PebbleDBCheckpointCommand) Handler(ctx context.Context, req *admin.CommandRequest) (interface{}, error) {
+	log.Info().Msgf("admintool: creating %v database checkpoint", c.dbname)
+
+	targetDir := nextTmpFolder(c.checkpointDir)
+
+	log.Info().Msgf("admintool: creating %v database checkpoint at: %v", c.dbname, targetDir)
+
+	err := c.pebbleDB.Checkpoint(targetDir)
+	if err != nil {
+		return nil, admin.NewInvalidAdminReqErrorf("failed to create %v pebbledb checkpoint at %v: %w", c.dbname, targetDir, err)
+	}
+
+	log.Info().Msgf("admintool: successfully created %v database checkpoint at: %v", c.dbname, targetDir)
+
+	return fmt.Sprintf("successfully created %v db checkpoint at %v", c.dbname, targetDir), nil
+}
+
+func (c *PebbleDBCheckpointCommand) Validator(req *admin.CommandRequest) error {
+	return nil
+}
+
+func nextTmpFolder(dir string) string {
+	// use timestamp as folder name
+	folderName := time.Now().Format("2006-01-02_15-04-05")
+	return path.Join(dir, folderName)
+}