refactor: improve transaction parsing logic and clean up code

Author: sergeytimoshin

src/ingester/parser/mod.rs

@@ -63,19 +63,55 @@ pub fn parse_transaction(tx: &TransactionInfo, slot: u64) -> Result<StateUpdate,
             is_compression_transaction = true;
             state_updates.push(state_update);
         } else {
-            for (index, _) in ordered_instructions.iter().enumerate() {
-                if ordered_instructions.len() - index > 2 {
-                    if let Some(state_update) = parse_public_transaction_event_v1(
-                        tx,
-                        slot,
-                        &ordered_instructions[index],
-                        &ordered_instructions[index + 1],
-                        &ordered_instructions[index + 2],
-                    )? {
-                        is_compression_transaction = true;
-                        state_updates.push(state_update);
+            for (index, instruction) in ordered_instructions.iter().enumerate() {
+                if ordered_instructions.len() - index > 1 {
+                    if get_compression_program_id() == instruction.program_id {
+                        // Look for a NOOP_PROGRAM_ID instruction after one or two SYSTEM_PROGRAM instructions
+                        // We handle up to two system program instructions in the case where we also have to pay a tree rollover fee
+                        let mut noop_instruction_index = None;
+                        let mut system_program_count = 0;
+                        let mut all_intermediate_are_system = true;
+
+                        // Search for the NOOP instruction, ensuring we find at least one SYSTEM_PROGRAM but no more than two
+                        for i in (index + 1)..ordered_instructions.len() {
+                            let current_instruction = &ordered_instructions[i];
+
+                            if current_instruction.program_id == NOOP_PROGRAM_ID {
+                                noop_instruction_index = Some(i);
+                                break;
+                            } else if current_instruction.program_id == SYSTEM_PROGRAM {
+                                system_program_count += 1;
+                                if system_program_count > 2 {
+                                    all_intermediate_are_system = false;
+                                    break;
+                                }
+                            } else {
+                                all_intermediate_are_system = false;
+                                break;
+                            }
+                        }
+
+                        // If we found a NOOP instruction, exactly one or two SYSTEM_PROGRAM instructions, and all intermediates were valid
+                        if let Some(noop_index) = noop_instruction_index {
+                            if system_program_count >= 1
+                                && system_program_count <= 2
+                                && all_intermediate_are_system
+                            {
+                                if let Some(state_update) = parse_public_transaction_event_v1(
+                                    tx,
+                                    slot,
+                                    instruction,
+                                    &ordered_instructions[noop_index],
+                                )? {
+                                    is_compression_transaction = true;
+                                    state_updates.push(state_update);
+                                }
+                            }
+                        }
                     }
-                } else if ordered_instructions.len() - index > 1 {
+                }
+
+                if ordered_instructions.len() - index > 1 {
                     if let Some(state_update) = parse_merkle_tree_event(
                         &ordered_instructions[index],
                         &ordered_instructions[index + 1],

src/ingester/parser/tx_event_parser.rs

@@ -3,7 +3,7 @@ use crate::ingester::error::IngesterError;
 use crate::ingester::parser::indexer_events::PublicTransactionEvent;
 use crate::ingester::parser::state_update::{AccountTransaction, StateUpdate};
 use crate::ingester::parser::tree_info::TreeInfo;
-use crate::ingester::parser::{get_compression_program_id, NOOP_PROGRAM_ID, SYSTEM_PROGRAM};
+use crate::ingester::parser::{get_compression_program_id, NOOP_PROGRAM_ID};
 use crate::ingester::typedefs::block_info::{Instruction, TransactionInfo};
 use anchor_lang::AnchorDeserialize;
 use light_compressed_account::TreeType;
@@ -14,28 +14,27 @@ use std::collections::HashMap;
 pub fn parse_public_transaction_event_v1(
     tx: &TransactionInfo,
     slot: u64,
-    instruction: &Instruction,
-    next_instruction: &Instruction,
-    next_next_instruction: &Instruction,
+    compression_instruction: &Instruction,
+    noop_instruction: &Instruction,
 ) -> Result<Option<StateUpdate>, IngesterError> {
-    if get_compression_program_id() == instruction.program_id
-        && next_instruction.program_id == SYSTEM_PROGRAM
-        && next_next_instruction.program_id == NOOP_PROGRAM_ID
+    if get_compression_program_id() == compression_instruction.program_id
+        && noop_instruction.program_id == NOOP_PROGRAM_ID
         && tx.error.is_none()
     {
         info!(
             "Indexing transaction with slot {} and id {}",
             slot, tx.signature
         );
 
-        let public_transaction_event =
-            PublicTransactionEvent::deserialize(&mut next_next_instruction.data.as_slice())
-                .map_err(|e| {
-                    IngesterError::ParserError(format!(
-                        "Failed to deserialize PublicTransactionEvent: {}",
-                        e
-                    ))
-                })?;
+        let public_transaction_event = PublicTransactionEvent::deserialize(
+            &mut noop_instruction.data.as_slice(),
+        )
+        .map_err(|e| {
+            IngesterError::ParserError(format!(
+                "Failed to deserialize PublicTransactionEvent: {}",
+                e
+            ))
+        })?;
         create_state_update_v1(tx.signature, slot, public_transaction_event.into()).map(Some)
     } else {
         Ok(None)

src/migration/migrations/standard/m20220101_000001_init.rs

@@ -29,8 +29,8 @@ impl MigrationTrait for Migration {
         if manager.get_database_backend() == DatabaseBackend::Postgres {
             // Max balance of an individual account or token account is 2**64. bigint2 is 5000x larger than that to account for batch updates where update the database with new account balances
             // before closing spent accounts and extreme cases where a single wallet owns an excessive amount of account / tokens with high number of lamports.
-            // 
-            // We go a bit overboard here, but it's ok because bigint2 is not the storage bottleneck. 
+            //
+            // We go a bit overboard here, but it's ok because bigint2 is not the storage bottleneck.
             execute_sql(
                 manager,
                 "

tests/data/transactions/transaction_with_tree_rollover_fee/2cBtegqLxQztcngNF4qWGZYEuGiwFvmSpak4dqNaGHHQRDBGuYg24ZSG54BpRaWS5Cr4v6AWLV42FWvEjQk2ESWy

@@ -0,0 +1 @@
+{"slot":380876342,"transaction":["AVBckWHufnpo5k31sra8b25PSBJ8wZHFHxTVzywKv0Md7zI6gWtYfZvsCWGTf1RRnAjpOZuyiwMGAXVQyZKKvgYBAAgOwZ1VN79Su7sp1VDkyLqdpMjhOkGz6rq65faB3x1IZfcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAuzCqyCzEXXw00QukxWWF121rpHf8sXrOfkw90a+RQ1DQHJKKtF02VZWjTodxf7sACKGdIvBSydofmIxHxX3joNAclJOOYCWqhG9HrdwjHLVUzp6/nSXH2t5N+WwRp7vg0BydoNu62NBXntAkJwqhoi8SO5svaEsnL1rvZDx6HSHuvEjPsyZLvDS/XFpag0/GdQAG8phtlvI1vIh1703UQJLDbsIvUXgwD9tEqqav/P8KRuHLxkHA4+0J2hO571CAMGRm/lIRcy/+ytunLDm+e8jOW7xfcSayxDmzpAAAAACRWjVyN5To+2XQdba3JpnDjdAuWUi3Ww5aBBjoCXW0TmyRiwvXzPkVREinq/ao85eCmh6P0Ip/DQs6q4eFL8MvuzKHUQXK6FtbREdgftv+FFJ7+0I5EcpAQjv9FSeiZ1C7wPwLtHyi90xBEulKsTz6PGNOXcF+rLA80aI81+eHwGp1X4ITkFTUQksVrwxDDPL0t/mHk62hJS1I82ZsbLziCaTFY1ZnIqrdkzA9GHuiwY069xIwQGrFOy4rAIF320AwgACQPI0AIAAAAAAAgABQK9MwYACREAAAoNBgwLBwkJCQkBBQIDBKQCozTI54wDRboYAQAAARmUiawkWKBeQgS9VpBohrNw/E/2OQpKx6pzowyS3rilGMyVSZ4X/7Vdcv5gdSVa4h/WuMi29PJZdF35DzCniggZckae63vUjghxUk09yCHBH9fwLWOsv3mCeXV4H+B7+ybQjx0RHTjqWYbG71iAZI45k77VJGJNaGl3g3ctnL2TkGZ00DAQ8bqxNZpC7wnKZj9A8c6E0wN52xW0emTnFsIAAQAAAACABldIcAAAAAABu3EBAADOAQAAAgAAAMGdVTe/Uru7KdVQ5Mi6naTI4TpBs+q6uuX2gd8dSGX3ALBOK95vAAAAAgDar8QcMl6GrjKzzx6mdme7F5jKQJxdMWqbMaIVnD+mAwDQtytqAAAAAAMAAAAAAA==","base64"],"meta":{"err":null,"status":{"Ok":null},"fee":80001,"preBalances":[2783445420,1,9647410200,10977200160,9525266460,9524739960,1392000,1141440,1,1141440,12348739,0,1141440,1141440],"postBalances":[2783359819,1,9647415200,10977200460,9525266760,9524739960,1392000,1141440,1,1141440,12348739,0,1141440,1141440],"innerInstructions":[{"index":2,"instructions":[{"programIdIndex":13,"accounts":[0,10,6,12,11,7,9,13,13,1,13,5,2,3,4],"data":"22TjKTEEcMHaSkGa3aqCjXZtU5FUj2njS6prCwFsjUx5oJKmMAjL1uVSAz8YmfqVZgxVQ92LhKQqeDKCuktXwfRpKZxGC6WqdYegv6Zo5fTWoE4QBMLst4prpuWeTQyuN4YUpqVrcDJc4T2WhSVNWeg5A3xpDrYd9VBYFGzkq8zfpZx9vQwwbEffgu5Mjubn5DZPkCJcZUTjgKYKC48xUugXTbQFdHGM1ExKdvgKdVD5vU2BQAgKUAUB8aUnpVQqgdq3jH4a5gQwAggw88MQm28Hqkw4oL7TTgrWkJC4teUvxAQkQ7NgD67bZtRRxGwVw93yHDzQt885zk3AqKaTL7kL5zwpQ1u9dTrUsgwkj8iJ3rW8aor1i3wgJpZYzGP1RiWdNR4L6s3Y9niGFrrQB4kLTdWHyQhUueYECFpufryV3RnTmmf5hY1x8F3zUjFquzgviWKX7y1RaBwmVXQ1qsdt9iMgNuSGrMCVpz92h27FK2Ugyu8rkUxJXDsYy9dgdSjnJbBhuQ3K8bX1f6qEmdiF4VsuoqTFQnra5R7VsMrHbW5Zu3gdX47gyvfbAjd3Rxdt9djw1YWPKYzzNeNwAypjVTePEqiQDrTPv4E7hjvrPQJG2QTNq6yds4h84q9acfpdZnLyZCWjDuynkBwjskymfxF72k8kjVrQfSc8WBtQ23pnW3sMwbeHbNXkzgJrao9jxiGdfry46QYsSLvDHDyuWvwo7cE4MbuKZZXhfD1fvWbNpxTHyC47BHADS4Tgsj6627FLixjRpBZ2a8utTvu14es8UZzspVtQraS47HGqXJPt7sZFKUFKw7GYNEiXpd7p34uLTeBNbBps6EgHfkPK1PYM4j2Trkkf7zVBMhjBJQffBMYMqWzDc7h","stackHeight":2},{"programIdIndex":7,"accounts":[0,11,6,1,2,5],"data":"WiZw2hSp2LSxinAV63skVmsJMhyg37nsMpMABKHpxqFPSz6PMHM9F2HemKq8","stackHeight":3},{"programIdIndex":1,"accounts":[0,2],"data":"3Bxs4PckVVt51W8w","stackHeight":3},{"programIdIndex":7,"accounts":[0,11,6,1,3,4],"data":"7UWNMdwFJEWBQPSoUMHBWZGDsTr9ra7LJEZ2ZmTTLqULxjy4U4sejJbU6HpLW7VUk3zMJTHCdKNJHGAHeUMw3hWKcdZiS2E9RD2Q7fLhR9P","stackHeight":3},{"programIdIndex":1,"accounts":[0,3],"data":"3Bxs48DZ3Zx6m4mD","stackHeight":4},{"programIdIndex":1,"accounts":[0,4],"data":"3Bxs48DZ3Zx6m4mD","stackHeight":4},{"programIdIndex":12,"accounts":[],"data":"6mLqCakjkxPs3hb7JKiVxBWv438hyQ6ziFhCSn8ahdACfu886HBNpMe4vpxbxubEWoYhboFCWqTtDknnRN6fr6akR2noRXYkEAURf9X51AHgiKQJog8uiGtccse18mZR7qk7cR1T5FZXZZsJUjoFLM2HXL2Q9nrGTNPvVTYKTk7pABQNcGmomLxfZzLPTXxzooJ26N3bBBfxv3zfJoTneP3Zd8o1PYKza6Vg3cva93rfByrKqzZw73jYQb36wtM3zdDhCV1uKkpYjVen1wKuqGMwikjgkhDDSRqXCT9hvZVZC4ahTpwPJW7y4MGCD6VFcVPkXAkvCnQohZZnqeZ5p9FUiqqP9BuFg9ofyTYbzjL5kuUaXk2B1dBHJD5AUumTRZfScw8TPhatXwgpdfY17JGJ5T4g7JW3yFcea9DJZR5HwnTQ8yUph1WKaXupzsXmAPdvsw1vnuPVa1qASuDyvEEAu6g3bvh39ut6CNCoCrPAtbh8h2efpiEMftSfgFVfEwnviAdMAjhCNEJrRzohVpCtz2LYyS5CVgPCN4z5xUJ5du8shU8Aa8JPgr7uuRvASi9KWfHqJLDfvSuxyZCtTzGRxgxu6jzFsUe1Q4gqhd3Fbm1mh88DMq9TgVn1oAnKLbvEgtJeFCLomsUvx5FhGY3FE3PDfehFZqMoZDEif81dasUXCxNycXswBL2tfFsQHpwhYV7hgJVfkwcf5mWBfK4bpMtUNUQAgU9iqVMNJXem3jSaexKQLUyvSgG5MXvgKUV4nWFBKbqoQthQMduXb4mV2qqX4iBWpctZAYaoC6FLhguMhu9Xz7PARixSB3AFiuBq3cYDc2G6anXRk6TPcVm1pYkthiuBguMeiZRn26e3ufBmWGykaizR6dW2e4NPYHyAeFpZq9","stackHeight":3}]}],"logMessages":["Program ComputeBudget111111111111111111111111111111 invoke [1]","Program ComputeBudget111111111111111111111111111111 success","Program ComputeBudget111111111111111111111111111111 invoke [1]","Program ComputeBudget111111111111111111111111111111 success","Program cTokenmWW8bLPjZEBAUgYy3zKxQZW6VKi7bqNFEVv3m invoke [1]","Program log: Instruction: Transfer","Program SySTEM1eSU2p4BGQfQpimFEWWSC1XDFeun3Nqzz3rT7 invoke [2]","Program log: Instruction: InvokeCpi","Program compr6CUsB5m2jS4Y3831ztGSTnDpnKJTKS95d64XVq invoke [3]","Program log: Instruction: InsertIntoNullifierQueues","Program compr6CUsB5m2jS4Y3831ztGSTnDpnKJTKS95d64XVq consumed 10550 of 210228 compute units","Program compr6CUsB5m2jS4Y3831ztGSTnDpnKJTKS95d64XVq success","Program 11111111111111111111111111111111 invoke [3]","Program 11111111111111111111111111111111 success","Program compr6CUsB5m2jS4Y3831ztGSTnDpnKJTKS95d64XVq invoke [3]","Program log: Instruction: AppendLeavesToMerkleTrees","Program 11111111111111111111111111111111 invoke [4]","Program 11111111111111111111111111111111 success","Program 11111111111111111111111111111111 invoke [4]","Program 11111111111111111111111111111111 success","Program compr6CUsB5m2jS4Y3831ztGSTnDpnKJTKS95d64XVq consumed 80781 of 167762 compute units","Program compr6CUsB5m2jS4Y3831ztGSTnDpnKJTKS95d64XVq success","Program noopb9bkMVfRPU8AsbpTUg8AQkHtKwMYZiFUjNRtMmV invoke [3]","Program noopb9bkMVfRPU8AsbpTUg8AQkHtKwMYZiFUjNRtMmV consumed 97 of 83046 compute units","Program noopb9bkMVfRPU8AsbpTUg8AQkHtKwMYZiFUjNRtMmV success","Program SySTEM1eSU2p4BGQfQpimFEWWSC1XDFeun3Nqzz3rT7 consumed 269947 of 352217 compute units","Program SySTEM1eSU2p4BGQfQpimFEWWSC1XDFeun3Nqzz3rT7 success","Program cTokenmWW8bLPjZEBAUgYy3zKxQZW6VKi7bqNFEVv3m consumed 325019 of 406161 compute units","Program cTokenmWW8bLPjZEBAUgYy3zKxQZW6VKi7bqNFEVv3m success"],"preTokenBalances":[],"postTokenBalances":[],"rewards":[],"loadedAddresses":{"writable":[],"readonly":[]},"computeUnitsConsumed":325319},"version":"legacy","blockTime":1747252042}

tests/integration_tests/e2e_tests.rs

@@ -691,7 +691,6 @@ async fn test_nullfiier_and_address_queue_transactions(
     }
 }
 
-
 #[named]
 #[rstest]
 #[tokio::test]
@@ -710,7 +709,8 @@ async fn test_transaction_with_tree_rollover_fee(
         },
     )
     .await;
-    let txn = "2cBtegqLxQztcngNF4qWGZYEuGiwFvmSpak4dqNaGHHQRDBGuYg24ZSG54BpRaWS5Cr4v6AWLV42FWvEjQk2ESWy";
+    let txn =
+        "2cBtegqLxQztcngNF4qWGZYEuGiwFvmSpak4dqNaGHHQRDBGuYg24ZSG54BpRaWS5Cr4v6AWLV42FWvEjQk2ESWy";
     let txn = cached_fetch_transaction(&name, setup.client.clone(), txn).await;
     let status_update = parse_transaction(&txn.try_into().unwrap(), 0).unwrap();
     // Assert that status update has at least one account