Merge branch 'develop' into feat/clarity-cli-serialized-output

Author: reedrosenbluth

Cargo.lock

@@ -43,7 +43,6 @@ name = "block_limits"
 harness = false
 
 [dependencies]
-tini = "0.2"
 rand = "=0.7.2"
 rand_chacha = "=0.2.2"
 serde = "1"

Cargo.toml

@@ -27,6 +27,9 @@ These events are sent to the configured endpoint at two URLs:
 This payload includes data related to a newly processed block,
 and any events emitted from Stacks transactions during the block.
 
+If the transaction originally comes from the parent microblock stream 
+preceding this block, the microblock related fields will be filled in.
+
 Example:
 
 ```json
@@ -58,15 +61,21 @@ Example:
       "raw_tx": "0x808000000004008bc5147525b8f477f0bc4522a88c8339b2494db50000000000000002000000000000000001015814daf929d8700af344987681f44e913890a12e38550abe8e40f149ef5269f40f4008083a0f2e0ddf65dcd05ecfc151c7ff8a5308ad04c77c0e87b5aeadad31010200000000040000000000000000000000000000000000000000000000000000000000000000",
       "status": "success",
       "tx_index": 0,
-      "txid": "0x3e04ada5426332bfef446ba0a06d124aace4ade5c11840f541bf88e2e919faf6"
+      "txid": "0x3e04ada5426332bfef446ba0a06d124aace4ade5c11840f541bf88e2e919faf6",
+      "microblock_sequence": "None",
+      "microblock_hash": "None",
+      "microblock_parent_hash": "None"
     },
     {
       "contract_abi": null,
       "raw_result": "0x03",
       "raw_tx": "0x80800000000400f942874ce525e87f21bbe8c121b12fac831d02f4000000000000000000000000000003e800006ae29867aec4b0e4f776bebdcea7f6d9a24eeff370c8c739defadfcbb52659b30736ad4af021e8fb741520a6c65da419fdec01989fdf0032fc1838f427a9a36102010000000000051ac2d519faccba2e435f3272ff042b89435fd160ff00000000000003e800000000000000000000000000000000000000000000000000000000000000000000",
       "status": "success",
       "tx_index": 1,
-      "txid": "0x738e4d44636023efa08374033428e44eca490582bd39a6e61f3b6cf749b4214c"
+      "txid": "0x738e4d44636023efa08374033428e44eca490582bd39a6e61f3b6cf749b4214c",
+      "microblock_sequence": "3",
+      "microblock_hash": "0x9304fcbcc6daf5ac3f264522e0df50eddb5be85df6ee8a9fc2384c54274daaac",
+      "microblock_parent_hash": "0x4893ab44636023efa08374033428e44eca490582bd39a6e61f3b6cf749b474bd"
     }
    ],
    "matured_miner_rewards": [
@@ -116,6 +125,63 @@ Example:
   PoX commitments during this block. These addresses may not actually receive rewards during
   this block if the block is faster than miners have an opportunity to commit.
 
+### `POST /new_microblocks`
+
+This payload includes data related to one or more microblocks that are either emmitted by the 
+node itself, or received through the network. 
+
+Example:
+
+```json
+{
+  "parent_index_block_hash": "0x999b38d44d6af72703a476dde4cea683ec965346d9e9a7ded2d773fb4f257a3b",
+  "events": [
+    {
+      "event_index": 1,
+      "committed": true,
+      "stx_transfer_event": {
+        "amount": "1000",
+        "recipient": "ST31DA6FTSJX2WGTZ69SFY11BH51NZMB0ZZ239N96",
+        "sender": "ST3WM51TCWMJYGZS1QFMC28DH5YP86782YGR113C1"
+      },
+      "txid": "0x738e4d44636023efa08374033428e44eca490582bd39a6e61f3b6cf749b4214c",
+      "type": "stx_transfer_event"
+    }
+  ],
+  "transactions": [
+    {
+      "contract_abi": null,
+      "raw_result": "0x03",
+      "raw_tx": "0x808000000004008bc5147525b8f477f0bc4522a88c8339b2494db50000000000000002000000000000000001015814daf929d8700af344987681f44e913890a12e38550abe8e40f149ef5269f40f4008083a0f2e0ddf65dcd05ecfc151c7ff8a5308ad04c77c0e87b5aeadad31010200000000040000000000000000000000000000000000000000000000000000000000000000",
+      "status": "success",
+      "tx_index": 0,
+      "txid": "0x3e04ada5426332bfef446ba0a06d124aace4ade5c11840f541bf88e2e919faf6",
+      "microblock_sequence": "3",
+      "microblock_hash": "0x9304fcbcc6daf5ac3f264522e0df50eddb5be85df6ee8a9fc2384c54274daaac",
+      "microblock_parent_hash": "0x4893ab44636023efa08374033428e44eca490582bd39a6e61f3b6cf749b474bd"
+    },
+    {
+      "contract_abi": null,
+      "raw_result": "0x03",
+      "raw_tx": "0x80800000000400f942874ce525e87f21bbe8c121b12fac831d02f4000000000000000000000000000003e800006ae29867aec4b0e4f776bebdcea7f6d9a24eeff370c8c739defadfcbb52659b30736ad4af021e8fb741520a6c65da419fdec01989fdf0032fc1838f427a9a36102010000000000051ac2d519faccba2e435f3272ff042b89435fd160ff00000000000003e800000000000000000000000000000000000000000000000000000000000000000000",
+      "status": "success",
+      "tx_index": 1,
+      "txid": "0x738e4d44636023efa08374033428e44eca490582bd39a6e61f3b6cf749b4214c",
+      "microblock_sequence": "4",
+      "microblock_hash": "0xfcd4fc34c6daf5ac3f264522e0df50eddb5be85df6ee8a9fc2384c5427459e43",
+      "microblock_parent_hash": "0x9304fcbcc6daf5ac3f264522e0df50eddb5be85df6ee8a9fc2384c54274daaac"
+    }
+  ],
+  "burn_block_hash": "0x4eaabcd105865e471f697eff5dd5bd85d47ecb5a26a3379d74fae0ae87c40904",
+  "burn_block_height": 331,
+  "burn_block_timestamp": 1651301734
+}
+```
+
+* `burn_block_{}` are the stats related to the burn block that is associated with the stacks 
+  block that precedes this microblock stream.
+* Each transaction json object includes information about the microblock the transaction was packaged into. 
+
 ### `POST /new_mempool_tx`
 
 This payload includes raw transactions newly received in the

docs/event-dispatcher.md

@@ -25,8 +25,6 @@ use std::path::PathBuf;
 use std::time;
 use std::time::Duration;
 
-use tini::Ini;
-
 use burnchains::bitcoin::blocks::BitcoinHeaderIPC;
 use burnchains::bitcoin::messages::BitcoinMessageHandler;
 use burnchains::bitcoin::spv::*;
@@ -137,165 +135,6 @@ impl BitcoinIndexerConfig {
             magic_bytes: BLOCKSTACK_MAGIC_MAINNET.clone(),
         }
     }
-
-    pub fn to_file(&self, path: &String) -> Result<(), btc_error> {
-        let username = self.username.clone().unwrap_or("".to_string());
-        let password = self.password.clone().unwrap_or("".to_string());
-
-        let conf = Ini::new()
-            .section("bitcoin")
-            .item("server", self.peer_host.as_str())
-            .item("p2p_port", format!("{}", self.peer_port).as_str())
-            .item("rpc_port", format!("{}", self.rpc_port).as_str())
-            .item("rpc_ssl", format!("{}", self.rpc_ssl).as_str())
-            .item("username", username.as_str())
-            .item("password", password.as_str())
-            .item("timeout", format!("{}", self.timeout).as_str())
-            .item("spv_path", self.spv_headers_path.as_str())
-            .item("first_block", format!("{}", self.first_block).as_str())
-            .section("blockstack")
-            .item(
-                "network_id",
-                format!(
-                    "{}{}",
-                    self.magic_bytes.as_bytes()[0] as char,
-                    self.magic_bytes.as_bytes()[1] as char
-                )
-                .as_str(),
-            );
-
-        conf.to_file(&path).map_err(|e| btc_error::Io(e))
-    }
-
-    pub fn from_file(path: &String) -> Result<BitcoinIndexerConfig, btc_error> {
-        let conf_path = PathBuf::from(path);
-        if !conf_path.is_file() {
-            return Err(btc_error::ConfigError(
-                "Failed to load BitcoinIndexerConfig file: No such file or directory".to_string(),
-            ));
-        }
-
-        let default_config = BitcoinIndexerConfig::default(0);
-
-        match Ini::from_file(path) {
-            Ok(ini_file) => {
-                // [bitcoin]
-                let peer_host = ini_file
-                    .get("bitcoin", "server")
-                    .unwrap_or(default_config.peer_host);
-
-                let peer_port = ini_file
-                    .get("bitcoin", "p2p_port")
-                    .unwrap_or(format!("{}", default_config.peer_port))
-                    .trim()
-                    .parse()
-                    .map_err(|_e| {
-                        btc_error::ConfigError("Invalid bitcoin:p2p_port value".to_string())
-                    })?;
-
-                if peer_port <= 1024 || peer_port == 65535 {
-                    return Err(btc_error::ConfigError("Invalid p2p_port".to_string()));
-                }
-
-                let rpc_port = ini_file
-                    .get("bitcoin", "rpc_port")
-                    .unwrap_or(format!("{}", default_config.rpc_port))
-                    .trim()
-                    .parse()
-                    .map_err(|_e| {
-                        btc_error::ConfigError("Invalid bitcoin:port value".to_string())
-                    })?;
-
-                if rpc_port <= 1024 || rpc_port == 65535 {
-                    return Err(btc_error::ConfigError("Invalid rpc_port".to_string()));
-                }
-
-                let username: Option<String> =
-                    ini_file.get("bitcoin", "username").and_then(|s| Some(s));
-                let password: Option<String> =
-                    ini_file.get("bitcoin", "password").and_then(|s| Some(s));
-
-                let timeout = ini_file
-                    .get("bitcoin", "timeout")
-                    .unwrap_or(format!("{}", default_config.timeout))
-                    .trim()
-                    .parse()
-                    .map_err(|_e| {
-                        btc_error::ConfigError("Invalid bitcoin:timeout value".to_string())
-                    })?;
-
-                let spv_headers_path_cfg = ini_file
-                    .get("bitcoin", "spv_path")
-                    .unwrap_or(default_config.spv_headers_path);
-
-                let spv_headers_path =
-                    if path::is_separator(spv_headers_path_cfg.chars().next().ok_or(
-                        btc_error::ConfigError("Invalid bitcoin:spv_path value".to_string()),
-                    )?) {
-                        // absolute
-                        spv_headers_path_cfg
-                    } else {
-                        // relative to config file
-                        let mut p = PathBuf::from(path);
-                        p.pop();
-                        let s = p.join(&spv_headers_path_cfg);
-                        s.to_str().unwrap().to_string()
-                    };
-
-                let first_block = ini_file
-                    .get("bitcoin", "first_block")
-                    .unwrap_or(format!("{}", 0))
-                    .trim()
-                    .parse()
-                    .map_err(|_e| {
-                        btc_error::ConfigError("Invalid bitcoin:first_block value".to_string())
-                    })?;
-
-                let rpc_ssl_str = ini_file
-                    .get("bitcoin", "ssl")
-                    .unwrap_or(format!("{}", default_config.rpc_ssl));
-
-                let rpc_ssl = rpc_ssl_str == "1" || rpc_ssl_str == "true";
-
-                // [blockstack]
-                let blockstack_magic_str =
-                    ini_file.get("blockstack", "network_id").unwrap_or(format!(
-                        "{}{}",
-                        BLOCKSTACK_MAGIC_MAINNET.as_bytes()[0] as char,
-                        BLOCKSTACK_MAGIC_MAINNET.as_bytes()[1] as char
-                    ));
-
-                if blockstack_magic_str.len() != 2 {
-                    return Err(btc_error::ConfigError(
-                        "Invalid blockstack:network_id value: must be two bytes".to_string(),
-                    ));
-                }
-
-                let blockstack_magic = MagicBytes([
-                    blockstack_magic_str.as_bytes()[0] as u8,
-                    blockstack_magic_str.as_bytes()[1] as u8,
-                ]);
-
-                let cfg = BitcoinIndexerConfig {
-                    peer_host: peer_host,
-                    peer_port: peer_port,
-                    rpc_port: rpc_port,
-                    rpc_ssl: rpc_ssl,
-                    username: username,
-                    password: password,
-                    timeout: timeout,
-                    spv_headers_path: spv_headers_path,
-                    first_block: first_block,
-                    magic_bytes: blockstack_magic,
-                };
-
-                Ok(cfg)
-            }
-            Err(_) => Err(btc_error::ConfigError(
-                "Failed to parse BitcoinConfigIndexer config file".to_string(),
-            )),
-        }
-    }
 }
 
 impl BitcoinIndexerRuntime {
@@ -323,18 +162,6 @@ impl BitcoinIndexer {
         }
     }
 
-    pub fn from_file(
-        network_id: BitcoinNetworkType,
-        config_file: &String,
-    ) -> Result<BitcoinIndexer, btc_error> {
-        let config = BitcoinIndexerConfig::from_file(config_file)?;
-        let runtime = BitcoinIndexerRuntime::new(network_id);
-        Ok(BitcoinIndexer {
-            config: config,
-            runtime: runtime,
-        })
-    }
-
     pub fn dup(&self) -> BitcoinIndexer {
         BitcoinIndexer {
             config: self.config.clone(),
@@ -775,57 +602,6 @@ impl Drop for BitcoinIndexer {
 impl BurnchainIndexer for BitcoinIndexer {
     type P = BitcoinBlockParser;
 
-    /// Instantiate the Bitcoin indexer, and connect to the peer network.
-    /// Instead, load our configuration state and sanity-check it.
-    ///
-    /// Pass a directory (working_dir) that contains a "bitcoin.ini" file.
-    fn init(
-        working_dir: &String,
-        network_name: &String,
-        first_block_height: u64,
-    ) -> Result<BitcoinIndexer, burnchain_error> {
-        let conf_path_str =
-            Burnchain::get_chainstate_config_path(working_dir, &"bitcoin".to_string());
-
-        let network_id_opt = match network_name.as_ref() {
-            BITCOIN_MAINNET_NAME => Some(BitcoinNetworkType::Mainnet),
-            BITCOIN_TESTNET_NAME => Some(BitcoinNetworkType::Testnet),
-            BITCOIN_REGTEST_NAME => Some(BitcoinNetworkType::Regtest),
-            _ => None,
-        };
-
-        if network_id_opt.is_none() {
-            return Err(burnchain_error::Bitcoin(btc_error::ConfigError(format!(
-                "Unrecognized network name '{}'",
-                network_name
-            ))));
-        }
-        let bitcoin_network_id = network_id_opt.unwrap();
-
-        if !PathBuf::from(&conf_path_str).exists() {
-            let default_config = BitcoinIndexerConfig::default(first_block_height);
-            default_config
-                .to_file(&conf_path_str)
-                .map_err(burnchain_error::Bitcoin)?;
-        }
-
-        let mut indexer = BitcoinIndexer::from_file(bitcoin_network_id, &conf_path_str)
-            .map_err(burnchain_error::Bitcoin)?;
-
-        SpvClient::new(
-            &indexer.config.spv_headers_path,
-            0,
-            None,
-            indexer.runtime.network_id,
-            true,
-            false,
-        )
-        .map_err(burnchain_error::Bitcoin)?;
-
-        indexer.connect()?;
-        Ok(indexer)
-    }
-
     /// Connect to the Bitcoin peer network.
     /// Use the peer host and peer port given in the config file,
     /// and loaded in on setup.  Don't call this before init().