Implement Ipfs read and write with verification in enclave

this implements gitcoin bounty #70 
Co-authored-by: bwty <[email protected]>

Author: whalelephant

enclave/Cargo.lock

@@ -50,6 +50,10 @@ itertools       = { version = "0.8",  default-features = false, features = []}
 bit-vec         = { version = "0.6",    default-features = false }
 base58          = { rev = "sgx_1.1.2", package="rust-base58", git = "https://github.com/mesalock-linux/rust-base58-sgx", default-features = false, features=["mesalock_sgx"] }
 
+cid = { git = "https://github.com/whalelephant/rust-cid", branch = "nstd", default-features = false }
+multibase = { git = "https://github.com/whalelephant/rust-multibase", branch = "nstd", default-features = false }
+
+
 [dependencies.webpki]
 git              = "https://github.com/mesalock-linux/webpki"
 branch           = "mesalock_sgx"
@@ -125,3 +129,10 @@ features = ["sgx"]
 
 [patch.crates-io]
 sp-io = { git = "https://github.com/scs/sgx-runtime", default-features = false, features = ["disable_oom", "disable_panic_handler", "disable_allocator", "sgx"]}
+
+[dependencies.ipfs-unixfs]
+git = "https://github.com/whalelephant/rust-ipfs"
+branch = "w-nstd"
+default-features = false
+
+

enclave/Cargo.toml

@@ -98,7 +98,6 @@ enclave {
 		);
 
 		sgx_status_t ocall_read_ipfs(
-        	[out, size = state_size] uint8_t * enc_state, uint32_t state_size,
         	[in, size = cid_size] uint8_t * cid, uint32_t cid_size
         );
 

enclave/Enclave.edl

@@ -0,0 +1,104 @@
+use cid::{Cid, Result as CidResult};
+use ipfs_unixfs::file::adder::FileAdder;
+use log::*;
+use multibase::Base;
+use std::convert::TryFrom;
+use std::vec::Vec;
+
+pub struct IpfsContent {
+    pub cid: CidResult<Cid>,
+    pub file_content: Vec<u8>,
+    pub stats: Stats,
+}
+#[derive(Debug, PartialEq)]
+pub enum IpfsError {
+    InputCidInvalid,
+    FinalCidMissing,
+    Verification,
+}
+
+impl IpfsContent {
+    pub fn new(_cid: &str, _content: Vec<u8>) -> IpfsContent {
+        IpfsContent {
+            cid: Cid::try_from(_cid),
+            file_content: _content,
+            stats: Stats::default(),
+        }
+    }
+
+    pub fn verify(&mut self) -> Result<(), IpfsError> {
+        let mut adder: FileAdder = FileAdder::default();
+        let mut total: usize = 0;
+        while total < self.file_content.len() {
+            let (blocks, consumed) = adder.push(&self.file_content[total..]);
+            total += consumed;
+            self.stats.process(blocks);
+        }
+        let blocks = adder.finish();
+        self.stats.process(blocks);
+
+        if let Some(last_cid) = self.stats.last.as_ref() {
+            let cid_str = Base::Base58Btc.encode(last_cid.hash().as_bytes());
+            info!(
+                "new cid: {} generated from {} blocks, total of {} bytes",
+                cid_str, self.stats.blocks, self.stats.block_bytes
+            );
+            match self.cid.as_ref() {
+				Ok(initial_cid) => {
+					if last_cid.hash().eq(&initial_cid.hash()) {
+						Ok(())
+					} else {
+						Err(IpfsError::Verification)
+					}
+				},
+				Err(_) => {
+					Err(IpfsError::InputCidInvalid)
+				}
+            }
+        } else {
+            Err(IpfsError::FinalCidMissing)
+        }
+    }
+}
+#[derive(Default)]
+pub struct Stats {
+    pub blocks: usize,
+    pub block_bytes: u64,
+    pub last: Option<Cid>,
+}
+
+impl Stats {
+    fn process<I: Iterator<Item = (Cid, Vec<u8>)>>(&mut self, new_blocks: I) {
+        for (cid, block) in new_blocks {
+            self.last = Some(cid);
+            self.blocks += 1;
+            self.block_bytes += block.len() as u64;
+        }
+    }
+}
+
+pub fn test_creates_ipfs_content_struct_works() {
+    let cid = "QmSaFjwJ2QtS3rZDKzC98XEzv2bqT4TfpWLCpphPPwyQTr";
+    let content: Vec<u8> = vec![20; 512 * 1024];
+    let ipfs_content = IpfsContent::new(cid, content.clone());
+
+    let cid_str = Base::Base58Btc.encode(ipfs_content.cid.as_ref().unwrap().hash().as_bytes());
+    assert_eq!(cid_str, cid);
+    assert_eq!(ipfs_content.file_content, content);
+}
+
+pub fn test_verification_ok_for_correct_content() {
+    let cid = "QmSaFjwJ2QtS3rZDKzC98XEzv2bqT4TfpWLCpphPPwyQTr";
+    let content: Vec<u8> = vec![20; 512 * 1024];
+    let mut ipfs_content = IpfsContent::new(cid, content);
+    let verification = ipfs_content.verify();
+    assert_eq!(verification.is_ok(), true);
+}
+
+pub fn test_verification_fails_for_incorrect_content() {
+    let cid = "QmSaFjwJ2QtS3rZDKzC98XEzv2bqT4TfpWLCpphPPwyQTr";
+    let content: Vec<u8> = vec![10; 512 * 1024];
+    let mut ipfs_content = IpfsContent::new(cid, content);
+    let verification = ipfs_content.verify();
+    assert_eq!(verification.unwrap_err(), IpfsError::Verification);
+}

enclave/src/ipfs.rs

@@ -50,7 +50,10 @@ use std::slice;
 use std::string::String;
 use std::vec::Vec;
 
+use ipfs::IpfsContent;
 use std::collections::HashMap;
+use std::fs::File;
+use std::io::Read;
 use utils::write_slice_and_whitespace_pad;
 
 use crate::constants::{CALL_WORKER, SHIELD_FUNDS};
@@ -62,13 +65,14 @@ use chain_relay::{
 use sp_runtime::OpaqueExtrinsic;
 use sp_runtime::{generic::SignedBlock, traits::Header as HeaderT};
 use substrate_api_client::extrinsic::xt_primitives::UncheckedExtrinsicV4;
-use substratee_stf::sgx::{OpaqueCall, shards_key_hash, storage_hashes_to_update_per_shard};
+use substratee_stf::sgx::{shards_key_hash, storage_hashes_to_update_per_shard, OpaqueCall};
 
 mod aes;
 mod attestation;
 mod constants;
 mod ed25519;
 mod io;
+mod ipfs;
 mod rsa3072;
 mod state;
 mod utils;
@@ -227,7 +231,7 @@ pub unsafe extern "C" fn get_state(
 
     let mut state = match state::load(&shard) {
         Ok(s) => s,
-        Err(status) => return status
+        Err(status) => return status,
     };
 
     let validator = match io::light_validation::unseal() {
@@ -363,7 +367,6 @@ pub unsafe extern "C" fn sync_chain_relay(
             Ok(c) => calls.extend(c.into_iter()),
             Err(_) => error!("Error executing relevant extrinsics"),
         };
-
     }
 
     if let Err(_e) = stf_post_actions(validator, calls, xt_slice, *nonce) {
@@ -391,7 +394,8 @@ pub fn update_states(header: Header) -> SgxResult<()> {
     if let Some(maybe_shards) = update_map.get(&shards_key_hash()) {
         match maybe_shards {
             Some(shards) => {
-                let shards: Vec<ShardIdentifier> = Decode::decode(&mut shards.as_slice()).sgx_error_with_log("error decoding shards")?;
+                let shards: Vec<ShardIdentifier> = Decode::decode(&mut shards.as_slice())
+                    .sgx_error_with_log("error decoding shards")?;
                 for s in shards {
                     if !state::exists(&s) {
                         info!("Initialized new shard that was found on chain: {:?}", s);
@@ -403,7 +407,8 @@ pub fn update_states(header: Header) -> SgxResult<()> {
                         .map(|key| WorkerRequest::ChainStorage(key, Some(header.hash())))
                         .collect();
 
-                    let responses: Vec<WorkerResponse<Vec<u8>>> = worker_request(per_shard_request)?;
+                    let responses: Vec<WorkerResponse<Vec<u8>>> =
+                        worker_request(per_shard_request)?;
                     let per_shard_update_map = verify_worker_responses(responses, header.clone())?;
 
                     let mut state = state::load(&s)?;
@@ -412,7 +417,7 @@ pub fn update_states(header: Header) -> SgxResult<()> {
                     state::write(state, &s)?;
                 }
             }
-            None => info!("No shards are on the chain yet")
+            None => info!("No shards are on the chain yet"),
         };
     };
     Ok(())
@@ -596,8 +601,6 @@ fn verify_worker_responses(
 extern "C" {
     pub fn ocall_read_ipfs(
         ret_val: *mut sgx_status_t,
-        enc_state: *mut u8,
-        enc_state_size: u32,
         cid: *const u8,
         cid_size: u32,
     ) -> sgx_status_t;
@@ -629,6 +632,9 @@ extern "C" {
 pub extern "C" fn test_main_entrance() -> size_t {
     rsgx_unit_tests!(
         state::test_encrypted_state_io_works,
+        ipfs::test_creates_ipfs_content_struct_works,
+        ipfs::test_verification_ok_for_correct_content,
+        ipfs::test_verification_fails_for_incorrect_content,
         test_ocall_read_write_ipfs,
         test_ocall_worker_request
     )
@@ -638,7 +644,7 @@ fn test_ocall_read_write_ipfs() {
     info!("testing IPFS read/write. Hopefully ipfs daemon is running...");
     let mut rt: sgx_status_t = sgx_status_t::SGX_ERROR_UNEXPECTED;
     let mut cid_buf: Vec<u8> = vec![0; 46];
-    let enc_state: Vec<u8> = vec![20; 36];
+    let enc_state: Vec<u8> = vec![20; 4 * 512 * 1024];
 
     let _res = unsafe {
         ocall_write_ipfs(
@@ -650,18 +656,28 @@ fn test_ocall_read_write_ipfs() {
         )
     };
 
-    let mut ret_state = vec![0; 36];
-    let _res = unsafe {
+    let res = unsafe {
         ocall_read_ipfs(
             &mut rt as *mut sgx_status_t,
-            ret_state.as_mut_ptr(),
-            ret_state.len() as u32,
             cid_buf.as_ptr(),
             cid_buf.len() as u32,
         )
     };
 
-    assert_eq!(enc_state, ret_state);
+    if res == sgx_status_t::SGX_SUCCESS {
+        let cid = std::str::from_utf8(&cid_buf).unwrap();
+        let mut f = File::open(&cid).unwrap();
+        let mut content_buf = Vec::new();
+        f.read_to_end(&mut content_buf).unwrap();
+        info!("reading file {:?} of size {} bytes", f, &content_buf.len());
+
+        let mut ipfs_content = IpfsContent::new(cid, content_buf);
+        let verification = ipfs_content.verify();
+        assert_eq!(verification.is_ok(), true);
+    } else {
+        error!("was not able to write to file");
+        assert!(false);
+    }
 }
 
 // TODO: this is redundantly defined in worker/src/main.rs

enclave/src/lib.rs

@@ -16,8 +16,8 @@
 */
 
 use std::fs;
-use std::vec::Vec;
 use std::io::Write;
+use std::vec::Vec;
 
 use log::*;
 use sgx_tcrypto::rsgx_sha256_slice;

enclave/src/state.rs

@@ -12,8 +12,8 @@ use sp_io::SgxExternalitiesTrait;
 use sp_runtime::traits::Dispatchable;
 
 use crate::{
-    AccountId, State, Stf, TrustedCall, TrustedCallSigned, TrustedGetter, TrustedGetterSigned,
-    ShardIdentifier, SUBSRATEE_REGISTRY_MODULE, UNSHIELD,
+    AccountId, ShardIdentifier, State, Stf, TrustedCall, TrustedCallSigned, TrustedGetter,
+    TrustedGetterSigned, SUBSRATEE_REGISTRY_MODULE, UNSHIELD,
 };
 use sp_core::blake2_256;
 
@@ -71,14 +71,12 @@ impl Stf {
 
     pub fn update_storage(ext: &mut State, map_update: &HashMap<Vec<u8>, Option<Vec<u8>>>) {
         ext.execute_with(|| {
-            map_update
-                .iter()
-                .for_each(|(k, v)| {
-                    match v {
-                        Some(value) => sp_io::storage::set(k, value),
-                        None => sp_io::storage::clear(k)
-                    };
-                });
+            map_update.iter().for_each(|(k, v)| {
+                match v {
+                    Some(value) => sp_io::storage::set(k, value),
+                    None => sp_io::storage::clear(k),
+                };
+            });
         });
     }
 
@@ -210,7 +208,10 @@ impl Stf {
     }
 
     pub fn get_storage_hashes_to_update_for_getter(getter: &TrustedGetterSigned) -> Vec<Vec<u8>> {
-        info!("No specific storage updates needed for getter. Returning those for on block: {:?}", getter.getter);
+        info!(
+            "No specific storage updates needed for getter. Returning those for on block: {:?}",
+            getter.getter
+        );
         Self::storage_hashes_to_update_on_block()
     }