Merge branch 'master' of github.com:openethereum/openethereum into adria0/fix/tests

Author: adria0

.github/workflows/build-test.yml

@@ -9,6 +9,9 @@ on:
 jobs:
   build-tests:
     name:                       Test and Build
+    env:
+      SCCACHE_CACHE_SIZE:       "1G"
+      SCCACHE_IDLE_TIMEOUT:     0
     strategy:
       matrix:
         platform:

.github/workflows/build.yml

@@ -10,6 +10,9 @@ on:
 jobs:
   build:
     name:                       Build Release
+    env:
+      SCCACHE_CACHE_SIZE:       "1G"
+      SCCACHE_IDLE_TIMEOUT:     0
     strategy:
       matrix:
         platform:
@@ -89,4 +92,4 @@ jobs:
           path:                 artifacts/
       - name:                   Prepare build directory for cache
         shell: bash
-        run:                    bash scripts/actions/clean-target.sh
+        run:                    bash scripts/actions/clean-target.sh

.github/workflows/check.yml

@@ -10,6 +10,9 @@ jobs:
   check:
     name:                       Check
     runs-on:                    ubuntu-16.04
+    env:
+      SCCACHE_CACHE_SIZE:       "1G"
+      SCCACHE_IDLE_TIMEOUT:     0    
     steps:
       - name:                   Checkout sources
         uses:                   actions/checkout@master

ethcore/snapshot/src/lib.rs

@@ -435,6 +435,8 @@ impl StateRebuilder {
 			}
 		}
 
+		let batch = self.db.drain_transaction_overlay()?;
+		self.db.backing().write(batch)?;
 		Ok(())
 	}
 

parity/db/rocksdb/migration.rs

@@ -186,7 +186,10 @@ fn migrate_database(version: u32, db_path: &Path, mut migrations: MigrationManag
 
 	// completely in-place migration leads to the paths being equal.
 	// in that case, no need to shuffle directories.
-	if temp_path == db_path { return Ok(()) }
+	if temp_path == db_path {
+		trace!(target: "migrate", "In-place migration ran; leaving old database in place.");
+		return Ok(())
+	}
 
 	// create backup
 	fs::rename(&db_path, &backup_path)?;

scripts/actions/build-windows.sh

@@ -1,9 +1,8 @@
 #!/bin/bash
 set -e # fail on any error
 set -u # treat unset variables as error
-
-  # NOTE: Enables the aes-ni instructions for RustCrypto dependency.
-  # If you change this please remember to also update .cargo/config
+# NOTE: Enables the aes-ni instructions for RustCrypto dependency.
+# If you change this please remember to also update .cargo/config
 export RUSTFLAGS=" -Ctarget-feature=+aes,+sse2,+ssse3 -Ctarget-feature=+crt-static  -Clink-arg=-s"
 
 echo "_____ Build OpenEthereum and tools _____"

scripts/actions/install-sccache.ps1

@@ -1,7 +1,5 @@
 #!/usr/bin/env pwsh
 $os=$args[0]
-$SCCACHE_CACHE_SIZE="1G"
-$SCCACHE_IDLE_TIMEOUT=0
 $version="0.2.12"
 echo "Current OS:" $os
 switch ($os){

scripts/docker/alpine/Dockerfile

@@ -11,6 +11,7 @@ RUN apk add --no-cache \
   linux-headers \
   perl \
   rust \
+  git \
   clang-dev \
   llvm-dev
 

util/journaldb/src/archivedb.rs

@@ -142,9 +142,8 @@ impl JournalDB for ArchiveDB {
 		Ok(0)
 	}
 
-	fn inject(&mut self, batch: &mut DBTransaction) -> io::Result<u32> {
-		let mut inserts = 0usize;
-		let mut deletes = 0usize;
+	fn drain_transaction_overlay(&mut self) -> io::Result<DBTransaction> {
+		let mut batch = DBTransaction::new();
 
 		for i in self.overlay.drain() {
 			let (key, (value, rc)) = i;
@@ -153,19 +152,17 @@ impl JournalDB for ArchiveDB {
 					return Err(error_key_already_exists(&key));
 				}
 				batch.put(self.column, key.as_bytes(), &value);
-				inserts += 1;
 			}
 			if rc < 0 {
 				assert!(rc == -1);
 				if self.backing.get(self.column, key.as_bytes())?.is_none() {
 					return Err(error_negatively_reference_hash(&key));
 				}
 				batch.delete(self.column, key.as_bytes());
-				deletes += 1;
 			}
 		}
 
-		Ok((inserts + deletes) as u32)
+		Ok(batch)
 	}
 
 	fn latest_era(&self) -> Option<u64> { self.latest_era }
@@ -209,7 +206,7 @@ mod tests {
 	use hash_db::{HashDB, EMPTY_PREFIX};
 	use super::*;
 	use kvdb_memorydb;
-	use crate::{JournalDB, inject_batch, commit_batch};
+	use crate::{JournalDB, drain_overlay, commit_batch};
 
 	#[test]
 	fn insert_same_in_fork() {
@@ -463,11 +460,11 @@ mod tests {
 	fn inject() {
 		let mut jdb = ArchiveDB::new(Arc::new(kvdb_memorydb::create(1)), 0);
 		let key = jdb.insert(EMPTY_PREFIX, b"dog");
-		inject_batch(&mut jdb).unwrap();
+		drain_overlay(&mut jdb).unwrap();
 
 		assert_eq!(jdb.get(&key, EMPTY_PREFIX).unwrap(), b"dog".to_vec());
 		jdb.remove(&key, EMPTY_PREFIX);
-		inject_batch(&mut jdb).unwrap();
+		drain_overlay(&mut jdb).unwrap();
 
 		assert!(jdb.get(&key, EMPTY_PREFIX).is_none());
 	}

util/journaldb/src/earlymergedb.rs

@@ -474,11 +474,9 @@ impl JournalDB for EarlyMergeDB {
 		Ok(0)
 	}
 
-	fn inject(&mut self, batch: &mut DBTransaction) -> io::Result<u32> {
-		let mut ops = 0;
+	fn drain_transaction_overlay(&mut self) -> io::Result<DBTransaction> {
+		let mut batch = DBTransaction::new();
 		for (key, (value, rc)) in self.overlay.drain() {
-			if rc != 0 { ops += 1 }
-
 			match rc {
 				0 => {}
 				1 => {
@@ -497,7 +495,7 @@ impl JournalDB for EarlyMergeDB {
 			}
 		}
 
-		Ok(ops)
+		Ok(batch)
 	}
 
 	fn consolidate(&mut self, with: super::MemoryDB) {
@@ -529,7 +527,7 @@ mod tests {
 	use hash_db::{HashDB, EMPTY_PREFIX};
 	use super::*;
 	use kvdb_memorydb;
-	use crate::{inject_batch, commit_batch};
+	use crate::{drain_overlay, commit_batch};
 
 	#[test]
 	fn insert_same_in_fork() {
@@ -1050,11 +1048,11 @@ mod tests {
 	fn inject() {
 		let mut jdb = new_db();
 		let key = jdb.insert(EMPTY_PREFIX, b"dog");
-		inject_batch(&mut jdb).unwrap();
+		drain_overlay(&mut jdb).unwrap();
 
 		assert_eq!(jdb.get(&key, EMPTY_PREFIX).unwrap(), b"dog".to_vec());
 		jdb.remove(&key, EMPTY_PREFIX);
-		inject_batch(&mut jdb).unwrap();
+		drain_overlay(&mut jdb).unwrap();
 
 		assert!(jdb.get(&key, EMPTY_PREFIX).is_none());
 	}

util/journaldb/src/lib.rs

@@ -72,11 +72,8 @@ pub trait JournalDB: HashDB<KeccakHasher, DBValue> {
 	/// Commit all queued insert and delete operations without affecting any journalling -- this requires that all insertions
 	/// and deletions are indeed canonical and will likely lead to an invalid database if that assumption is violated.
 	///
-	/// Any keys or values inserted or deleted must be completely independent of those affected
-	/// by any previous `commit` operations. Essentially, this means that `inject` can be used
-	/// either to restore a state to a fresh database, or to insert data which may only be journalled
-	/// from this point onwards.
-	fn inject(&mut self, batch: &mut DBTransaction) -> io::Result<u32>;
+	/// Returns a transaction to be committed.
+	fn drain_transaction_overlay(&mut self) -> io::Result<DBTransaction>;
 
 	/// State data query
 	fn state(&self, _id: &H256) -> Option<Bytes>;
@@ -213,12 +210,11 @@ pub fn new_memory_db() -> MemoryDB {
 	MemoryDB::from_null_node(&rlp::NULL_RLP, rlp::NULL_RLP.as_ref().into())
 }
 
-#[cfg(test)]
 /// Inject all changes in a single batch.
-pub fn inject_batch(jdb: &mut dyn JournalDB) -> io::Result<u32> {
-	let mut batch = jdb.backing().transaction();
-	let res = jdb.inject(&mut batch)?;
-	jdb.backing().write(batch).map(|_| res).map_err(Into::into)
+#[cfg(test)]
+pub fn drain_overlay(jdb: &mut dyn JournalDB) -> io::Result<()> {
+	let batch = jdb.drain_transaction_overlay()?;
+	jdb.backing().write(batch).map_err(Into::into)
 }
 
 /// Commit all changes in a single batch

util/journaldb/src/overlayrecentdb.rs

@@ -397,11 +397,9 @@ impl JournalDB for OverlayRecentDB {
 		Ok(ops as u32)
 	}
 
-	fn inject(&mut self, batch: &mut DBTransaction) -> io::Result<u32> {
-		let mut ops = 0;
+	fn drain_transaction_overlay(&mut self) -> io::Result<DBTransaction> {
+		let mut batch = DBTransaction::new();
 		for (key, (value, rc)) in self.transaction_overlay.drain() {
-			if rc != 0 { ops += 1 }
-
 			match rc {
 				0 => {}
 				_ if rc > 0 => {
@@ -417,7 +415,7 @@ impl JournalDB for OverlayRecentDB {
 			}
 		}
 
-		Ok(ops)
+		Ok(batch)
 	}
 
 	fn state(&self, key: &H256) -> Option<Bytes> {
@@ -507,7 +505,7 @@ mod tests {
 	use super::*;
 	use hash_db::{HashDB, EMPTY_PREFIX};
 	use kvdb_memorydb;
-	use crate::{JournalDB, inject_batch, commit_batch};
+	use crate::{JournalDB, drain_overlay, commit_batch};
 
 	fn new_db() -> OverlayRecentDB {
 		let backing = Arc::new(kvdb_memorydb::create(1));
@@ -1026,11 +1024,11 @@ mod tests {
 	fn inject() {
 		let mut jdb = new_db();
 		let key = jdb.insert(EMPTY_PREFIX, b"dog");
-		inject_batch(&mut jdb).unwrap();
+		drain_overlay(&mut jdb).unwrap();
 
 		assert_eq!(jdb.get(&key, EMPTY_PREFIX).unwrap(), b"dog".to_vec());
 		jdb.remove(&key, EMPTY_PREFIX);
-		inject_batch(&mut jdb).unwrap();
+		drain_overlay(&mut jdb).unwrap();
 
 		assert!(jdb.get(&key, EMPTY_PREFIX).is_none());
 	}

util/journaldb/src/refcounteddb.rs

@@ -193,12 +193,13 @@ impl JournalDB for RefCountedDB {
 		Ok(r)
 	}
 
-	fn inject(&mut self, batch: &mut DBTransaction) -> io::Result<u32> {
+	fn drain_transaction_overlay(&mut self) -> io::Result<DBTransaction> {
 		self.inserts.clear();
 		for remove in self.removes.drain(..) {
 			self.forward.remove(&remove, EMPTY_PREFIX);
 		}
-		self.forward.commit_to_batch(batch)
+		let mut batch = DBTransaction::new();
+		self.forward.commit_to_batch(&mut batch).map(|_| batch)
 	}
 
 	fn consolidate(&mut self, mut with: super::MemoryDB) {
@@ -224,7 +225,7 @@ mod tests {
 	use hash_db::{HashDB, EMPTY_PREFIX};
 	use super::*;
 	use kvdb_memorydb;
-	use crate::{JournalDB, inject_batch, commit_batch};
+	use crate::{JournalDB, drain_overlay, commit_batch};
 
 	fn new_db() -> RefCountedDB {
 		let backing = Arc::new(kvdb_memorydb::create(1));
@@ -338,11 +339,11 @@ mod tests {
 	fn inject() {
 		let mut jdb = new_db();
 		let key = jdb.insert(EMPTY_PREFIX, b"dog");
-		inject_batch(&mut jdb).unwrap();
+		drain_overlay(&mut jdb).unwrap();
 
 		assert_eq!(jdb.get(&key, EMPTY_PREFIX).unwrap(), b"dog".to_vec());
 		jdb.remove(&key, EMPTY_PREFIX);
-		inject_batch(&mut jdb).unwrap();
+		drain_overlay(&mut jdb).unwrap();
 
 		assert!(jdb.get(&key, EMPTY_PREFIX).is_none());
 	}

util/migration-rocksdb/src/lib.rs

@@ -21,7 +21,7 @@ use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::{fs, io, error};
 
-use log::{info, trace};
+use log::{info, trace, warn};
 use kvdb::DBTransaction;
 use kvdb_rocksdb::{CompactionProfile, Database, DatabaseConfig};
 
@@ -97,10 +97,12 @@ pub trait Migration {
 	/// Whether this migration alters any existing columns.
 	/// if not, then column families will simply be added and `migrate` will never be called.
 	fn alters_existing(&self) -> bool { true }
+	/// Whether this migration deletes data in any of the existing columns.
+	fn deletes_existing(&self) -> bool { false }
 	/// Version of the database after the migration.
 	fn version(&self) -> u32;
 	/// Migrate a source to a destination.
-	fn migrate(&mut self, source: Arc<Database>, config: &Config, destination: &mut Database, col: u32) -> io::Result<()>;
+	fn migrate(&mut self, source: Arc<Database>, config: &Config, destination: Option<&mut Database>, col: u32) -> io::Result<()>;
 }
 
 /// A simple migration over key-value pairs of a single column.
@@ -123,8 +125,15 @@ impl<T: SimpleMigration> Migration for T {
 
 	fn version(&self) -> u32 { SimpleMigration::version(self) }
 
-	fn migrate(&mut self, source: Arc<Database>, config: &Config, dest: &mut Database, col: u32) -> io::Result<()> {
+	fn migrate(&mut self, source: Arc<Database>, config: &Config, dest: Option<&mut Database>, col: u32) -> io::Result<()> {
 		let migration_needed = col == SimpleMigration::migrated_column_index(self);
+		let dest = match dest {
+			None => {
+				warn!(target: "migration", "No destination db provided. No changes made.");
+				return Ok(());
+			}
+			Some(dest) => dest,
+		};
 		let mut batch = Batch::new(config, col);
 
 		for (key, value) in source.iter(col) {
@@ -156,7 +165,7 @@ impl Migration for ChangeColumns {
 	fn columns(&self) -> u32 { self.post_columns }
 	fn alters_existing(&self) -> bool { false }
 	fn version(&self) -> u32 { self.version }
-	fn migrate(&mut self, _: Arc<Database>, _: &Config, _: &mut Database, _: u32) -> io::Result<()> {
+	fn migrate(&mut self, _: Arc<Database>, _: &Config, _: Option<&mut Database>, _: u32) -> io::Result<()> {
 		Ok(())
 	}
 }
@@ -170,10 +179,11 @@ pub struct VacuumAccountsBloom {
 impl Migration for VacuumAccountsBloom {
 	fn pre_columns(&self) -> u32 { self.columns }
 	fn columns(&self) -> u32 { self.columns }
-	fn alters_existing(&self) -> bool { true }
+	fn alters_existing(&self) -> bool { false }
+	fn deletes_existing(&self) -> bool { true }
 	fn version(&self) -> u32 { self.version }
 
-	fn migrate(&mut self, db: Arc<Database>, _config: &Config, _dest: &mut Database, col: u32) -> io::Result<()> {
+	fn migrate(&mut self, db: Arc<Database>, _config: &Config, _dest: Option<&mut Database>, col: u32) -> io::Result<()> {
 		if col != self.column_to_vacuum {
 			return Ok(())
 		}
@@ -300,7 +310,7 @@ impl Manager {
 				let mut new_db = Database::open(&db_config, temp_path_str)?;
 
 				for col in 0..current_columns {
-					migration.migrate(cur_db.clone(), &config, &mut new_db, col)?
+					migration.migrate(cur_db.clone(), &config, Some(&mut new_db), col)?
 				}
 
 				// next iteration, we will migrate from this db into the other temp.
@@ -309,6 +319,11 @@ impl Manager {
 
 				// remove the other temporary migration database.
 				let _ = fs::remove_dir_all(temp_idx.path(&db_root));
+			} else if migration.deletes_existing() {
+				// Migration deletes data in an existing column.
+				for col in 0..db_config.columns {
+					migration.migrate(cur_db.clone(), &config, None, col)?
+				}
 			} else {
 				// migrations which simply add or remove column families.
 				// we can do this in-place.
@@ -322,6 +337,8 @@ impl Manager {
 				}
 			}
 		}
+		// If `temp_path` is different from `old_path` we will shuffle database
+		// directories and delete the old paths.
 		Ok(temp_path)
 	}