Bloom Search Update

CHANGELOG.md

@@ -1,3 +1,15 @@
+## Version: 0.1.18052022
+- Added checksum for sha256
+- Unsupported type ERROR in fl now print detected Type
+- Pubkey conversion functions added pub2upub, to_cpub, point_to_cpub
+- function bloom_check_add_mcpu added for later use
+- bloom_para, Fill_in_bloom, check_in_bloom helpful wrappers for bloom creation and test
+- prepare_bin_file, prepare_bin_file_work ideally to be used with rmd or eth to create .bin file
+- Once created the bloom can be saved and reloaded by dump_bloom_file, read_bloom_file
+- Option from .bin sorted file __20 byte each item__ to directly load into RAM using Load_data_to_memory
+- check for existence using check_collision function alongside Load_data_to_memory
+
+
 ## Version: 0.1.29122021
 - Added version
 - privatekey_loop_h160_sse for using SSE advantage and privatekey_loop_h160 for old cpu

README.md

@@ -136,6 +136,35 @@ ice.privatekey_to_coinaddress(ice.COIN_DASH, 0, True, 0x1b1f)
 
 ice.privatekey_to_coinaddress(ice.COIN_RVN, 0, True, 0x1b1f)
 : 'RDePbshJ2nudXVcN1kQbZv88jwwKWs42X6'
+
+ice.checksum('What is the use of it?').hex()
+: '6bbe6051'
+
+xx = ['43253', 'hfoiefcope', 'cvt9', '4329r32hf39', '4e329jf4iehgf43']
+_bits, _hashes, _bf = ice.Fill_in_bloom(xx, 0.000001)
+print(ice.check_in_bloom('cvt9', _bits, _hashes, _bf))
+: True
+
+ice.dump_bloom_file("my_bloom_file.bin", _bits, _hashes, _bf)
+_bits, _hashes, _bf = ice.read_bloom_file("my_bloom_file.bin")
+print(ice.check_in_bloom('cvt9', _bits, _hashes, _bf))
+: True
+
+P = ice.pub2upub('02CEB6CBBCDBDF5EF7150682150F4CE2C6F4807B349827DCDBDD1F2EFA885A2630')
+print(P.hex())
+: '04ceb6cbbcdbdf5ef7150682150f4ce2c6f4807b349827dcdbdd1f2efa885a26302b195386bea3f5f002dc033b92cfc2c9e71b586302b09cfe535e1ff290b1b5ac'
+
+ice.point_to_cpub(P)
+: '02ceb6cbbcdbdf5ef7150682150f4ce2c6f4807b349827dcdbdd1f2efa885a2630'
+
+ice.to_cpub('04ceb6cbbcdbdf5ef7150682150f4ce2c6f4807b349827dcdbdd1f2efa885a26302b195386bea3f5f002dc033b92cfc2c9e71b586302b09cfe535e1ff290b1b5ac')
+: '02ceb6cbbcdbdf5ef7150682150f4ce2c6f4807b349827dcdbdd1f2efa885a2630'
+
+ice.prepare_bin_file("eth_addr_file.txt", "eth_sorted.bin", True, True)
+ice.Load_data_to_memory("eth_sorted.bin", False)
+ice.check_collision(this_key_eth_bytes)
+: True
+
 ```
 # Speed
 On my old Laptop with i7 4810 MQ CPU

secp256k1.py

@@ -8,7 +8,8 @@
 import os
 import sys
 import ctypes
-
+import math
+import pickle
 
 ###############################################################################
 N = 0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141
@@ -182,7 +183,15 @@
 #==============================================================================
 ice.bloom_batch_add.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_int, ctypes.c_ulonglong, ctypes.c_ubyte, ctypes.c_char_p] #chunk, buff, len, 0_1, _bits, _hashes, _bf
 #==============================================================================
+ice.bloom_check_add_mcpu.argtypes = [ctypes.c_void_p, ctypes.c_ulonglong, ctypes.c_char_p, ctypes.c_int, ctypes.c_int, ctypes.c_ulonglong, ctypes.c_ubyte, ctypes.c_char_p] #buff, num_items, found_array, len, 0_1, _bits, _hashes, _bf
+#==============================================================================
 ice.test_bit_set_bit.argtypes = [ctypes.c_char_p, ctypes.c_ulonglong, ctypes.c_int] #_bf, _bits, 0_1
+#==============================================================================
+#==============================================================================
+ice.Load_data_to_memory.argtypes = [ctypes.c_char_p, ctypes.c_bool] #sorted_bin_file_h160, verbose
+#==============================================================================
+ice.check_collision.argtypes = [ctypes.c_char_p] #h160
+ice.check_collision.restype = ctypes.c_bool #True or False
 
 ice.init_secp256_lib()
 #==============================================================================
@@ -426,12 +435,18 @@ def btc_pvk_to_wif(pvk, is_compressed=True):
         return b58_encode(inp + res2[:4])
     else: return inp
 #==============================================================================
+def checksum(inp):
+    ''' Input string output double sha256 checksum 4 bytes'''
+    res = get_sha256(inp)
+    res2 = get_sha256(res)
+    return res2[:4]
+#==============================================================================
 def fl(sstr, length=64):
     ''' Fill input to exact 32 bytes. If input is int or str the return is str. if input is bytes return is bytes'''
     if type(sstr) == int: fixed = hex(sstr)[2:].zfill(length)
     elif type(sstr) == str: fixed = sstr[2:].zfill(length) if sstr[:2].lower() == '0x' else sstr.zfill(length)
     elif type(sstr) == bytes: fixed = (b'\x00') * (32 - len(sstr)) + sstr
-    else: print("[Error] Input format [Integer] [Hex] [Bytes] allowed only")
+    else: print("[Error] Input format [Integer] [Hex] [Bytes] allowed only. Detected : ", type(sstr))
     return fixed
 #==============================================================================
 def pbkdf2_hmac_sha512_dll(words):
@@ -613,4 +628,101 @@ def privatekey_group_to_ETH_address_bytes(pvk_int, m):
     start_pvk = fl(pvk_int).encode('utf8')
     res = _privatekey_group_to_ETH_address_bytes(start_pvk, m)
     return bytes(bytearray(res))
-#==============================================================================
+#==============================================================================
+def bloom_check_add_mcpu(bigbuff, num_items, sz, check_add, bloom_bits, bloom_hashes, bloom_filter):
+    found_array = (b'\x00') * num_items
+#    sz = 32; check_add = 0 for check and 1 for add
+    ice.bloom_check_add_mcpu(bigbuff, num_items, found_array, sz, check_add, bloom_bits, bloom_hashes, bloom_filter)
+    return found_array
+#==============================================================================
+def to_cpub(pub_hex):
+    P = pub_hex
+    if len(pub_hex) > 70:
+        P = '02' + pub_hex[2:66] if int(pub_hex[66:],16)%2 == 0 else '03' + pub_hex[2:66]
+    return P
+#==============================================================================
+def point_to_cpub(pubkey_bytes):
+    P = pubkey_bytes.hex()
+    if len(P) > 70:
+        P = '02' + P[2:66] if int(P[66:],16)%2 == 0 else '03' + P[2:66]
+    return P
+#==============================================================================
+def pub2upub(pub_hex):
+    ''' Covert [C or U] pubkey to Point'''
+    x = pub_hex[2:66]
+    if len(pub_hex) < 70:
+        y = get_x_to_y(x, int(pub_hex[:2],16)%2 == 0).hex()
+    else:
+        y = pub_hex[66:].zfill(64)
+    return bytes.fromhex('04'+ x + y)
+#==============================================================================
+def bloom_para(_items, _fp = 0.000001):
+    _bits = math.ceil((_items * math.log(_fp)) / math.log(1 / pow(2, math.log(2))))
+    if _bits % 8: _bits = 8*(1 + (_bits//8))
+    _hashes = round((_bits / _items) * math.log(2))
+    return _bits, _hashes
+#==============================================================================
+def Fill_in_bloom(inp_list, _fp = 0.000001):
+    _bits, _hashes = bloom_para(len(inp_list), _fp)
+    _bf = (b'\x00') * (_bits//8)
+    for line in inp_list:
+        if type(line) != bytes: tt = str(line).encode("utf-8")
+        else: tt = line
+        res = ice.bloom_check_add(tt, len(tt), 1, _bits, _hashes, _bf)  # 1 = Add
+    del res
+    return _bits, _hashes, _bf
+#==============================================================================
+def dump_bloom_file(output_bloom_file_name, _bits, _hashes, _bf):
+    with open(output_bloom_file_name, 'wb') as f:
+        pickle.dump((_bits, _hashes, _bf), f)
+
+def read_bloom_file(bloom_file_name):
+    '''It will return the 3 output as _bits, _hashes, _bf'''
+    with open(bloom_file_name, 'rb') as f:
+        return pickle.load(f)
+#==============================================================================
+def check_in_bloom(this_line, _bits, _hashes, _bf):
+    if type(this_line) != bytes: tt = str(this_line).encode("utf-8")
+    else: tt = this_line
+    if ice.bloom_check_add(tt, len(tt), 0, _bits, _hashes, _bf) > 0: return True
+    else: return False
+#==============================================================================
+def prepare_bin_file_work(in_file, out_file, lower = False):
+    use0x = False
+    inp_list = [line.split()[0].lower() if lower else line.split()[0] for line in open(in_file,'r')]
+    if inp_list[0][:2] == '0x': use0x = True
+
+    with open(out_file, 'wb') as f:
+        if use0x:
+            tmp_list = [line[2:] for line in inp_list]
+        else:
+            tmp_list = inp_list
+        tmp_list.sort()
+        for line in tmp_list:
+            f.write(bytes.fromhex(line))
+#==============================================================================
+def prepare_bin_file(in_file, out_file, overwrite = False, lower = False):
+
+    if os.path.isfile(out_file) == False:
+        prepare_bin_file_work(in_file, out_file, lower)
+
+    else:
+        if not overwrite:
+            print(f'[+] File {out_file} already exist. It will be used as it is...')
+
+        else:
+            print(f'[+] File {out_file} already exist. Overwriting it...')
+            prepare_bin_file_work(in_file, out_file)
+#==============================================================================
+def Load_data_to_memory(input_bin_file, verbose = False):
+    '''input_bin_file is sorted h160 data of 20 bytes each element. 
+    ETH address can also work without 0x if sorted binary format'''
+    ice.Load_data_to_memory(input_bin_file.encode("utf-8"), verbose)
+
+#==============================================================================
+def check_collision(h160):
+    ''' h160 is the 20 byte hash to check for collision in data, already loaded in RAM.
+    Use the function Load_data_to_memory before calling this check'''
+
+    found = ice.check_collision(h160)
+    return found