cpython vm for privileged accounts

Author: learnforpractice

libraries/chain/eosio_contract.cpp

@@ -159,20 +159,11 @@ void apply_eosio_setcode(apply_context& context) {
 
    jit_account_deactivate(act.account.value);
 
-   if (act.vmtype == 1) {
-      apply_eosio_setcode_py(context);
-      return;
-   } else if (act.vmtype == 2) {
-      apply_eosio_setcode_evm(context);
-      return;
-   } else if (act.vmtype == 3) {
-      apply_eosio_setcode_rpc(context);
-      return;
-   }
    context.require_authorization(act.account);
 
-   FC_ASSERT( act.vmtype == 0 );
+//   FC_ASSERT( act.vmtype == 0 );
    FC_ASSERT( act.vmversion == 0 );
+   FC_ASSERT(act.code.size() > 0);
 
    fc::sha256 code_id; /// default ID == 0
 
@@ -196,9 +187,7 @@ void apply_eosio_setcode(apply_context& context) {
       a.last_code_update = context.control.pending_block_time();
       a.code_version = code_id;
       a.code.resize( code_size );
-      if( code_size > 0 )
-         memcpy( a.code.data(), act.code.data(), code_size );
-
+      memcpy( a.code.data(), act.code.data(), code_size );
    });
 
    const auto& account_sequence = db.get<account_sequence_object, by_name>(act.account);
@@ -209,55 +198,9 @@ void apply_eosio_setcode(apply_context& context) {
    if (new_size != old_size) {
       context.trx_context.add_ram_usage( act.account, new_size - old_size );
    }
-   vm_manager::get().setcode(0, act.account);
+   vm_manager::get().setcode(act.vmtype, act.account);
 }
 
-void apply_eosio_setcode_py(apply_context& context) {
-   auto& db = context.db;
-   auto  act = context.act.data_as<setcode>();
-   context.require_authorization(act.account);
-
-   FC_ASSERT( act.vmtype == 1 );
-   FC_ASSERT( act.vmversion == 0 );
-
-   auto code_id = fc::sha256::hash( act.code.data(), (uint32_t)act.code.size() );
-
-   if (act.code.size() > 5*1024) {
-      elog("++++act.code.size() ${n}", ("n", act.code.size()));
-      throw FC_EXCEPTION( fc::exception, "code size must be <= 5KB, actual code size is ${n}", ("n", act.code.size()));
-   }
-
-   const auto& account = db.get<account_object,by_name>(act.account);
-
-   int64_t code_size = (int64_t)act.code.size();
-   int64_t old_size = (int64_t)account.code.size() * config::setcode_ram_bytes_multiplier;
-   int64_t new_size = code_size * config::setcode_ram_bytes_multiplier;
-
-   FC_ASSERT( account.code_version != code_id, "contract is already running this version of code" );
-//   wlog( "set code: ${size}", ("size",act.code.size()));
-   db.modify( account, [&]( auto& a ) {
-      /** TODO: consider whether a microsecond level local timestamp is sufficient to detect code version changes*/
-      #warning TODO: update setcode message to include the hash, then validate it in validate
-      a.vm_type = act.vmtype;
-      a.last_code_update = context.control.pending_block_time();
-      a.code_version = code_id;
-      a.code.resize( code_size );
-      if( code_size > 0 )
-         memcpy( a.code.data(), act.code.data(), code_size );
-
-   });
-
-   const auto& account_sequence = db.get<account_sequence_object, by_name>(act.account);
-   db.modify( account_sequence, [&]( auto& aso ) {
-      aso.code_sequence += 1;
-   });
-
-   if (new_size != old_size) {
-      context.trx_context.add_ram_usage( act.account, new_size - old_size );
-   }
-   vm_manager::get().setcode(1, act.account);
-   //   micropython_interface::get().on_setcode(act.account, act.code);
-}
 
 void apply_eosio_setcode_evm(apply_context& context) {
    auto& db = context.db;
@@ -309,49 +252,6 @@ void apply_eosio_setcode_evm(apply_context& context) {
    }
 }
 
-void apply_eosio_setcode_rpc(apply_context& context) {
-   auto& db = context.db;
-   auto  act = context.act.data_as<setcode>();
-   context.require_authorization(act.account);
-
-   FC_ASSERT( act.vmtype == 3);
-   FC_ASSERT( act.vmversion == 0 );
-
-   auto code_id = fc::sha256::hash( act.code.data(), (uint32_t)act.code.size() );
-
-   const auto& account = db.get<account_object,by_name>(act.account);
-
-
-   int64_t code_size = (int64_t)act.code.size();
-   int64_t old_size = (int64_t)account.code.size() * config::setcode_ram_bytes_multiplier;
-   int64_t new_size = code_size * config::setcode_ram_bytes_multiplier;
-
-   FC_ASSERT( account.code_version != code_id, "contract is already running this version of code" );
-//   wlog( "set code: ${size}", ("size",act.code.size()));
-   db.modify( account, [&]( auto& a ) {
-      /** TODO: consider whether a microsecond level local timestamp is sufficient to detect code version changes*/
-      #warning TODO: update setcode message to include the hash, then validate it in validate
-      a.vm_type = act.vmtype;
-      a.last_code_update = context.control.pending_block_time();
-      a.code_version = code_id;
-      a.code.resize( code_size );
-      a.last_code_update = context.control.pending_block_time();
-      memcpy( a.code.data(), act.code.data(), code_size );
-
-   });
-
-   const auto& account_sequence = db.get<account_sequence_object, by_name>(act.account);
-   db.modify( account_sequence, [&]( auto& aso ) {
-      aso.code_sequence += 1;
-   });
-
-   if (new_size != old_size) {
-      context.trx_context.add_ram_usage( act.account, new_size - old_size );
-   }
-}
-
-
-
 void apply_eosio_setabi(apply_context& context) {
    auto& db  = context.db;
    auto  act = context.act.data_as<setabi>();

libraries/vm/CMakeLists.txt

@@ -2,3 +2,4 @@ add_subdirectory( vm_py )
 add_subdirectory( vm_wasm )
 add_subdirectory( vm_eth )
 add_subdirectory( vm_native )
+add_subdirectory( vm_cpython )

libraries/vm/vm_cpython/CMakeLists.txt

@@ -0,0 +1,32 @@
+set(PYTHON3 "LD_LIBRARY_PATH=${CMAKE_SOURCE_DIR}/libraries/python;${CMAKE_SOURCE_DIR}/libraries/python/dist/bin/python3")
+
+add_custom_command(
+    OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/vm_cpython.wrap.cpp
+    COMMAND ${PYTHON3} -m cython --cplus ${CMAKE_CURRENT_SOURCE_DIR}/vm_cpython.pyx -o ${CMAKE_CURRENT_SOURCE_DIR}/vm_cpython.wrap.cpp
+    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/vm_cpython.pyx
+)
+
+add_library(vm_cpython SHARED 
+            vm_cpython.wrap.cpp
+            vm_cpython.cpp)
+
+target_link_libraries(vm_cpython PRIVATE eosiolib_native db_api python3)
+
+target_include_directories(vm_cpython PRIVATE ${Boost_INCLUDE_DIR}
+#   PRIVATE ${CMAKE_SOURCE_DIR}/contracts/libc++/upstream/include
+#   PRIVATE ${CMAKE_SOURCE_DIR}/contracts/libc++/include/libc++/upstream/include
+#   PRIVATE ${CMAKE_SOURCE_DIR}/contracts/libc++/musl/include/musl/upstream/include
+#    PRIVATE ${CMAKE_SOURCE_DIR}/contracts/musl/upstream/include
+    PRIVATE ${CMAKE_SOURCE_DIR}/externals/magic_get/include
+    PRIVATE ${CMAKE_BINARY_DIR}/libraries/appbase/include
+    PUBLIC ${CMAKE_SOURCE_DIR}/libraries/chainbase/include
+    PUBLIC ${CMAKE_SOURCE_DIR}/libraries/softfloat/source/include
+
+    PUBLIC ${CMAKE_SOURCE_DIR}/libraries/python/Include
+    PUBLIC ${CMAKE_SOURCE_DIR}/libraries/python
+
+    PRIVATE ${CMAKE_BINARY_DIR}/libraries/fc/include
+#    PRIVATE ${CMAKE_BINARY_DIR}/libraries/chain/include
+    PRIVATE ${CMAKE_SOURCE_DIR}/contracts
+)
+

libraries/vm/vm_cpython/vm_cpython.cpp

@@ -0,0 +1,50 @@
+#include "vm_cpython.h"
+
+#include <Python.h>
+
+#include <eosiolib_native/vm_api.h>
+
+static struct vm_api* s_api;
+
+extern "C" PyObject* PyInit_vm_cpython();
+
+int init_cpython();
+int cpython_setcode(uint64_t account, string& code);
+int cpython_apply(unsigned long long receiver, unsigned long long account, unsigned long long action);
+
+void get_code(uint64_t account, string& code) {
+   size_t size;
+   const char* _code = get_vm_api()->get_code(account, &size);
+   code = string(_code, size);
+}
+
+int init_cpython_() {
+   PyInit_vm_cpython();
+   return 1;
+}
+
+void vm_init(struct vm_api* api) {
+   s_api = api;
+   vm_register_api(api);
+   PyInit_vm_cpython();
+}
+
+void vm_deinit() {
+   printf("vm_native finalize\n");
+}
+
+struct vm_api* get_vm_api() {
+   return s_api;
+}
+
+int vm_setcode(uint64_t account) {
+   string code;
+   get_code(account, code);
+   return cpython_setcode(account, code);
+}
+
+int vm_apply(uint64_t receiver, uint64_t account, uint64_t act) {
+   return cpython_apply(receiver, account, act);
+}
+
+

libraries/vm/vm_cpython/vm_cpython.h

@@ -0,0 +1,6 @@
+#include <string>
+#include <vector>
+using namespace std;
+
+void get_code(uint64_t account, string& code);
+int init_cpython_();

libraries/vm/vm_cpython/vm_cpython.pyx

@@ -0,0 +1,56 @@
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+from libcpp.map cimport map
+import os
+import sys
+import imp
+import logging
+import traceback
+
+import db
+import eoslib
+
+cdef extern from "<stdint.h>":
+    ctypedef unsigned long long uint64_t
+
+cdef extern from "vm_cpython.h":
+    void get_code(uint64_t account, string& code)
+
+sys.modules['ustruct'] = sys.modules['struct']
+
+py_modules = {}
+
+def _get_code(uint64_t account):
+    cdef string code
+    get_code(account,  code)
+    return code
+
+def _load_module(account, code):
+    try:
+        module = imp.new_module(eoslib.n2s(account))
+        exec(code, module.__dict__)
+        py_modules[account] = module
+        return module
+    except Exception as e:
+        logging.exception(e)
+    return None
+
+cdef extern int cpython_setcode(uint64_t account, string& code) with gil:
+    if _load_module(account, code):
+        return 1
+    return 0
+
+cdef extern int cpython_apply(unsigned long long receiver, unsigned long long account, unsigned long long action) with gil:
+    try:
+        if receiver in py_modules:
+            py_modules[receiver].apply(receiver, account, action)
+            return 1
+        code = _get_code(receiver)
+        mod = _load_module(receiver, code)
+        if not mod:
+            return 0
+        mod.apply(receiver, account, action)
+        return 1
+    except Exception as e:
+        logging.exception(e)
+    return 0

libraries/vm_manager/vm_manager.cpp

@@ -26,12 +26,13 @@ using namespace eosio::chain;
 namespace bio = boost::iostreams;
 
 
-static const int TYPE_BINARYEN = 0;
-static const int TYPE_PY = 1;
-static const int TYPE_ETH = 2;
-static const int TYPE_WAVM = 3;
-static const int TYPE_IPC = 4;
-static const int TYPE_NATIVE = 5;
+static const int VM_TYPE_BINARYEN = 0;
+static const int VM_TYPE_PY = 1;
+static const int VM_TYPE_ETH = 2;
+static const int VM_TYPE_WAVM = 3;
+static const int VM_TYPE_IPC = 4;
+static const int VM_TYPE_NATIVE = 5;
+static const int VM_TYPE_CPYTHON = 6;
 
 
 namespace eosio {
@@ -153,6 +154,7 @@ static const char* vm_libs_path[] = {
 
 static const char * ipc_server_lib = "../libs/libipc_server" DYLIB_SUFFIX;
 static const char * vm_native_lib = "../libs/libvm_native" DYLIB_SUFFIX;
+static const char * vm_cpython_lib = "../libs/libvm_cpython" DYLIB_SUFFIX;
 
 vm_manager& vm_manager::get() {
    static vm_manager *mngr = nullptr;
@@ -197,11 +199,13 @@ bool vm_manager::init(struct vm_api* api) {
 
    if (!get_vm_api()->has_option("no-ipc")) {
       if (this->api->run_mode() == 0) {//server
-         load_vm_from_path(TYPE_IPC, ipc_server_lib);
+         load_vm_from_path(VM_TYPE_IPC, ipc_server_lib);
       }
    }
 
-   load_vm_from_path(TYPE_NATIVE, vm_native_lib);
+   load_vm_from_path(VM_TYPE_NATIVE, vm_native_lib);
+
+   load_vm_from_path(VM_TYPE_CPYTHON, vm_cpython_lib);
 
    return true;
 }
@@ -356,9 +360,9 @@ int vm_manager::load_vm_from_path(int vm_type, const char* vm_path) {
       dlclose(__itr->second->handle);
    }
 
+   wlog("+++++++++++loading ${n1} cost: ${n2}", ("n1",vm_path)("n2", get_microseconds() - start));
    vm_init(this->api);
 
-   wlog("+++++++++++loading ${n1} cost: ${n2}", ("n1",vm_path)("n2", get_microseconds() - start));
    std::unique_ptr<vm_calls> calls = std::make_unique<vm_calls>();
    calls->version = 0;
    calls->handle = handle;
@@ -508,8 +512,8 @@ int vm_manager::setcode(int type, uint64_t account) {
    if (this->api->run_mode() == 0) {
       if (is_trusted_account(account)) {
       } else {
-         if (vm_map.find(TYPE_IPC) != vm_map.end()) {
-            type = TYPE_IPC;
+         if (vm_map.find(VM_TYPE_IPC) != vm_map.end()) {
+            type = VM_TYPE_IPC;
          }
       }
    }
@@ -529,9 +533,9 @@ int vm_manager::apply(int type, uint64_t receiver, uint64_t account, uint64_t ac
    if (is_trusted_account(account)) {
 
    } else {
-      auto itr = vm_map.find(TYPE_IPC);
+      auto itr = vm_map.find(VM_TYPE_IPC);
       if (itr != vm_map.end()) {
-         type = TYPE_IPC;
+         type = VM_TYPE_IPC;
       }
    }
    return local_apply(type, receiver, account, act);
@@ -543,7 +547,7 @@ int vm_manager::local_apply(int type, uint64_t receiver, uint64_t account, uint6
       load_vm_from_ram(type, vm_names[type]);
    }
 */
-   if (vm_map[TYPE_NATIVE]->apply(receiver, account, act)) {
+   if (vm_map[VM_TYPE_NATIVE]->apply(receiver, account, act)) {
       return 1;
    }
 
@@ -580,7 +584,7 @@ int vm_manager::local_apply(int type, uint64_t receiver, uint64_t account, uint6
             //there is a small probability that the next BP can not execute all the code in one block time
             //at the situation of network overloading, that may cause a fork and BP will lose some revenue
             //a vote on jit loading can solve this problem.
-            if (vm_map[TYPE_WAVM]->apply(receiver, account, act)) {
+            if (vm_map[VM_TYPE_WAVM]->apply(receiver, account, act)) {
                return 1;
             }
          }