CCS in Charm4Py (#289)

* initial changes

* fix string allocation in cython (still need to fix registration)

* start python interfacing

* register recvRemoteMessage:

* ccs server example

* build finished

* add to gitignore

* example compiling

* fix registration of handler

* better reduction

* readme for the example

* ccs communication working

* remove vscode json

* readme changes, everything is working

* Delete .vscode/settings.json

* fix bug in example with uncaught null at the end of msg

* remove comment

* variable renaming

---------

Co-authored-by: Steven Qie <[email protected]>
Co-authored-by: Steven Qie <[email protected]>

Author: ritvikrao

.gitignore

@@ -6,8 +6,13 @@ charm4py/_version.py
 charm4py/charmlib/*.o
 charm4py/charmlib/*.so
 charm4py/charmlib/charmlib_cython.c
+charm4py/charmlib/charmlib_cython.cpp
+charm4py/charmlib/charmlib_cython.html
 charm4py/charmlib/_charmlib_cffi.c
 charm4py/charmlib/build/
+charm4py.egg-info/
+charm4py/c_object_store.cpp
+charm4py/c_object_store.html
 __pycache__
 
 # ignore some common tmp files

charm4py/charm.py

@@ -153,6 +153,9 @@ def __init__(self):
         # TODO: maybe implement this buffer in c++
         self.future_get_buffer = {}
 
+        #registered methods for ccs
+        self.ccs_methods = {}
+
     def __init_profiling__(self):
         # these are attributes used only in profiling mode
         # list of Chare types that are registered and used internally by the runtime
@@ -1092,6 +1095,24 @@ def LBTurnInstrumentOn(self):
     def LBTurnInstrumentOff(self):
         self.lib.LBTurnInstrumentOff()
 
+    #functions for ccs 
+    def CcsRegisterHandler(self, handlername, handler):
+        self.ccs_methods[handlername] = handler
+        self.lib.CcsRegisterHandler(handlername, handler)
+
+    def CcsIsRemoteRequest(self):
+        self.lib.isRemoteRequest()
+    
+    def CcsSendReply(self, message):
+        self.lib.CcsSendReply(message)
+
+    def callHandler(self, handlername, data):
+        if handlername in self.ccs_methods:
+            self.ccs_methods[handlername](data)
+        else:
+            raise Charm4PyError('Handler ' + handlername + ' not registered')
+
+
 
 class CharmRemote(Chare):
 

charm4py/charmlib/ccharm.pxd

@@ -70,6 +70,39 @@ cdef extern from "charm.h":
     void CkStartQDExt_SectionCallback(int sid_pe, int sid_cnt, int rootPE, int ep);
     void CcdCallFnAfter(void (*CcdVoidFn)(void *userParam,double curWallTime), void *arg, double msecs);
 
+cdef extern from "conv-header.h":
+    ctypedef void (*CmiHandler)(void* )
+    cdef const int CmiReservedHeaderSize
+
+cdef extern from "sockRoutines.h":
+
+    ctypedef struct skt_ip_t:
+        int tag
+    
+    ctypedef struct ChMessageInt_t:
+        unsigned char[4] data
+
+cdef extern from "ccs-server.h":
+
+    ctypedef struct CcsSecAttr:
+        skt_ip_t ip
+        ChMessageInt_t port
+        ChMessageInt_t replySalt
+        unsigned char auth
+        unsigned char level
+    
+    ctypedef struct CcsImplHeader:
+        CcsSecAttr attr
+        char[32] handler
+        ChMessageInt_t pe
+        ChMessageInt_t replyFd
+        ChMessageInt_t len
+
+cdef extern from "conv-ccs.h":
+    void CcsRegisterHandlerExt(const char *ccs_handlername, void *fn);
+    int CcsIsRemoteRequest();
+    void CcsSendReply(int replyLen, const void *replyData);
+
 
 cdef extern from "spanningTree.h":
     void getPETopoTreeEdges(int pe, int rootPE, int *pes, int numpes, unsigned int bfactor,

charm4py/charmlib/charmlib_cython.pyx

@@ -9,6 +9,8 @@ from cpython.buffer  cimport PyObject_GetBuffer, PyBuffer_Release, PyBUF_ANY_CON
 from cpython.tuple   cimport PyTuple_New, PyTuple_SET_ITEM
 from cpython.int cimport PyInt_FromSsize_t
 from cpython.ref cimport Py_INCREF
+from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free
+from cython.operator cimport dereference
 
 from ..charm import Charm4PyError
 from .. import reduction as red
@@ -36,6 +38,8 @@ cdef object np_number = np.number
 
 # ------ global constants ------
 
+cdef dict _ccs_handlers = {}
+
 cdef enum:
   NUM_DCOPY_BUFS = 60   # max number of dcopy buffers
 
@@ -312,6 +316,28 @@ cdef object times = [0.0] * 3 # track time in [charm reduction callbacks, custom
 cdef bytes localMsg = b'L:' + (b' ' * sizeof(int))
 cdef char* localMsg_ptr = <char*>localMsg
 
+#cdef const int CmiReservedHeaderSize s= getCmiReservedHeaderSize()
+
+cdef struct remoteMsg:
+  int header_length
+  int data_length
+  char handler_name[32] # we know it can't be longer
+  char data[1024] #assume generous length at end
+
+cdef void recvRemoteMessage(void *msg) noexcept:
+
+    cdef void *shiftedMsg = msg + CmiReservedHeaderSize #move past reserved header
+    cdef remoteMsg* incomingMsgPtr = <remoteMsg*> shiftedMsg
+    cdef int handler_length = incomingMsgPtr.header_length
+    cdef int data_length = incomingMsgPtr.data_length
+
+    # turn char arrays into strings
+
+    handler_name = incomingMsgPtr.handler_name[:handler_length].decode('utf-8')
+    data = incomingMsgPtr.data[:data_length].decode('utf-8')
+    
+    charm.callHandler(handler_name, data)
+
 
 class CharmLib(object):
 
@@ -826,6 +852,21 @@ class CharmLib(object):
   def scheduleTagAfter(self, int tag, double msecs):
     CcdCallFnAfter(CcdCallFnAfterCallback, <void*>tag, msecs)
 
+  def CcsRegisterHandler(self, str handlername, object handler):
+    cdef bytes handler_bytes = handlername.encode("utf-8")
+    cdef const char* c_handlername = handler_bytes
+    CcsRegisterHandlerExt(c_handlername, <void *>recvRemoteMessage)
+  
+  def isRemoteRequest(self):
+    return bool(CcsIsRemoteRequest())
+  
+  def CcsSendReply(self, str message):
+    cdef bytes message_bytes = message.encode("utf-8")
+    cdef const char* replyData = message_bytes
+
+    cdef int replyLen = len(message_bytes)
+    CcsSendReply(replyLen, <const void*>replyData)
+
 
 # first callback from Charm++ shared library
 cdef void registerMainModule() noexcept:

docs/install.rst

@@ -58,6 +58,12 @@ the charm directory and run the following build command, then build Charm4Py::
     $ cd ../..
     $ pip install .
 
+On MacOS, you may need to set your DYLD_LIBRARY_PATH so that the dynamic linker can find Charm++ libraries. For example: 
+
+    $ export DYLD_LIBRARY_PATH=<charm_install_folder>/lib
+
+Replace <charm_install_folder> with the path where you built/installed Charm++(usually this is the charm_src/charm folder)
+
 Finally, if necessary, when installing dependencies or when running the install script, add the --user
 option to the Python command to complete the installation without permission errors.
 

examples/ccs/README.md

@@ -0,0 +1,6 @@
+### Instructions to run CCS Example 
+
+1. Start up your python virtual env if using one 
+2. Boot up the server through the command: python -m charmrun.start +p<N> ++server ++local ccs_server.py 
+3. Grab and store the the Server IP and Server Port that will appear in stdout when you run the above command 
+4. In another terminal, cd to the converse/ccstest example in the charm++ repository and run a client program through the command: ./client server_ip server_port 

examples/ccs/ccs_server.py

@@ -0,0 +1,28 @@
+from charm4py import charm, Chare, Array, Future, Reducer, Group
+
+def handler(msg):
+    print("CCS Ping handler called on " + str(charm.myPe()))
+
+    msg = msg.rstrip('\x00') #removes null characters from the end
+    answer = "Hello to sender " + str(msg) + " from PE " + str(charm.myPe()) + ".\n"
+    # send the answer back to the client
+    charm.CcsSendReply(answer)
+
+class RegisterPerChare(Chare):
+
+    def register(self, return_future, handler):
+        charm.CcsRegisterHandler("ping2", handler)
+        charm.CcsRegisterHandler("ping", handler)
+        self.reduce(return_future, Reducer.nop)
+
+def main(args):
+    # No need to initialize converse, because charm.start does this
+    # just register the handler
+    reg_wait = Future()
+    registers = Group(RegisterPerChare)
+    registers.register(reg_wait, handler)
+    reg_wait.get()
+    print("CCS Handlers registered . Waiting for net requests...")
+
+
+charm.start(main)

setup.py

@@ -263,9 +263,11 @@ class _renameInstalled(_install_lib):
     def __init__(self, *args, **kwargs):
         _install_lib.__init__(self, *args, **kwargs)
 
+    
     def install(self):
         log.info("Renaming libraries")
         outfiles = _install_lib.install(self)
+        '''
         for file in outfiles:
             if "c_object_store" in file and system == "darwin":
                 direc = os.path.dirname(file)
@@ -285,6 +287,7 @@ def install(self):
                 install_name_command += "/charmlib_cython.*.so"
                 log.info(install_name_command)
                 os.system(install_name_command)
+        '''
         return outfiles