feat(CTransformers): add support to CTransformers

extra/grpc/c_transformers/Makefile

@@ -0,0 +1,18 @@
+.PONY: ctransformers
+ctransformers:
+	@echo "Creating virtual environment..."
+	@conda create -n ctransformers python=3.11 -y
+	@echo "Virtual environment created."
+
+	@echo "Activating virtual environment..."
+	@. activate ctransformers
+
+	@echo "Installing dependencies..."
+	@pip install grpcio==1.59.0 protobuf==4.24.4
+
+	# Install ctransformers from JLLLLLL's cuBLAS wheels will append cu117to version of ctransformer, this will cause creating from file failed.
+	@echo "Installing ctransformers..."
+	@pip install ctransformers==0.2.27 --prefer-binary --extra-index-url=https://jllllll.github.io/ctransformers-cuBLAS-wheels/AVX2/cu117
+
+	@echo "Deactivating virtual environment..."
+	@. deactivate

extra/grpc/c_transformers/README.md

@@ -0,0 +1,5 @@
+# Creating a separate environment for ctransformers project
+
+```
+make ctransformers
+```

extra/grpc/c_transformers/c_transformers.py

@@ -0,0 +1,108 @@
+"""
+This is the extra gRPC server of LocalAI
+"""
+
+from __future__ import annotations
+from typing import List
+from concurrent import futures
+import time
+import argparse
+import signal
+import sys
+import os
+
+import grpc
+import backend_pb2
+import backend_pb2_grpc
+
+from ctransformers import AutoModelForCausalLM, AutoConfig, Config
+
+# Adapted from https://github.com/marella/ctransformers/tree/main#supported-models
+# License: MIT
+# Adapted by AIsuko
+class ModelType:
+    GPT = "gpt2"
+    GPT_J_GPT4_ALL_J= "gptj"
+    GPT_NEOX_STABLE_LM = "gpt_neox"
+    FALCON= "falcon"
+    LLaMA_LLaMA2 = "llama"
+    MPT="mpt"
+    STAR_CODER_CHAT="gpt_bigcode"
+    DOLLY_V2="dolly-v2"
+    REPLIT="replit"
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+# If MAX_WORKERS are specified in the environment use it, otherwise default to 1
+MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1'))
+
+
+class BackendServicer(backend_pb2_grpc.BackendServicer):
+    """
+    BackendServicer is the class that implements the gRPC service
+    """
+    def Health(self, request, context):
+        return backend_pb2.Reply(message=bytes("OK", 'utf-8'))
+
+    def LoadModel(self, request, context):
+        try:
+            model_path = request.Model
+            if not os.path.exists(model_path):
+                return backend_pb2.Result(success=False, message=f"Model path {model_path} does not exist")
+            model_type = request.ModelType
+            if model_type not in ModelType.__dict__.values():
+                return backend_pb2.Result(success=False, message=f"Model type {model_type} not supported")            
+            llm = AutoModelForCausalLM.from_pretrained(model_file=model_path, model_type=model_type)
+            self.model=llm
+        except Exception as err:
+            return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
+        return backend_pb2.Result(message="Model loaded successfully", success=True)
+
+    def Predict(self, request, context):
+        try:
+            generated_text=self.model(request.prompt)
+        except Exception as err:
+            return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
+        return backend_pb2.Result(message=bytes(generated_text), encoding="utf-8")
+
+    def PredictStream(self, request, context):
+        return super().PredictStream(request, context)
+
+    def TokenizeString(self, request, context):
+        try:
+            tokens: List[int]=self.model.tokenize(request.prompt, add_bos_token=False)
+            l=len(tokens)
+        except Exception as err:
+            return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
+        return backend_pb2.TokenizationResponse(length=l, tokens=tokens)
+
+def serve(address):
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=MAX_WORKERS))
+    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
+    server.add_insecure_port(address)
+    server.start()
+    print("Server started. Listening on: " + address, file=sys.stderr)
+
+    # Define the signal handler function
+    def signal_handler(sig, frame):
+        print("Received termination signal. Shutting down...")
+        server.stop(0)
+        sys.exit(0)
+
+    # Set the signal handlers for SIGINT and SIGTERM
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+
+    try:
+        while True:
+            time.sleep(_ONE_DAY_IN_SECONDS)
+    except KeyboardInterrupt:
+        server.stop(0)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run the gRPC server.")
+    parser.add_argument(
+        "--addr", default="localhost:50051", help="The address to bind the server to."
+    )
+    args = parser.parse_args()
+
+    serve(args.addr)

extra/grpc/c_transformers/run.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+##
+## A bash script wrapper that runs the ctransformers server with conda
+
+export PATH=$PATH:/opt/conda/bin
+
+# Activate conda environment
+source activate ctransformers
+
+# get the directory where the bash script is located
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+python $DIR/c_transformers.py $@

extra/grpc/huggingface/backend_pb2_grpc.py

@@ -32,6 +32,7 @@ def __init__(self, channel):
         self.PredictStream = channel.unary_stream(
                 '/backend.Backend/PredictStream',
                 request_serializer=backend__pb2.PredictOptions.SerializeToString,
+
                 response_deserializer=backend__pb2.Reply.FromString,
                 )
         self.Embedding = channel.unary_unary(