From 74827ced1857897a716d999fafac7e030a248998 Mon Sep 17 00:00:00 2001 From: UranusSeven <109661872+UranusSeven@users.noreply.github.com> Date: Mon, 19 Jun 2023 12:02:11 +0800 Subject: [PATCH] FEAT: prototype (#3) * Fix pre-commit * FEAT: prototype * Remove xorbits related items from .gitignore --- .gitignore | 144 +++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 2 +- plexar/__init__.py | 27 +++++- plexar/actor/__init__.py | 15 ++++ plexar/actor/api.py | 15 ++++ plexar/actor/gradio.py | 15 ++++ plexar/actor/model.py | 29 +++++++ plexar/common.py | 20 +++++ plexar/deploy/__init__.py | 13 +++ plexar/deploy/cmdline.py | 71 ++++++++++++++++ plexar/model/__init__.py | 82 ++++++++++++++++++ plexar/model/llm/__init__.py | 29 +++++++ plexar/model/llm/core.py | 159 +++++++++++++++++++++++++++++++++++ plexar/model/llm/vicuna.py | 42 +++++++++ setup.cfg | 3 + 15 files changed, 664 insertions(+), 2 deletions(-) create mode 100644 .gitignore create mode 100644 plexar/actor/__init__.py create mode 100644 plexar/actor/api.py create mode 100644 plexar/actor/gradio.py create mode 100644 plexar/actor/model.py create mode 100644 plexar/common.py create mode 100644 plexar/deploy/__init__.py create mode 100644 plexar/deploy/cmdline.py create mode 100644 plexar/model/__init__.py create mode 100644 plexar/model/llm/__init__.py create mode 100644 plexar/model/llm/core.py create mode 100644 plexar/model/llm/vicuna.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..5940628faa --- /dev/null +++ b/.gitignore @@ -0,0 +1,144 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +generated/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# IDEs +.idea +.vscode +*.iml + +# web staff +node_modules/ +static/ + +# doc +doc/source/savefig/ + +asv/results + +.DS_Store \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2bf51cc460..4f2c8de412 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -files: python/plexar +files: plexar repos: - repo: https://github.com/psf/black rev: 23.1.0 diff --git a/plexar/__init__.py b/plexar/__init__.py index ecd3379a87..27261e73f0 100644 --- a/plexar/__init__.py +++ b/plexar/__init__.py @@ -1,3 +1,28 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from . import _version -__version__ = _version.get_versions()['version'] + +__version__ = _version.get_versions()["version"] + + +def install(): + from .model import install as install_model + + install_model() + + +install() +del install diff --git a/plexar/actor/__init__.py b/plexar/actor/__init__.py new file mode 100644 index 0000000000..d3f48dd6fc --- /dev/null +++ b/plexar/actor/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .model import ModelActor diff --git a/plexar/actor/api.py b/plexar/actor/api.py new file mode 100644 index 0000000000..1618e1d94f --- /dev/null +++ b/plexar/actor/api.py @@ -0,0 +1,15 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: add an API actor running along with the model actor. diff --git a/plexar/actor/gradio.py b/plexar/actor/gradio.py new file mode 100644 index 0000000000..2f3e74d10d --- /dev/null +++ b/plexar/actor/gradio.py @@ -0,0 +1,15 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: add a gradio actor running on the controller. diff --git a/plexar/actor/model.py b/plexar/actor/model.py new file mode 100644 index 0000000000..aaff6279e7 --- /dev/null +++ b/plexar/actor/model.py @@ -0,0 +1,29 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import xoscar as xo + +from ..model.llm.core import Model + + +class ModelActor(xo.Actor): + def __init__(self, model: Model): + super().__init__() + self._model = model + + async def __post_create__(self): + self._model.load() + + def __getattr__(self, item): + return getattr(self._model, item) diff --git a/plexar/common.py b/plexar/common.py new file mode 100644 index 0000000000..71e6442b37 --- /dev/null +++ b/plexar/common.py @@ -0,0 +1,20 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from pathlib import Path + +PLEXAR_HOME = str(Path.home() / ".plexar") +PLEXAR_CACHE_DIR = os.path.join(PLEXAR_HOME, "cache") +PLEXAR_LOG_DIR = os.path.join(PLEXAR_HOME, "logs") diff --git a/plexar/deploy/__init__.py b/plexar/deploy/__init__.py new file mode 100644 index 0000000000..37f6558d95 --- /dev/null +++ b/plexar/deploy/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/plexar/deploy/cmdline.py b/plexar/deploy/cmdline.py new file mode 100644 index 0000000000..f558b4c953 --- /dev/null +++ b/plexar/deploy/cmdline.py @@ -0,0 +1,71 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from plexar.actor import ModelActor + +from .. import __version__ + + +@click.group(name="plexar") +@click.version_option(__version__, "--version", "-v") +def cli(): + pass + + +@cli.group() +def model(): + pass + + +@model.command("list") +def model_list(): + raise NotImplemented + + +@model.command("launch") +@click.option("--path", "-p") +def model_launch(path): + import asyncio + import textwrap + + import xoscar as xo + + from plexar.model.llm.vicuna import VicunaUncensoredGgml + + async def _run(): + await xo.create_actor_pool(address="localhost:9999", n_process=1) + + vu = VicunaUncensoredGgml(model_path=path, llamacpp_model_config={}) + vu_ref = await xo.create_actor( + ModelActor, address="localhost:9999", uid="vu", model=vu + ) + + while True: + i = input("User:\n") + if i == "exit": + break + + completion = await vu_ref.chat(i) + text = "\n".join(textwrap.wrap(completion["text"], width=80)) + print(f"Assistant:\n{text}") + + loop = asyncio.get_event_loop() + loop.run_until_complete(_run()) + loop.close() + + +if __name__ == "__main__": + cli() diff --git a/plexar/model/__init__.py b/plexar/model/__init__.py new file mode 100644 index 0000000000..306593d428 --- /dev/null +++ b/plexar/model/__init__.py @@ -0,0 +1,82 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import urllib.request +from typing import List, Optional, Type + +from tqdm import tqdm + +from ..common import PLEXAR_CACHE_DIR + + +class ModelSpec: + name: str + n_parameters_in_billions: Optional[int] = None + format: Optional[str] = None + quantization: Optional[str] = None + url: Optional[str] = None + cls: Optional[Type] = None + + def __init__( + self, + name: str, + n_parameters_in_billions: Optional[int], + fmt: Optional[str] = None, + quantization: Optional[str] = None, + url: Optional[str] = None, + cls: Optional[Type] = None, + ): + self.name = name + self.n_parameters_in_billions = n_parameters_in_billions + self.format = fmt + self.quantization = quantization + self.url = url + self.cls = cls + + def __str__(self): + return f"{self.name}-{self.n_parameters_in_billions}b-{self.format}-{self.quantization}" + + def cache(self): + assert self.url is not None + + save_path = os.path.join(PLEXAR_CACHE_DIR, str(self), "model.bin") + + if os.path.exists(save_path): + os.remove(save_path) + + with tqdm( + unit="B", + unit_scale=True, + unit_divisor=1024, + miniters=1, + desc="Downloading", + ncols=80, + ) as progress: + urllib.request.urlretrieve( + self.url, + save_path, + reporthook=lambda blocknum, blocksize, totalsize: progress.update( + blocksize + ), + ) + + +MODEL_SPECS: List[ModelSpec] = [] + + +def install(): + from .llm import install as llm_install + + llm_install() diff --git a/plexar/model/llm/__init__.py b/plexar/model/llm/__init__.py new file mode 100644 index 0000000000..d096e592c8 --- /dev/null +++ b/plexar/model/llm/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def install(): + from .. import MODEL_SPECS, ModelSpec + from .vicuna import VicunaUncensoredGgml + + MODEL_SPECS.append( + ModelSpec( + name="vicuna-uncensored", + n_parameters_in_billions=7, + fmt="ggml", + quantization="q4_0", + url="https://huggingface.co/vicuna/ggml-vicuna-7b-1.1/blob/main/ggml-vic7b-uncensored-q4_0.bin", + cls=VicunaUncensoredGgml, + ), + ) diff --git a/plexar/model/llm/core.py b/plexar/model/llm/core.py new file mode 100644 index 0000000000..4bc0daed4a --- /dev/null +++ b/plexar/model/llm/core.py @@ -0,0 +1,159 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import contextlib +import logging +import textwrap +from abc import abstractmethod +from time import time +from typing import List, Optional, TypedDict + +logger = logging.getLogger(__name__) + + +class Completion(TypedDict): + text: str + prompt_tokens: int + completion_tokens: int + finish_reason: Optional[str] + elapsed_time: Optional[int] + + +class LlamaCppGenerateConfig(TypedDict): + # TODO + pass + + +class LlamaCppModelConfig(TypedDict): + # TODO + pass + + +class ChatHistory: + _inputs: List[str] + _outputs: List[str] + + def __init__(self): + self._inputs = [] + self._outputs = [] + + def to_prompt( + self, + system_prompt: str, + sep: str, + user_name: str, + assistant_name: str, + input: str, + ) -> str: + ret = system_prompt + for i, o in zip(self._inputs, self._outputs): + ret += f"{sep} {user_name}: {i}" + ret += f"{sep} {assistant_name}: {o}" + ret += f"{sep} {user_name}: {input}" + ret += f"{sep} {assistant_name}:" + return ret + + def append(self, i: str, o: str): + self._inputs.append(i) + self._outputs.append(o) + + +class Model(abc.ABC): + @abstractmethod + def load(self): + pass + + +class LlamaCppModel(Model): + def __init__( + self, + model_path: str, + llamacpp_model_config: Optional[LlamaCppModelConfig] = None, + ): + super().__init__() + self._model_path = model_path + self._llamacpp_model_config = llamacpp_model_config or {} + self._llm = None + + def load(self): + from llama_cpp import Llama + + with contextlib.redirect_stdout(None): + self._llm = Llama( + model_path=self._model_path, + verbose=False, + **self._llamacpp_model_config, + ) + + def generate( + self, prompt: str, generate_config: Optional[LlamaCppGenerateConfig] = None + ) -> Completion: + logger.debug("prompt:\n%s", "\n".join(textwrap.wrap(prompt, width=80))) + generate_config = generate_config or {} + + start = time() + with contextlib.redirect_stdout(None): + assert self._llm is not None + completion = self._llm(prompt=prompt, **generate_config) + elapsed = time() - start + + completion = Completion( + text=self._format_completion(completion["choices"][0]["text"]), + prompt_tokens=completion["usage"]["prompt_tokens"], + completion_tokens=completion["usage"]["completion_tokens"], + finish_reason=completion["choices"][0]["finish_reason"], + elapsed_time=int(elapsed), + ) + + logger.debug("completion:\n%s", completion) + return completion + + @staticmethod + def _format_completion(text: str): + return text.strip() + + +class LlamaCppChatModel(LlamaCppModel): + _history = ChatHistory() + + def __init__( + self, + model_path: str, + system_prompt: str, + sep: str, + user_name: str, + assistant_name: str, + llamacpp_model_config: Optional[LlamaCppModelConfig] = None, + ): + super().__init__(model_path, llamacpp_model_config) + self._system_prompt: str = system_prompt + self._sep: str = sep + self._user_name: str = user_name + self._assistant_name: str = assistant_name + + def chat( + self, prompt: str, generate_config: Optional[LlamaCppGenerateConfig] = None + ) -> Completion: + full_prompt = self._history.to_prompt( + self._system_prompt, + self._sep, + self._user_name, + self._assistant_name, + prompt, + ) + + completion = self.generate(full_prompt, generate_config) + self._history.append(prompt, completion["text"]) + return completion diff --git a/plexar/model/llm/vicuna.py b/plexar/model/llm/vicuna.py new file mode 100644 index 0000000000..c399afc1d5 --- /dev/null +++ b/plexar/model/llm/vicuna.py @@ -0,0 +1,42 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +from .core import LlamaCppChatModel, LlamaCppModelConfig + + +class VicunaUncensoredGgml(LlamaCppChatModel): + _system_prompt = ( + "A chat between a curious human and an artificial intelligence assistant." + " The assistant gives helpful, detailed, and polite answers to the human's" + " questions. " + ) + _sep = "###" + _user_name = "User" + _assistant_name = "Assistant" + + def __init__( + self, + model_path: str, + llamacpp_model_config: Optional[LlamaCppModelConfig] = None, + ): + super().__init__( + model_path, + system_prompt=self._system_prompt, + sep=self._sep, + user_name=self._user_name, + assistant_name=self._assistant_name, + llamacpp_model_config=llamacpp_model_config, + ) diff --git a/setup.cfg b/setup.cfg index 2ecc50ce85..0d3193de7c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,6 +25,9 @@ include_package_data = True packages = find: install_requires = xoscar + gradio + click + tqdm [options.packages.find] exclude =