diff --git a/plexar/client.py b/plexar/client.py index a788984472..ea1c3c1b4e 100644 --- a/plexar/client.py +++ b/plexar/client.py @@ -14,14 +14,22 @@ import asyncio import uuid -from typing import List, Optional, Tuple +from typing import Iterator, List, Optional, Tuple, Union +import requests import xoscar as xo from .core.model import ModelActor from .core.service import SupervisorActor from .isolation import Isolation from .model import ModelSpec +from .model.llm.types import ( + ChatCompletion, + ChatCompletionChunk, + ChatCompletionMessage, + Completion, + CompletionChunk, +) class Client: @@ -44,7 +52,7 @@ def launch_model( model_size_in_billions: Optional[int] = None, model_format: Optional[str] = None, quantization: Optional[str] = None, - **kwargs + **kwargs, ) -> str: model_uid = self.gen_model_uid() @@ -54,7 +62,7 @@ def launch_model( model_size_in_billions=model_size_in_billions, model_format=model_format, quantization=quantization, - **kwargs + **kwargs, ) self._isolation.call(coro) @@ -73,6 +81,92 @@ def get_model(self, model_uid: str) -> xo.ActorRefType["ModelActor"]: return self._isolation.call(coro) +class RESTfulClient: + def __init__(self, base_url): + self.base_url = base_url + + @classmethod + def gen_model_uid(cls) -> str: + # generate a time-based uuid. + return str(uuid.uuid1()) + + def list_models(self) -> List[str]: + url = f"{self.base_url}/v1/models" + + response = requests.get(url) + response_data = response.json() + return response_data + + def launch_model( + self, + model_name: str, + model_size_in_billions: Optional[int] = None, + model_format: Optional[str] = None, + quantization: Optional[str] = None, + **kwargs, + ) -> str: + url = f"{self.base_url}/v1/models" + + model_uid = self.gen_model_uid() + payload = { + "model_uid": model_uid, + "model_name": model_name, + "model_size_in_billions": model_size_in_billions, + "model_format": model_format, + "quantization": quantization, + "kwargs": kwargs, + } + response = requests.post(url, json=payload) + response_data = response.json() + model_uid = response_data["model_uid"] + return model_uid + + def terminate_model(self, model_uid: str): + url = f"{self.base_url}/v1/models/{model_uid}" + + response = requests.delete(url) + if response.status_code != 200: + raise Exception(f"Error terminating the model.") + + def generate( + self, model_uid: str, prompt: str, **kwargs + ) -> Union[Completion, Iterator[CompletionChunk]]: + url = f"{self.base_url}/v1/completions" + + request_body = {"model": model_uid, "prompt": prompt, **kwargs} + response = requests.post(url, json=request_body) + response_data = response.json() + return response_data + + def chat( + self, + model_uid: str, + prompt: str, + system_prompt: Optional[str] = None, + chat_history: Optional[List[ChatCompletionMessage]] = None, + **kwargs, + ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]: + url = f"{self.base_url}/v1/chat/completions" + + if chat_history is None: + chat_history = [] + + if chat_history and chat_history[0]["role"] == "system": + if system_prompt is not None: + chat_history[0]["content"] = system_prompt + else: + if system_prompt is not None: + chat_history.insert( + 0, ChatCompletionMessage(role="system", content=system_prompt) + ) + + chat_history.append(ChatCompletionMessage(role="user", content=prompt)) + request_body = {"model": model_uid, "messages": chat_history, **kwargs} + response = requests.post(url, json=request_body) + response_data = response.json() + return response_data + + class AsyncClient: def __init__(self, supervisor_address: str): self._supervisor_address = supervisor_address @@ -96,7 +190,7 @@ async def launch_model( model_size_in_billions: Optional[int] = None, model_format: Optional[str] = None, quantization: Optional[str] = None, - **kwargs + **kwargs, ) -> str: model_uid = self.gen_model_uid() @@ -107,7 +201,7 @@ async def launch_model( model_size_in_billions=model_size_in_billions, model_format=model_format, quantization=quantization, - **kwargs + **kwargs, ) return model_uid diff --git a/plexar/core/restful_api.py b/plexar/core/restful_api.py new file mode 100644 index 0000000000..a09df11b7b --- /dev/null +++ b/plexar/core/restful_api.py @@ -0,0 +1,345 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from typing import Dict, List, Literal, Optional, Union + +import xoscar as xo +from fastapi import APIRouter, FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse +from pydantic import BaseModel, Field +from typing_extensions import NotRequired, TypedDict +from uvicorn import Config, Server + +from plexar.core.service import SupervisorActor +from plexar.model.llm.types import ChatCompletion, Completion + +max_tokens_field = Field( + default=16, ge=1, le=2048, description="The maximum number of tokens to generate." +) + +temperature_field = Field( + default=0.8, + ge=0.0, + le=2.0, + description="Adjust the randomness of the generated text.\n\n" + + "Temperature is a hyperparameter that controls the randomness of the generated text. It affects the probability distribution of the model's output tokens. A higher temperature (e.g., 1.5) makes the output more random and creative, while a lower temperature (e.g., 0.5) makes the output more focused, deterministic, and conservative. The default value is 0.8, which provides a balance between randomness and determinism. At the extreme, a temperature of 0 will always pick the most likely next token, leading to identical outputs in each run.", +) + +top_p_field = Field( + default=0.95, + ge=0.0, + le=1.0, + description="Limit the next token selection to a subset of tokens with a cumulative probability above a threshold P.\n\n" + + "Top-p sampling, also known as nucleus sampling, is another text generation method that selects the next token from a subset of tokens that together have a cumulative probability of at least p. This method provides a balance between diversity and quality by considering both the probabilities of tokens and the number of tokens to sample from. A higher value for top_p (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text.", +) + +stop_field = Field( + default=None, + description="A list of tokens at which to stop generation. If None, no stop tokens are used.", +) + +stream_field = Field( + default=False, + description="Whether to stream the results as they are generated. Useful for chatbots.", +) + +top_k_field = Field( + default=40, + ge=0, + description="Limit the next token selection to the K most probable tokens.\n\n" + + "Top-k sampling is a text generation method that selects the next token only from the top k most likely tokens predicted by the model. It helps reduce the risk of generating low-probability or nonsensical tokens, but it may also limit the diversity of the output. A higher value for top_k (e.g., 100) will consider more tokens and lead to more diverse text, while a lower value (e.g., 10) will focus on the most probable tokens and generate more conservative text.", +) + +repeat_penalty_field = Field( + default=1.1, + ge=0.0, + description="A penalty applied to each token that is already generated. This helps prevent the model from repeating itself.\n\n" + + "Repeat penalty is a hyperparameter used to penalize the repetition of token sequences during text generation. It helps prevent the model from generating repetitive or monotonous text. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient.", +) + +presence_penalty_field = Field( + default=0.0, + ge=-2.0, + le=2.0, + description="Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.", +) + +frequency_penalty_field = Field( + default=0.0, + ge=-2.0, + le=2.0, + description="Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.", +) + +mirostat_mode_field = Field( + default=0, + ge=0, + le=2, + description="Enable Mirostat constant-perplexity algorithm of the specified version (1 or 2; 0 = disabled)", +) + +mirostat_tau_field = Field( + default=5.0, + ge=0.0, + le=10.0, + description="Mirostat target entropy, i.e. the target perplexity - lower values produce focused and coherent text, larger values produce more diverse and less coherent text", +) + +mirostat_eta_field = Field( + default=0.1, ge=0.001, le=1.0, description="Mirostat learning rate" +) + + +class CreateCompletionRequest(BaseModel): + prompt: str + suffix: Optional[str] = Field(None) + max_tokens: int = max_tokens_field + temperature: float = temperature_field + top_p: float = top_p_field + mirostat_mode: int = mirostat_mode_field + mirostat_tau: float = mirostat_tau_field + mirostat_eta: float = mirostat_eta_field + echo: bool = Field( + default=False, + description="Whether to echo the prompt in the generated text. Useful for chatbots.", + ) + stop: Optional[Union[str, List[str]]] = stop_field + stream: bool = stream_field + logprobs: Optional[int] = Field( + default=None, + ge=0, + description="The number of logprobs to generate. If None, no logprobs are generated.", + ) + presence_penalty: Optional[float] = presence_penalty_field + frequency_penalty: Optional[float] = frequency_penalty_field + logit_bias: Optional[Dict[str, float]] = Field(None) + + model: str + n: Optional[int] = 1 + best_of: Optional[int] = 1 + user: Optional[str] = Field(None) + + # llama.cpp specific parameters + top_k: int = top_k_field + repeat_penalty: float = repeat_penalty_field + logit_bias_type: Optional[Literal["input_ids", "tokens"]] = Field(None) + + class Config: + schema_extra = { + "example": { + "prompt": "\n\n### Instructions:\nWhat is the capital of France?\n\n### Response:\n", + "stop": ["\n", "###"], + } + } + + +# TODO: create embedding request and response +class CreateEmbeddingRequest(BaseModel): + model: str + input: Union[str, List[str]] = Field(description="The input to embed.") + user: Optional[str] + + class Config: + schema_extra = { + "example": { + "input": "The food was delicious and the waiter...", + } + } + + +class ChatCompletionRequestMessage(TypedDict): + role: Literal["assistant", "user", "system"] + content: str + user: NotRequired[str] + + +class CreateChatCompletionRequest(BaseModel): + messages: List[ChatCompletionRequestMessage] = Field( + default=[], description="A list of messages to generate completions for." + ) + max_tokens: int = max_tokens_field + temperature: float = temperature_field + top_p: float = top_p_field + mirostat_mode: int = mirostat_mode_field + mirostat_tau: float = mirostat_tau_field + mirostat_eta: float = mirostat_eta_field + stop: Optional[List[str]] = stop_field + stream: bool = stream_field + presence_penalty: Optional[float] = presence_penalty_field + frequency_penalty: Optional[float] = frequency_penalty_field + logit_bias: Optional[Dict[str, float]] = Field(None) + + model: str + n: Optional[int] = 1 + user: Optional[str] = Field(None) + + # llama.cpp specific parameters + top_k: int = top_k_field + repeat_penalty: float = repeat_penalty_field + logit_bias_type: Optional[Literal["input_ids", "tokens"]] = Field(None) + + class Config: + schema_extra = { + "example": { + "messages": [ + {"role": "system", "content": "you are a helpful AI assistant"}, + {"role": "user", "content": "Hello!"}, + {"role": "assistant", "content": "Hi what can I help you?"}, + ] + } + } + + +class RESTfulAPIActor(xo.Actor): + def __init__(self, host: str, port: int): + super().__init__() + self._supervisor_ref: xo.ActorRefType["SupervisorActor"] + app = FastAPI() + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + self.router = APIRouter() + self.router.add_api_route("/v1/models", self.list_models, methods=["GET"]) + self.router.add_api_route("/v1/models", self.launch_model, methods=["POST"]) + self.router.add_api_route( + "/v1/models/{model_uid}", self.terminate_model, methods=["DELETE"] + ) + self.router.add_api_route( + "/v1/completions", + self.create_completion, + methods=["POST"], + response_model=Completion, + ) + self.router.add_api_route( + "/v1/embeddings", self.create_embedding, methods=["POST"] + ) + self.router.add_api_route( + "/v1/chat/completions", + self.create_chat_completion, + methods=["POST"], + response_model=ChatCompletion, + ) + app.include_router(self.router) + + # uvicorn + loop = asyncio.get_event_loop() + config = Config(app=app, loop=loop, host=host, port=port) + server = Server(config) + loop.create_task(server.serve()) + + @classmethod + def uid(cls) -> str: + return "plexar_RESTfulAPI" + + async def __post_create__(self): + self._supervisor_ref = await xo.actor_ref( + address=self.address, uid=SupervisorActor.uid() + ) + + async def list_models(self) -> List[str]: + models = await self._supervisor_ref.list_models() + return [model_uid for model_uid, _ in models] + + async def launch_model(self, request: Request) -> str: + payload = await request.json() + model_uid = payload.get("model_uid") + model_name = payload.get("model_name") + model_size_in_billions = payload.get("model_size_in_billions") + model_format = payload.get("model_format") + quantization = payload.get("quantization") + kwargs = payload.get("kwargs", {}) or {} + + await self._supervisor_ref.launch_builtin_model( + model_uid=model_uid, + model_name=model_name, + model_size_in_billions=model_size_in_billions, + model_format=model_format, + quantization=quantization, + **kwargs, + ) + return JSONResponse(content={"model_uid": model_uid}) + + async def terminate_model(self, model_uid: str): + await self._supervisor_ref.terminate_model(model_uid) + + async def create_completion(self, request: Request, body: CreateCompletionRequest): + exclude = { + "prompt", + "model", + "n", + "best_of", + "logit_bias", + "logit_bias_type", + "user", + } + kwargs = body.dict(exclude=exclude) + + if body.logit_bias is not None: + raise NotImplementedError + model_uid = body.model + model = await self._supervisor_ref.get_model(model_uid) + + if body.stream: + raise NotImplementedError + else: + return await model.generate(body.prompt, kwargs) + + async def create_embedding(self, request: CreateEmbeddingRequest): + raise NotImplementedError + + async def create_chat_completion( + self, + request: Request, + body: CreateChatCompletionRequest, + ): + exclude = { + "n", + "model", + "messages", + "logit_bias", + "logit_bias_type", + "user", + } + kwargs = body.dict(exclude=exclude) + + if body.logit_bias is not None: + raise NotImplementedError + + user_messages = [ + msg["content"] for msg in body.messages if msg["role"] == "user" + ] + if user_messages: + prompt = user_messages[-1] + else: + raise Exception("no prompt given") + system_prompt = next( + (msg["content"] for msg in body.messages if msg["role"] == "system"), None + ) + + chat_history = body.messages + + model_uid = body.model + model = await self._supervisor_ref.get_model(model_uid) + + if body.stream: + raise NotImplementedError + + else: + return await model.chat(prompt, system_prompt, chat_history, kwargs) diff --git a/plexar/deploy/supervisor.py b/plexar/deploy/supervisor.py index 4108f43791..d47d5ab46a 100644 --- a/plexar/deploy/supervisor.py +++ b/plexar/deploy/supervisor.py @@ -17,12 +17,20 @@ import xoscar as xo from ..core.gradio import GradioActor +from ..core.restful_api import RESTfulAPIActor from ..core.service import SupervisorActor async def start_supervisor_components(address: str, share: bool, host: str, port: int): await xo.create_actor(SupervisorActor, address=address, uid=SupervisorActor.uid()) - # TODO: start RESTful actor + await xo.create_actor( + RESTfulAPIActor, + address=address, + uid=RESTfulAPIActor.uid(), + host="0.0.0.0", + port=8000, + ) + gradio = await xo.create_actor( GradioActor, xoscar_endpoint=address, diff --git a/versioneer.py b/versioneer.py index e994d060ec..6e4d8bfdb4 100644 --- a/versioneer.py +++ b/versioneer.py @@ -1,4 +1,3 @@ - # Version: 0.28 """The Versioneer - like a rocketeer, but for versions. @@ -348,11 +347,13 @@ def get_root(): setup_py = os.path.join(root, "setup.py") versioneer_py = os.path.join(root, "versioneer.py") if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ("Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND').") + err = ( + "Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND')." + ) raise VersioneerBadRootError(err) try: # Certain runtime workflows (setup.py install/develop in a setuptools @@ -365,8 +366,10 @@ def get_root(): me_dir = os.path.normcase(os.path.splitext(my_path)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) if me_dir != vsr_dir and "VERSIONEER_PEP518" not in globals(): - print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(my_path), versioneer_py)) + print( + "Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(my_path), versioneer_py) + ) except NameError: pass return root @@ -384,9 +387,9 @@ def get_config_from_root(root): section = None if pyproject_toml.exists() and have_tomllib: try: - with open(pyproject_toml, 'rb') as fobj: + with open(pyproject_toml, "rb") as fobj: pp = tomllib.load(fobj) - section = pp['tool']['versioneer'] + section = pp["tool"]["versioneer"] except (tomllib.TOMLDecodeError, KeyError): pass if not section: @@ -398,7 +401,7 @@ def get_config_from_root(root): section = parser["versioneer"] cfg = VersioneerConfig() - cfg.VCS = section['VCS'] + cfg.VCS = section["VCS"] cfg.style = section.get("style", "") cfg.versionfile_source = section.get("versionfile_source") cfg.versionfile_build = section.get("versionfile_build") @@ -421,15 +424,16 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs, method): # decorator """Create decorator to mark a method as the handler of a VCS.""" + def decorate(f): """Store f in HANDLERS[vcs][method].""" HANDLERS.setdefault(vcs, {})[method] = f return f + return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): """Call the given command(s).""" assert isinstance(commands, list) process = None @@ -445,10 +449,14 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, try: dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen([command] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None), **popen_kwargs) + process = subprocess.Popen( + [command] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + **popen_kwargs, + ) break except OSError: e = sys.exc_info()[1] @@ -471,7 +479,9 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, return stdout, process.returncode -LONG_VERSION_PY['git'] = r''' +LONG_VERSION_PY[ + "git" +] = r''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -1187,7 +1197,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} + tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -1196,7 +1206,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r'\d', r)} + tags = {r for r in refs if re.search(r"\d", r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -1204,24 +1214,31 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] # Filter out refs that exactly match prefix or that don't start # with a number once the prefix is stripped (mostly a concern # when prefix is '') - if not re.match(r'\d', r): + if not re.match(r"\d", r): continue if verbose: print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } @register_vcs_handler("git", "pieces_from_vcs") @@ -1243,8 +1260,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): env.pop("GIT_DIR", None) runner = functools.partial(runner, env=env) - _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=not verbose) + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -1252,10 +1268,19 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner(GITS, [ - "describe", "--tags", "--dirty", "--always", "--long", - "--match", f"{tag_prefix}[[:digit:]]*" - ], cwd=root) + describe_out, rc = runner( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + f"{tag_prefix}[[:digit:]]*", + ], + cwd=root, + ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -1270,8 +1295,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None - branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=root) + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) # --abbrev-ref was added in git-1.6.3 if rc != 0 or branch_name is None: raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") @@ -1311,17 +1335,16 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] + git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparsable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) + pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out return pieces # tag @@ -1330,10 +1353,12 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + full_tag, + tag_prefix, + ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -1407,15 +1432,21 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -1444,11 +1475,13 @@ def versions_from_file(filename): contents = f.read() except OSError: raise NotThisMethod("unable to read _version.py") - mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S + ) if not mo: - mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\r\n(.*)''' # END VERSION_JSON", contents, re.M | re.S + ) if not mo: raise NotThisMethod("no version_json in _version.py") return json.loads(mo.group(1)) @@ -1457,8 +1490,7 @@ def versions_from_file(filename): def write_to_version_file(filename, versions): """Write the given version number to the given _version.py file.""" os.unlink(filename) - contents = json.dumps(versions, sort_keys=True, - indent=1, separators=(",", ": ")) + contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) with open(filename, "w") as f: f.write(SHORT_VERSION_PY % contents) @@ -1490,8 +1522,7 @@ def render_pep440(pieces): rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -1520,8 +1551,7 @@ def render_pep440_branch(pieces): rendered = "0" if pieces["branch"] != "master": rendered += ".dev0" - rendered += "+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -1682,11 +1712,13 @@ def render_git_describe_long(pieces): def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } if not style or style == "default": style = "pep440" # the default @@ -1710,9 +1742,13 @@ def render(pieces, style): else: raise ValueError("unknown style '%s'" % style) - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } class VersioneerBadRootError(Exception): @@ -1735,8 +1771,9 @@ def get_versions(verbose=False): handlers = HANDLERS.get(cfg.VCS) assert handlers, "unrecognized VCS '%s'" % cfg.VCS verbose = verbose or cfg.verbose - assert cfg.versionfile_source is not None, \ - "please set versioneer.versionfile_source" + assert ( + cfg.versionfile_source is not None + ), "please set versioneer.versionfile_source" assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" versionfile_abs = os.path.join(root, cfg.versionfile_source) @@ -1790,9 +1827,13 @@ def get_versions(verbose=False): if verbose: print("unable to compute version") - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, "error": "unable to compute version", - "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } def get_version(): @@ -1845,6 +1886,7 @@ def run(self): print(" date: %s" % vers.get("date")) if vers["error"]: print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version # we override "build_py" in setuptools @@ -1866,8 +1908,8 @@ def run(self): # but the build_py command is not expected to copy any files. # we override different "build_py" commands for both environments - if 'build_py' in cmds: - _build_py = cmds['build_py'] + if "build_py" in cmds: + _build_py = cmds["build_py"] else: from setuptools.command.build_py import build_py as _build_py @@ -1884,14 +1926,14 @@ def run(self): # now locate _version.py in the new build/ directory and replace # it with an updated value if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) + target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py - if 'build_ext' in cmds: - _build_ext = cmds['build_ext'] + if "build_ext" in cmds: + _build_ext = cmds["build_ext"] else: from setuptools.command.build_ext import build_ext as _build_ext @@ -1911,15 +1953,17 @@ def run(self): # it with an updated value if not cfg.versionfile_build: return - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) + target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) if not os.path.exists(target_versionfile): - print(f"Warning: {target_versionfile} does not exist, skipping " - "version update. This can happen if you are running build_ext " - "without first running build_py.") + print( + f"Warning: {target_versionfile} does not exist, skipping " + "version update. This can happen if you are running build_ext " + "without first running build_py." + ) return print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) + cmds["build_ext"] = cmd_build_ext if "cx_Freeze" in sys.modules: # cx_freeze enabled? @@ -1945,17 +1989,21 @@ def run(self): os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["build_exe"] = cmd_build_exe del cmds["build_py"] - if 'py2exe' in sys.modules: # py2exe enabled? + if "py2exe" in sys.modules: # py2exe enabled? try: from py2exe.setuptools_buildexe import py2exe as _py2exe except ImportError: @@ -1974,18 +2022,22 @@ def run(self): os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["py2exe"] = cmd_py2exe # sdist farms its file list building out to egg_info - if 'egg_info' in cmds: - _egg_info = cmds['egg_info'] + if "egg_info" in cmds: + _egg_info = cmds["egg_info"] else: from setuptools.command.egg_info import egg_info as _egg_info @@ -1998,7 +2050,7 @@ def find_sources(self): # Modify the filelist and normalize it root = get_root() cfg = get_config_from_root(root) - self.filelist.append('versioneer.py') + self.filelist.append("versioneer.py") if cfg.versionfile_source: # There are rare cases where versionfile_source might not be # included by default, so we must be explicit @@ -2011,18 +2063,21 @@ def find_sources(self): # We will instead replicate their final normalization (to unicode, # and POSIX-style paths) from setuptools import unicode_utils - normalized = [unicode_utils.filesys_decode(f).replace(os.sep, '/') - for f in self.filelist.files] - manifest_filename = os.path.join(self.egg_info, 'SOURCES.txt') - with open(manifest_filename, 'w') as fobj: - fobj.write('\n'.join(normalized)) + normalized = [ + unicode_utils.filesys_decode(f).replace(os.sep, "/") + for f in self.filelist.files + ] + + manifest_filename = os.path.join(self.egg_info, "SOURCES.txt") + with open(manifest_filename, "w") as fobj: + fobj.write("\n".join(normalized)) - cmds['egg_info'] = cmd_egg_info + cmds["egg_info"] = cmd_egg_info # we override different "sdist" commands for both environments - if 'sdist' in cmds: - _sdist = cmds['sdist'] + if "sdist" in cmds: + _sdist = cmds["sdist"] else: from setuptools.command.sdist import sdist as _sdist @@ -2044,8 +2099,10 @@ def make_release_tree(self, base_dir, files): # updated value target_versionfile = os.path.join(base_dir, cfg.versionfile_source) print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, - self._versioneer_generated_versions) + write_to_version_file( + target_versionfile, self._versioneer_generated_versions + ) + cmds["sdist"] = cmd_sdist return cmds @@ -2105,11 +2162,9 @@ def do_setup(): root = get_root() try: cfg = get_config_from_root(root) - except (OSError, configparser.NoSectionError, - configparser.NoOptionError) as e: + except (OSError, configparser.NoSectionError, configparser.NoOptionError) as e: if isinstance(e, (OSError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", - file=sys.stderr) + print("Adding sample versioneer config to setup.cfg", file=sys.stderr) with open(os.path.join(root, "setup.cfg"), "a") as f: f.write(SAMPLE_CONFIG) print(CONFIG_ERROR, file=sys.stderr) @@ -2118,15 +2173,18 @@ def do_setup(): print(" creating %s" % cfg.versionfile_source) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), - "__init__.py") + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") if os.path.exists(ipy): try: with open(ipy, "r") as f: @@ -2203,4 +2261,4 @@ def setup_command(): if __name__ == "__main__": cmd = sys.argv[1] if cmd == "setup": - setup_command() \ No newline at end of file + setup_command()