Skip to content

[AQUA] Added support for deploy stack model. #1223

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: feature/model_group
Choose a base branch
from
Open

[AQUA] Added support for deploy stack model. #1223

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2d4f16c
Added support for deploy stack model.
lu-ohai Jul 10, 2025
4095c8b
Updated pr.
lu-ohai Jul 16, 2025
41f543e
Added unit tests.
lu-ohai Jul 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 80 additions & 34 deletions ads/aqua/model/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,9 +147,9 @@ def create(
freeform_tags: Optional[Dict] = None,
defined_tags: Optional[Dict] = None,
**kwargs,
) -> DataScienceModel:
) -> Union[DataScienceModel, DataScienceModelGroup]:
"""
Creates a custom Aqua model from a service model.
Creates a custom Aqua model or model group from a service model.

Parameters
----------
Expand All @@ -167,28 +167,21 @@ def create(

Returns
-------
DataScienceModel
The instance of DataScienceModel.
Union[DataScienceModel, DataScienceModelGroup]
The instance of DataScienceModel or DataScienceModelGroup.
"""
fine_tune_weights = (
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

looks like there are several place where we do same validation. Wouldn't it be better to have something like:

if isinstance(model_id, AquaMultiModelRef):
     ....
else:
     ....

model_id.fine_tune_weights
if isinstance(model_id, AquaMultiModelRef)
else []
)
model_id = (
model_id.model_id if isinstance(model_id, AquaMultiModelRef) else model_id
)
service_model = DataScienceModel.from_id(model_id)
target_project = project_id or PROJECT_OCID
target_compartment = compartment_id or COMPARTMENT_OCID

# Skip model copying if it is registered model or fine-tuned model
if (
service_model.freeform_tags.get(Tags.BASE_MODEL_CUSTOM, None) is not None
or service_model.freeform_tags.get(Tags.AQUA_FINE_TUNED_MODEL_TAG)
is not None
):
logger.info(
f"Aqua Model {model_id} already exists in the user's compartment."
"Skipped copying."
)
return service_model

# combine tags
combined_freeform_tags = {
**(service_model.freeform_tags or {}),
Expand All @@ -199,23 +192,66 @@ def create(
**(defined_tags or {}),
}

custom_model = (
DataScienceModel()
.with_compartment_id(target_compartment)
.with_project_id(target_project)
.with_model_file_description(json_dict=service_model.model_file_description)
.with_display_name(service_model.display_name)
.with_description(service_model.description)
.with_freeform_tags(**combined_freeform_tags)
.with_defined_tags(**combined_defined_tags)
.with_custom_metadata_list(service_model.custom_metadata_list)
.with_defined_metadata_list(service_model.defined_metadata_list)
.with_provenance_metadata(service_model.provenance_metadata)
.create(model_by_reference=True, **kwargs)
)
logger.info(
f"Aqua Model {custom_model.id} created with the service model {model_id}."
)
custom_model = None
if fine_tune_weights:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wouldn’t it be better, instead of handling everything in one place, to split the implementation into two separate methods, one for creating the MC record and another for creating the group?

custom_model = (
DataScienceModelGroup()
.with_compartment_id(target_compartment)
.with_project_id(target_project)
.with_display_name(service_model.display_name)
.with_description(service_model.description)
.with_freeform_tags(**combined_freeform_tags)
.with_defined_tags(**combined_defined_tags)
.with_custom_metadata_list(service_model.custom_metadata_list)
.with_base_model_id(model_id)
.with_member_models(
[
{
"inference_key": fine_tune_weight.model_name,
"model_id": fine_tune_weight.model_id,
}
for fine_tune_weight in fine_tune_weights
]
)
.create()
)

logger.info(
f"Aqua Model Group {custom_model.id} created with the service model {model_id}."
)
else:
# Skip model copying if it is registered model or fine-tuned model
if (
service_model.freeform_tags.get(Tags.BASE_MODEL_CUSTOM, None)
is not None
or service_model.freeform_tags.get(Tags.AQUA_FINE_TUNED_MODEL_TAG)
is not None
):
logger.info(
f"Aqua Model {model_id} already exists in the user's compartment."
"Skipped copying."
)
return service_model

custom_model = (
DataScienceModel()
.with_compartment_id(target_compartment)
.with_project_id(target_project)
.with_model_file_description(
json_dict=service_model.model_file_description
)
.with_display_name(service_model.display_name)
.with_description(service_model.description)
.with_freeform_tags(**combined_freeform_tags)
.with_defined_tags(**combined_defined_tags)
.with_custom_metadata_list(service_model.custom_metadata_list)
.with_defined_metadata_list(service_model.defined_metadata_list)
.with_provenance_metadata(service_model.provenance_metadata)
.create(model_by_reference=True, **kwargs)
)
logger.info(
f"Aqua Model {custom_model.id} created with the service model {model_id}."
)

# Track unique models that were created in the user's compartment
self.telemetry.record_event_async(
Expand Down Expand Up @@ -271,6 +307,16 @@ def create_multi(
DataScienceModelGroup
Instance of DataScienceModelGroup object.
"""
member_model_ids = [{"model_id": model.model_id} for model in models]
for model in models:
if model.fine_tune_weights:
member_model_ids.extend(
[
{"model_id": fine_tune_model.model_id}
for fine_tune_model in model.fine_tune_weights
]
)

custom_model_group = (
DataScienceModelGroup()
.with_compartment_id(compartment_id)
Expand All @@ -281,7 +327,7 @@ def create_multi(
.with_defined_tags(**(defined_tags or {}))
.with_custom_metadata_list(model_custom_metadata)
# TODO: add member model inference key
.with_member_models([{"model_id": model.model_id for model in models}])
.with_member_models(member_model_ids)
)
custom_model_group.create()

Expand Down
1 change: 1 addition & 0 deletions ads/aqua/modeldeployment/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,4 @@

DEFAULT_WAIT_TIME = 12000
DEFAULT_POLL_INTERVAL = 10
DEFAULT_DEPLOYMENT_TYPE = "MODEL_STACK"
52 changes: 46 additions & 6 deletions ads/aqua/modeldeployment/deployment.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,13 @@
from ads.aqua.common.errors import AquaRuntimeError, AquaValueError
from ads.aqua.common.utils import (
DEFINED_METADATA_TO_FILE_MAP,
build_params_string,
build_pydantic_error_message,
find_restricted_params,
get_combined_params,
get_container_params_type,
get_ocid_substring,
get_params_dict,
get_params_list,
get_preferred_compatible_family,
get_resource_name,
Expand Down Expand Up @@ -61,7 +63,11 @@
ModelDeploymentConfigSummary,
MultiModelDeploymentConfigLoader,
)
from ads.aqua.modeldeployment.constants import DEFAULT_POLL_INTERVAL, DEFAULT_WAIT_TIME
from ads.aqua.modeldeployment.constants import (
DEFAULT_DEPLOYMENT_TYPE,
DEFAULT_POLL_INTERVAL,
DEFAULT_WAIT_TIME,
)
from ads.aqua.modeldeployment.entities import (
AquaDeployment,
AquaDeploymentDetail,
Expand All @@ -76,6 +82,7 @@
AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME,
AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME,
AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME,
AQUA_MODEL_DEPLOYMENT_FOLDER,
AQUA_TELEMETRY_BUCKET,
AQUA_TELEMETRY_BUCKET_NS,
COMPARTMENT_OCID,
Expand Down Expand Up @@ -162,6 +169,7 @@ def create(
cmd_var (Optional[List[str]]): Command variables for the container runtime.
freeform_tags (Optional[Dict]): Freeform tags for model deployment.
defined_tags (Optional[Dict]): Defined tags for model deployment.
deployment_type (Optional[str]): The type of model deployment.

Returns
-------
Expand Down Expand Up @@ -206,13 +214,28 @@ def create(

# Create an AquaModelApp instance once to perform the deployment creation.
model_app = AquaModelApp()
if create_deployment_details.model_id:
if (
create_deployment_details.model_id
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this assumption is a bit wrong, no?
create_deployment_details.model_id can be provided only for a single model deployment. In case of MMD or Stacked deployment we should rely on create_deployment_details.models, no?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we talked about this before, that is for single dployment, we adopt model_id and for stacked deployment, we adopt models with deployment_type. This assumption will filter out both single and stack deployment cases.

or create_deployment_details.deployment_type == DEFAULT_DEPLOYMENT_TYPE
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is a bit dangerous, what if someone change the default deployment type?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can add a validation for it and only MODEL_STACK is supported at this moment.

):
model_id = create_deployment_details.model_id
if not model_id:
if len(create_deployment_details.models) != 1:
raise AquaValueError(
"Invalid 'models' provided. Only one base model is required for model stack deployment."
)
model_id = create_deployment_details.models[0]

service_model_id = (
model_id if isinstance(model_id, str) else model_id.model_id
)
logger.debug(
f"Single model ({create_deployment_details.model_id}) provided. "
f"Single model ({service_model_id}) provided. "
"Delegating to single model creation method."
)

aqua_model = model_app.create(
model_id=create_deployment_details.model_id,
model_id=model_id,
compartment_id=compartment_id,
project_id=project_id,
freeform_tags=freeform_tags,
Expand Down Expand Up @@ -677,7 +700,7 @@ def _build_model_group_config(

def _create(
self,
aqua_model: DataScienceModel,
aqua_model: Union[DataScienceModel, DataScienceModelGroup],
create_deployment_details: CreateModelDeploymentDetails,
container_config: Dict,
) -> AquaDeployment:
Expand Down Expand Up @@ -711,7 +734,10 @@ def _create(
tags.update({Tags.TASK: aqua_model.freeform_tags.get(Tags.TASK, UNKNOWN)})

# Set up info to get deployment config
config_source_id = create_deployment_details.model_id
config_source_id = (
create_deployment_details.model_id
or create_deployment_details.models[0].model_id
)
model_name = aqua_model.display_name

# set up env and cmd var
Expand Down Expand Up @@ -862,6 +888,20 @@ def _create(
deployment_params = get_combined_params(config_params, user_params)

params = f"{params} {deployment_params}".strip()

if isinstance(aqua_model, DataScienceModelGroup):
env_var.update({"VLLM_ALLOW_RUNTIME_LORA_UPDATING": "true"})
env_var.update(
{"MODEL": f"{AQUA_MODEL_DEPLOYMENT_FOLDER}{aqua_model.base_model_id}/"}
)

params_dict = get_params_dict(params)
# updates `--served-model-name` with service model id
params_dict.update({"--served-model-name": aqua_model.base_model_id})
# adds `--enable_lora` to parameters
params_dict.update({"--enable_lora": UNKNOWN})
params = build_params_string(params_dict)

if params:
env_var.update({"PARAMS": params})
env_vars = container_spec.env_vars if container_spec else []
Expand Down
3 changes: 3 additions & 0 deletions ads/aqua/modeldeployment/entities.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,9 @@ class CreateModelDeploymentDetails(BaseModel):
defined_tags: Optional[Dict] = Field(
None, description="Defined tags for model deployment."
)
deployment_type: Optional[str] = Field(
None, description="The type of model deployment."
)

@model_validator(mode="before")
@classmethod
Expand Down
48 changes: 31 additions & 17 deletions ads/model/datascience_model_group.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
ModelGroup,
ModelGroupDetails,
ModelGroupSummary,
StackedModelGroupDetails,
UpdateModelGroupDetails,
)
except ModuleNotFoundError as err:
Expand Down Expand Up @@ -511,28 +512,38 @@ def create(

def _build_model_group_details(self) -> dict:
"""Builds model group details dict for creating or updating oci model group."""
model_group_details = HomogeneousModelGroupDetails(
custom_metadata_list=[
CustomMetadata(
key=custom_metadata.key,
value=custom_metadata.value,
description=custom_metadata.description,
category=custom_metadata.category,
)
for custom_metadata in self.custom_metadata_list._to_oci_metadata()
]
)
custom_metadata_list = [
CustomMetadata(
key=custom_metadata.key,
value=custom_metadata.value,
description=custom_metadata.description,
category=custom_metadata.category,
)
for custom_metadata in self.custom_metadata_list._to_oci_metadata()
]
member_model_details = [
MemberModelDetails(**member_model) for member_model in self.member_models
]

if self.base_model_id:
model_group_details = StackedModelGroupDetails(
custom_metadata_list=custom_metadata_list,
base_model_id=self.base_model_id,
)
member_model_details.append(MemberModelDetails(model_id=self.base_model_id))
else:
model_group_details = HomogeneousModelGroupDetails(
custom_metadata_list=custom_metadata_list
)

member_model_entries = MemberModelEntries(
member_model_details=[
MemberModelDetails(**member_model)
for member_model in self.member_models
]
member_model_details=member_model_details
)

build_model_group_details = copy.deepcopy(self._spec)
build_model_group_details.pop(self.CONST_CUSTOM_METADATA_LIST)
build_model_group_details.pop(self.CONST_MEMBER_MODELS)
build_model_group_details.pop(self.CONST_CUSTOM_METADATA_LIST, None)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add a comment here, why we are doing this?

build_model_group_details.pop(self.CONST_MEMBER_MODELS, None)
build_model_group_details.pop(self.CONST_BASE_MODEL_ID, None)
build_model_group_details.update(
{
self.CONST_COMPARTMENT_ID: self.compartment_id or COMPARTMENT_OCID,
Expand Down Expand Up @@ -581,6 +592,9 @@ def _update_from_oci_model(
)
self.set_spec(self.CONST_CUSTOM_METADATA_LIST, model_custom_metadata)

if hasattr(model_group_details, "base_model_id"):
self.set_spec(self.CONST_BASE_MODEL_ID, model_group_details.base_model_id)

# only updates member_models when oci_model_group_instance is an instance of
# oci.data_science.models.ModelGroup as oci.data_science.models.ModelGroupSummary
# doesn't have member_model_entries property.
Expand Down
Loading