diff --git a/src/helm/benchmark/model_metadata_registry.py b/src/helm/benchmark/model_metadata_registry.py index 4c25c596486..b650477c3c6 100644 --- a/src/helm/benchmark/model_metadata_registry.py +++ b/src/helm/benchmark/model_metadata_registry.py @@ -32,6 +32,7 @@ GOOGLE_PALM_2_MODEL_TAG: str = "GOOGLE_PALM_2_MODEL_TAG" GOOGLE_GEMINI_MODEL_TAG: str = "GOOGLE_GEMINI_MODEL_TAG" +GOOGLE_GEMINI_PRO_VISION_V1_TAG: str = "GOOGLE_GEMINI_PRO_VISION_V1_TAG" GOOGLE_GEMMA_INSTRUCT_MODEL_TAG: str = "GOOGLE_GEMMA_INSTRUCT_MODEL_TAG" # Models which emit garbage tokens when temperature=0. diff --git a/src/helm/benchmark/run_spec_factory.py b/src/helm/benchmark/run_spec_factory.py index dbce6eb19d8..d53c5752246 100644 --- a/src/helm/benchmark/run_spec_factory.py +++ b/src/helm/benchmark/run_spec_factory.py @@ -16,7 +16,7 @@ ANTHROPIC_CLAUDE_3_MODEL_TAG, BUGGY_TEMP_0_TAG, CHATML_MODEL_TAG, - GOOGLE_GEMINI_MODEL_TAG, + GOOGLE_GEMINI_PRO_VISION_V1_TAG, IDEFICS_INSTRUCT_MODEL_TAG, LLAVA_MODEL_TAG, OPEN_FLAMINGO_MODEL_TAG, @@ -130,10 +130,10 @@ def alter_run_spec(run_spec: RunSpec) -> RunSpec: if ANTHROPIC_CLAUDE_3_MODEL_TAG in model.tags: run_spec = singleton(AnthropicClaude3RunExpander().expand(run_spec)) - # Google Gemini Vision returns an empty completion or throws an error if max_tokens is 1 + # Google Gemini Vision v1.0 returns an empty completion or throws an error if max_tokens is 1 if ( VISION_LANGUAGE_MODEL_TAG in model.tags - and GOOGLE_GEMINI_MODEL_TAG in model.tags + and GOOGLE_GEMINI_PRO_VISION_V1_TAG in model.tags and run_spec.adapter_spec.max_tokens == 1 ): run_spec = singleton(IncreaseMaxTokensRunExpander(value=1).expand(run_spec)) diff --git a/src/helm/config/model_metadata.yaml b/src/helm/config/model_metadata.yaml index 71d89aed273..9d13ab279ce 100644 --- a/src/helm/config/model_metadata.yaml +++ b/src/helm/config/model_metadata.yaml @@ -695,10 +695,10 @@ models: creator_organization_name: Google access: limited release_date: 2023-12-13 - tags: [VISION_LANGUAGE_MODEL_TAG, GOOGLE_GEMINI_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG] + tags: [VISION_LANGUAGE_MODEL_TAG, GOOGLE_GEMINI_MODEL_TAG, GOOGLE_GEMINI_PRO_VISION_V1_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG] - name: google/gemini-1.5-pro-preview-0409 - display_name: Gemini 1.5 Pro + display_name: Gemini 1.5 Pro (preview - 0409) description: Gemini 1.5 Pro is a multimodal mixture-of-experts model capable of recalling and reasoning over fine-grained information from long contexts. ([paper](https://arxiv.org/abs/2403.05530)) creator_organization_name: Google access: limited