Enabling VLLM test

scripts/Jenkinsfile

@@ -41,7 +41,8 @@ pipeline {
                            mkdir -p $PWD/Non_cli_qaic &&
                            export TOKENIZERS_PARALLELISM=false &&
                            export QEFF_HOME=$PWD/Non_cli_qaic &&
-                           pytest tests -m '(not cli) and (not on_qaic)' -n auto --junitxml=tests/tests_log1.xml &&
+                           pytest tests -m '(not cli) and (not on_qaic)' --ignore tests/vllm -n auto --junitxml=tests/tests_log1.xml &&
+                           junitparser merge tests/tests_log1.xml tests/tests_log.xml &&
                            deactivate"
                            '''
                        }
@@ -57,7 +58,8 @@ pipeline {
                            mkdir -p $PWD/Non_qaic &&
                            export TOKENIZERS_PARALLELISM=false &&
                            export QEFF_HOME=$PWD/Non_qaic &&
-                           pytest tests -m '(not cli) and (on_qaic) and (not qnn)' -n 4 --junitxml=tests/tests_log2.xml &&
+                           pytest tests -m '(not cli) and (on_qaic) and (not qnn)' --ignore tests/vllm -n 4 --junitxml=tests/tests_log2.xml &&
+                           junitparser merge tests/tests_log2.xml tests/tests_log.xml &&
                            deactivate"
                            '''
                        }
@@ -77,12 +79,23 @@ pipeline {
                            mkdir -p $PWD/cli &&
                            export TOKENIZERS_PARALLELISM=false &&
                            export QEFF_HOME=$PWD/cli &&
-                           pytest tests -m '(cli and not qnn)' --junitxml=tests/tests_log3.xml &&
+                           pytest tests -m '(cli and not qnn)' --ignore tests/vllm --junitxml=tests/tests_log3.xml &&
+                           junitparser merge tests/tests_log3.xml tests/tests_log.xml &&
                            deactivate"
                            '''
                        }
                    }
         }
+        stage('vLLM Tests') {
+
+                steps
+                {
+                    build job: 'qefficient_vllm_upstream', 
+                    parameters: [string(name: 'NAME', value: "${BUILD_TAG}")],
+                    propagate: true,
+                    wait: true
+                }
+        }
         stage('QNN CLI Tests') {
             steps {
                 timeout(time: 30, unit: 'MINUTES') {
@@ -95,7 +108,8 @@ pipeline {
                     mkdir -p $PWD/Qnn_cli &&
                     export TOKENIZERS_PARALLELISM=false &&
                     export QEFF_HOME=$PWD/Qnn_cli &&
-                    pytest tests -m '(cli and qnn)' --junitxml=tests/tests_log4.xml &&
+                    pytest tests -m '(cli and qnn)' --ignore tests/vllm --junitxml=tests/tests_log4.xml &&
+                    junitparser merge tests/tests_log4.xml tests/tests_log.xml &&
                     deactivate"
                     '''
                 }
@@ -113,8 +127,8 @@ pipeline {
                     mkdir -p $PWD/Qnn_non_cli &&
                     export TOKENIZERS_PARALLELISM=false &&
                     export QEFF_HOME=$PWD/Qnn_non_cli &&
-                    pytest tests -m '(not cli) and (qnn) and (on_qaic)' --junitxml=tests/tests_log5.xml &&
-                    junitparser merge tests/tests_log1.xml tests/tests_log2.xml tests/tests_log3.xml tests/tests_log4.xml tests/tests_log5.xml tests/tests_log.xml &&
+                    pytest tests -m '(not cli) and (qnn) and (on_qaic)' --ignore tests/vllm --junitxml=tests/tests_log5.xml &&
+                    junitparser merge tests/tests_log5.xml tests/tests_log.xml &&
                     deactivate"
                     '''
                 }

tests/vllm/test_qaic_output_consistency.py

@@ -0,0 +1,101 @@
+# -----------------------------------------------------------------------------
+#
+# Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved.
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# -----------------------------------------------------------------------------
+
+import random
+
+import pytest
+from vllm import LLM, SamplingParams
+
+# Model to test
+test_models = [
+    "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+]
+
+# Constants for configuration
+SEQ_LEN = 128
+CTX_LEN = 256
+DECOE_BSZ = 4
+DTYPE = "mxfp6"
+KV_DTYPE = "mxint8"
+
+
+@pytest.mark.vllm
+@pytest.mark.parametrize("model_name", test_models)
+def test_output_consistency(model_name):
+    """This pytest function is used to check the consistency of vLLM.
+       1) Single prompt test to check if the output generated in 5 different
+          runs yields the same results
+       2) Multiple prompt check to test if multiple prompts yield same results
+          if run in different slots.
+
+    Parameters
+    ----------
+    model_name : string
+        Huggingface model card name.
+    """
+    sampling_params = SamplingParams(temperature=0.0, max_tokens=None)
+
+    # Creating LLM Object
+    qllm = LLM(
+        model=model_name,
+        max_num_seqs=DECOE_BSZ,
+        max_model_len=CTX_LEN,
+        max_seq_len_to_capture=SEQ_LEN,
+        quantization=DTYPE,
+        kv_cache_dtype=KV_DTYPE,
+        device="qaic",
+    )
+
+    # Single prompt test
+    single_prompt = ["My name is"]
+
+    single_prompt_output = qllm.generate(single_prompt * 5, sampling_params)
+
+    check_output = []
+    for i, op in enumerate(single_prompt_output):
+        check_output.append(op.outputs[0].text)
+
+        # Assertion to check the consistency of single prompt.
+    assert len(set(check_output)) == 1, "Outputs from different slots for same prompt does not match!!"
+
+    # Multiple prompt test
+    outputDict = dict()
+    multiple_prompt = [
+        "My name is",
+        "How to eat mangosteen?",
+        "How many people died in World War II",
+        "Hello ",
+        "Who is the president of United States",
+        "Who is the president of India",
+        "When it snowfalls in San Diego",
+        "In which country yamana river flows",
+        "How many people died in World War II",
+        "Thy youth is proud livery, so gazed on now",
+        "Will be a tattered weed, of small worth held:Then being asked where all thy beauty lies",
+        "Where all the treasure of thy lusty days",
+        "To say, within thine own deep-sunken eyes",
+        "Where is Statue of Liberty located?",
+    ]
+
+    for p in multiple_prompt:
+        outputDict[p] = []
+
+    for _ in range(5):
+        random.shuffle(multiple_prompt)
+        multiple_prompt_output = qllm.generate(multiple_prompt, sampling_params)
+        for i, op in enumerate(multiple_prompt_output):
+            generated_text = op.outputs[0].text
+            outputDict[multiple_prompt[i]].append(str(multiple_prompt[i] + generated_text))
+
+    # Assertion to check multiple prompts.
+    for key in outputDict.keys():
+        assert len(set(outputDict[key])) == 1, "Outputs from different slots for same prompt does not match!!"
+
+    # Assertion to check if any prompts are missed.
+    assert len(multiple_prompt) == len(multiple_prompt_output), (
+        "Number of Generated Tokens do not match the number of valid inputs!!"
+    )