VQE callback not returning all parameters

[radumarg]

Environment

• Qiskit Algorithms version:
• Python version:
• Operating system:

What is happening?

Tried to run VQE, it runs just fine, but in previous versions I could add a callback that returns the optimization parameters and the variance of the result in addition to the mean results. Right now, the callback returns an array with a single value instead of a full list of parameters, and the metadata argument in the callback no longer contains the variance of the result. I have attached the code below.

How can we reproduce the issue?

import time

from qiskit.providers.exceptions import QiskitBackendNotFoundError
from qiskit.qasm3 import loads
from qiskit_algorithms import VQE
from qiskit_algorithms.optimizers import COBYLA
from qiskit_ibm_runtime import EstimatorV2
from qiskit_ibm_runtime import QiskitRuntimeService
from qiskit_ibm_runtime import SamplerV2
from qiskit_nature.second_q.circuit.library import HartreeFock
from qiskit_nature.second_q.circuit.library import UCCSD
from qiskit_nature.second_q.drivers import MethodType
from qiskit_nature.second_q.drivers import PySCFDriver
from qiskit_nature.second_q.formats import MoleculeInfo
from qiskit_nature.second_q.mappers import ParityMapper
from qiskit_nature.second_q.transformers import FreezeCoreTransformer
from qiskit_nature.units import DistanceUnit
from qiskit_aer import AerSimulator
from qiskit.transpiler.preset_passmanagers import generate_preset_pass_manager

def extend_observable(base_op, num_qubits, target_qubit=0):
    """
    Extend a SparsePauliOp to match the number of qubits in the circuit.

    Args:
        base_op (SparsePauliOp): Base observable to extend.
        num_qubits (int): Total number of qubits in the circuit.
        target_qubit (int): Qubit index where the base_op starts (default 0).

    Returns:
        SparsePauliOp: Extended observable.

    Raises:
        ValueError: If base operator size plus target qubit exceeds circuit qubit count.
    """
    if base_op.num_qubits == num_qubits and target_qubit == 0:
        return base_op

    if base_op.num_qubits + target_qubit > num_qubits:
        raise ValueError(
            "Base operator size plus target qubit exceeds circuit qubit count"
        )

    num_before = target_qubit
    num_after = num_qubits - (target_qubit + base_op.num_qubits)

    new_paulis = [
        f"{'I' * num_before}{str(pauli)}{'I' * num_after}" for pauli in base_op.paulis
    ]

    return SparsePauliOp(new_paulis, coeffs=base_op.coeffs)

service = QiskitRuntimeService(channel="local")

try:
    backend = AerSimulator()
    estimator = EstimatorV2(mode=backend, options={"default_shots": 10000})
except QiskitBackendNotFoundError:
    raise Exception(
        f"The backend name specified: fake_vigo is not supported by IBMQ."
    )


molecule_info = MoleculeInfo(
    symbols=['H', 'H'],
    coords=[[0, 0, 0], [0, 0, 0.735]],
    multiplicity=1,
    charge=0,
    units=DistanceUnit.ANGSTROM,
    masses=None,
)

driver = PySCFDriver.from_molecule(
    molecule_info, basis="sto-3g", method=MethodType.RHF
)
electronic_structure_problem = driver.run()

# freeze core orbitals
transformer = FreezeCoreTransformer()
electronic_structure_problem = transformer.transform(electronic_structure_problem)

second_q_ops = electronic_structure_problem.second_q_ops()
hamiltonian = second_q_ops[0]

mapper =  ParityMapper()
qubit_operator = mapper.map(hamiltonian)

num_spatial_orbitals = electronic_structure_problem.num_spatial_orbitals
num_particles = electronic_structure_problem.num_particles
ansatz = UCCSD(
    num_spatial_orbitals,
    num_particles,
    qubit_mapper=mapper,
    initial_state=HartreeFock(
        num_spatial_orbitals,
        num_particles,
        mapper,
    ),
)

pm = generate_preset_pass_manager(optimization_level=1, backend=backend)
ansatz = pm.run(ansatz)

qubit_operator = extend_observable(qubit_operator, ansatz.num_qubits)

optimizer = COBYLA()

start_time = time.time()

class Callback:
    def __init__(self):
        self.counts = []
        self.values = []
        self.min_values = []
        self.variances = []
        self.parameters = []
        self.min_value = None
        self.variance = 0

    def __call__(self, eval_count, parameters, mean, metadata):
        print(eval_count)
        print(parameters)
        print(mean)
        print(metadata)

callback = Callback()


vqe = VQE(
    estimator=estimator, 
    ansatz=ansatz, 
    optimizer=optimizer, 
    callback=callback
)

result = vqe.compute_minimum_eigenvalue(qubit_operator)

What should happen?

In previous version I could add a callback that returns the optimization parameters and the variance of the result in addition to the mean results.

Any suggestions?

The callback should return the full list of ansatz parameters and the variance of the result in metadata call-back argument duting each iteration like it used to do in previous versions.