From e944a1d2776bad730ad8bebe920099394779e037 Mon Sep 17 00:00:00 2001 From: Cameron Booker Date: Tue, 13 Aug 2024 09:59:29 +0100 Subject: [PATCH] Further updates --- .../examples/qcvv/qcvv_css.ipynb | 6 +- .../supermarq/qcvv/base_experiment.py | 58 ++++++++++++------- .../supermarq/qcvv/base_experiment_test.py | 56 ++++++++++++------ 3 files changed, 78 insertions(+), 42 deletions(-) diff --git a/supermarq-benchmarks/examples/qcvv/qcvv_css.ipynb b/supermarq-benchmarks/examples/qcvv/qcvv_css.ipynb index c12f0b530..510a2e08e 100644 --- a/supermarq-benchmarks/examples/qcvv/qcvv_css.ipynb +++ b/supermarq-benchmarks/examples/qcvv/qcvv_css.ipynb @@ -99,7 +99,7 @@ " records.append({**sample.data, **sample.probabilities})\n", " return pd.DataFrame(records)\n", "\n", - " def analyse_results(self, plot_results: bool = True) -> NaiveExperiment:\n", + " def analyze_results(self, plot_results: bool = True) -> NaiveExperiment:\n", " \"\"\"To analyse the results to fit a simple exponential decay. This can be done easily\n", " by fitting a linear model to the logarithm of the equation above.\n", " \"\"\"\n", @@ -212,7 +212,7 @@ ], "source": [ "if experiment.collect_data():\n", - " results = experiment.analyse_results(plot_results=True)\n", + " results = experiment.analyze_results(plot_results=True)\n", " print(results)" ] }, @@ -324,7 +324,7 @@ ], "source": [ "if experiment.collect_data():\n", - " results = experiment.analyse_results(plot_results=True)\n", + " results = experiment.analyze_results(plot_results=True)\n", " print(results)" ] }, diff --git a/supermarq-benchmarks/supermarq/qcvv/base_experiment.py b/supermarq-benchmarks/supermarq/qcvv/base_experiment.py index d0b3f0bfb..b636618c7 100644 --- a/supermarq-benchmarks/supermarq/qcvv/base_experiment.py +++ b/supermarq-benchmarks/supermarq/qcvv/base_experiment.py @@ -33,8 +33,8 @@ class Sample: that is needed for analysis """ - circuit: cirq.Circuit - """The sample circuit.""" + raw_circuit: cirq.Circuit + """The raw (i.e. pre-compiled) sample circuit.""" data: dict[str, Any] """The corresponding data about the circuit""" probabilities: dict[str, float] = field(init=False) @@ -42,8 +42,8 @@ class Sample: job: css.Job | None = None """The superstaq job corresponding to the sample. Defaults to None if no job is associated with the sample.""" - raw_circuit: cirq.Circuit = field(init=False) - """The raw (pre-compiled) circuit. Only used if the circuits are compiled for a specific + compiled_circuit: cirq.Circuit = field(init=False) + """The compiled circuit. Only used if the circuits are compiled for a specific target.""" @property @@ -63,6 +63,17 @@ def target(self) -> str: # Otherwise the experiment hasn't yet been run so there is no target. return "No target" + @property + def circuit(self) -> cirq.Circuit: + """Returns: + The circuit used for the experiment. Defaults to the compiled circuit if available + and if not returns the raw circuit. + """ + if hasattr(self, "compiled_circuit"): + return self.compiled_circuit + + return self.raw_circuit + @dataclass(frozen=True) class BenchmarkingResults(ABC): @@ -111,7 +122,7 @@ class BenchmarkingExperiment(ABC, Generic[ResultsT]): .. code:: if experiment.collect_data(): - results = experiment.analyse_results(<>) + results = experiment.analyze_results(<>) #. The final results of the experiment will be stored in the :code:`results` attribute as a :class:`BenchmarkingResults` of values, while all the data from the experiment will be @@ -146,7 +157,7 @@ class BenchmarkingExperiment(ABC, Generic[ResultsT]): computational basis resulting from running each circuit and combine the relevant details into a :class:`pandas.DataFrame`. - #. :meth:`analyse_results`: Analyse the data in the :attr:`raw_data` dataframe and return a + #. :meth:`analyze_results`: Analyse the data in the :attr:`raw_data` dataframe and return a :class:`BenchmarkingResults` object containing the results of the experiment. #. :meth:`plot_results`: Produce any relevant plots that are useful for understanding the @@ -378,7 +389,7 @@ def _validate_circuits(self) -> None: # Public Methods # ################### @abstractmethod - def analyse_results(self, plot_results: bool = True) -> ResultsT: + def analyze_results(self, plot_results: bool = True) -> ResultsT: """Perform the experiment analysis and store the results in the `results` attribute. Args: @@ -450,8 +461,7 @@ def compile_circuits(self, target: str, **kwargs: Any) -> None: ).circuits for k, sample in enumerate(self.samples): - sample.raw_circuit = sample.circuit.copy() - sample.circuit = compiled_circuits[k] # type: ignore[assignment] + sample.compiled_circuit = compiled_circuits[k] # type: ignore[assignment] @abstractmethod def plot_results(self) -> None: @@ -488,7 +498,7 @@ def prepare_experiment( def run_on_device( self, target: str, - shots: int = 10_000, + repetitions: int = 10_000, method: str | None = None, overwrite: bool = False, **target_options: Any, @@ -502,7 +512,7 @@ def run_on_device( Args: target: The name of a Superstaq target. - shots: The number of shots to sample. Defaults to 10,000. + repetitions: The number of shots to sample. Defaults to 10,000. method: Optional method to use on the Superstaq device. Defaults to None corresponding to normal running. target_options: Optional configuration dictionary passed when submitting the job. @@ -519,18 +529,21 @@ def run_on_device( [sample.circuit for sample in self.samples], target=target, method=method, - repetitions=shots, + repetitions=repetitions, **target_options, ) + compiled_circuits = experiment_job.compiled_circuits() + for k, sample in enumerate(self.samples): sample.job = experiment_job[k] + sample.compiled_circuit = compiled_circuits[k] return experiment_job def run_with_simulator( self, simulator: cirq.Sampler | None = None, - shots: int = 10_000, + repetitions: int = 10_000, overwrite: bool = False, ) -> None: """Use the local simulator to sample the circuits and store the resulting probabilities. @@ -538,7 +551,7 @@ def run_with_simulator( Args: simulator: A local :class:`~cirq.Sampler` to use. If None then the default :class:`cirq.Simulator` simulator is used. Defaults to None. - shots: The number of shots to sample. Defaults to 10,000. + repetitions: The number of shots to sample. Defaults to 10,000. overwrite: Whether to force an experiment run even if there is existing data that would be over written in the process. Defaults to False. """ @@ -549,10 +562,10 @@ def run_with_simulator( simulator = cirq.Simulator() for sample in tqdm(self.samples, desc="Simulating circuits"): - result = simulator.run(sample.circuit, repetitions=shots) + result = simulator.run(sample.circuit, repetitions=repetitions) hist = result.histogram(key=cirq.measurement_key_name(sample.circuit)) sample.probabilities = { - f"{i:0{self.num_qubits}b}": count / shots for i, count in hist.items() + f"{i:0{self.num_qubits}b}": count / repetitions for i, count in hist.items() } def run_with_callable( @@ -572,18 +585,21 @@ def run_with_callable( kwargs: Additional arguments to pass to the custom function. Raises: - RuntimeError: If the returned probabilities dictionary keys is missing bitstrings. + RuntimeError: If the returned probabilities dictionary keys is missing include + an incorrect number of bits. RuntimeError: If the returned probabilities dictionary values do not sum to 1.0. """ if not overwrite: self._run_check() for sample in tqdm(self.samples, desc="Running circuits"): probability = circuit_eval_func(sample.circuit, **kwargs) - if sorted(probability.keys()) != sorted( - f"{i:0{self.num_qubits}b}" for i in range(2**self.num_qubits) - ): - raise RuntimeError("Returned probabilities are missing bitstrings.") + if not all(len(key) == self.num_qubits for key in probability.keys()): + raise RuntimeError("Returned probabilities include an incorrect number of bits.") if not math.isclose(sum(probability.values()), 1.0): raise RuntimeError("Returned probabilities do not sum to 1.0.") + for k in range(2**self.num_qubits): + if (bitstring := format(k, f"0{self.num_qubits}b")) not in probability: + probability[bitstring] = 0.0 + sample.probabilities = probability diff --git a/supermarq-benchmarks/supermarq/qcvv/base_experiment_test.py b/supermarq-benchmarks/supermarq/qcvv/base_experiment_test.py index 495b774ce..ee7d2fac9 100644 --- a/supermarq-benchmarks/supermarq/qcvv/base_experiment_test.py +++ b/supermarq-benchmarks/supermarq/qcvv/base_experiment_test.py @@ -55,15 +55,17 @@ def sample_circuits() -> list[Sample]: qubits = cirq.LineQubit.range(2) return [ Sample( - circuit=cirq.Circuit(cirq.CZ(*qubits), cirq.CZ(*qubits), cirq.measure(*qubits)), + raw_circuit=cirq.Circuit(cirq.CZ(*qubits), cirq.CZ(*qubits), cirq.measure(*qubits)), data={"circuit": 1}, ), - Sample(circuit=cirq.Circuit(cirq.CX(*qubits), cirq.measure(*qubits)), data={"circuit": 2}), + Sample( + raw_circuit=cirq.Circuit(cirq.CX(*qubits), cirq.measure(*qubits)), data={"circuit": 2} + ), ] def test_sample_target_property() -> None: - sample = Sample(circuit=MagicMock(), data={}) + sample = Sample(raw_circuit=MagicMock(), data={}) assert sample.target == "No target" sample.probabilities = {"0": 0.25, "1": 0.75} @@ -111,7 +113,7 @@ def test_empty_samples_error(abc_experiment: BenchmarkingExperiment[ExampleResul def test_prepare_experiment_overwrite_error( abc_experiment: BenchmarkingExperiment[ExampleResults], ) -> None: - abc_experiment._samples = [Sample(circuit=MagicMock(), data={})] + abc_experiment._samples = [Sample(raw_circuit=MagicMock(), data={})] abc_experiment._build_circuits = MagicMock() with pytest.raises( @@ -125,7 +127,7 @@ def test_prepare_experiment_overwrite_error( def test_prepare_experiment_overwrite( abc_experiment: BenchmarkingExperiment[ExampleResults], ) -> None: - abc_experiment._samples = [Sample(circuit=MagicMock(), data={})] + abc_experiment._samples = [Sample(raw_circuit=MagicMock(), data={})] abc_experiment._build_circuits = MagicMock() abc_experiment._validate_circuits = MagicMock() @@ -154,7 +156,7 @@ def test_run_with_simulator( mock_result.histogram.return_value = {0: 0, 1: 100, 2: 0, 3: 0} test_sim.run.return_value = mock_result - abc_experiment.run_with_simulator(simulator=test_sim, shots=100) + abc_experiment.run_with_simulator(simulator=test_sim, repetitions=100) # Test simulator calls test_sim.run.assert_has_calls( @@ -197,7 +199,7 @@ def test_run_with_simulator_default_target( mock_result.histogram.return_value = {0: 0, 1: 100, 2: 0, 3: 0} target().run.return_value = mock_result - abc_experiment.run_with_simulator(shots=100) + abc_experiment.run_with_simulator(repetitions=100) # Test simulator calls target().run.assert_has_calls( @@ -220,11 +222,11 @@ def test_run_on_device( abc_experiment._service = (mock_service := MagicMock()) job = abc_experiment.run_on_device( - target="example_target", shots=100, overwrite=False, **{"some": "options"} + target="example_target", repetitions=100, overwrite=False, **{"some": "options"} ) mock_service.create_job.assert_called_once_with( - [sample_circuits[0].circuit, sample_circuits[1].circuit], + [sample_circuits[0].raw_circuit, sample_circuits[1].raw_circuit], target="example_target", method=None, repetitions=100, @@ -232,6 +234,14 @@ def test_run_on_device( ) assert job == mock_service.create_job.return_value + assert ( + sample_circuits[0].compiled_circuit + == mock_service.create_job.return_value.compiled_circuits.return_value[0] + ) + assert ( + sample_circuits[1].compiled_circuit + == mock_service.create_job.return_value.compiled_circuits.return_value[1] + ) def test_run_on_device_existing_probabilties( @@ -258,15 +268,23 @@ def test_run_on_device_dry_run( abc_experiment._samples = sample_circuits abc_experiment._service = (mock_service := MagicMock()) - job = abc_experiment.run_on_device(target="example_target", shots=100, method="dry-run") + job = abc_experiment.run_on_device(target="example_target", repetitions=100, method="dry-run") mock_service.create_job.assert_called_once_with( - [sample_circuits[0].circuit, sample_circuits[1].circuit], + [sample_circuits[0].raw_circuit, sample_circuits[1].raw_circuit], target="example_target", method="dry-run", repetitions=100, ) assert job == mock_service.create_job.return_value + assert ( + sample_circuits[0].compiled_circuit + == mock_service.create_job.return_value.compiled_circuits.return_value[0] + ) + assert ( + sample_circuits[1].compiled_circuit + == mock_service.create_job.return_value.compiled_circuits.return_value[1] + ) def test_interleave_circuit() -> None: @@ -342,14 +360,14 @@ def test_validate_circuits( abc_experiment._validate_circuits() # Add a gate so not all measurements are terminal - abc_experiment._samples[0].circuit += cirq.X(abc_experiment.qubits[0]) + abc_experiment._samples[0].raw_circuit += cirq.X(abc_experiment.qubits[0]) with pytest.raises( ValueError, match="QCVV experiment circuits can only contain terminal measurements." ): abc_experiment._validate_circuits() # Remove measurements - abc_experiment._samples[0].circuit = abc_experiment._samples[0].circuit[:-2] + cirq.measure( + abc_experiment._samples[0].raw_circuit = abc_experiment._samples[0].circuit[:-2] + cirq.measure( abc_experiment.qubits[0] ) with pytest.raises( @@ -359,7 +377,7 @@ def test_validate_circuits( abc_experiment._validate_circuits() # Remove all measurements - abc_experiment._samples[0].circuit = abc_experiment._samples[0].circuit[:-2] + abc_experiment._samples[0].raw_circuit = abc_experiment._samples[0].circuit[:-2] with pytest.raises( ValueError, match="QCVV experiment circuits must contain measurements.", @@ -553,7 +571,7 @@ def test_run_with_callable( ) -> None: abc_experiment._samples = sample_circuits test_callable = MagicMock() - test_callable.return_value = {"00": 0.0, "01": 0.2, "10": 0.7, "11": 0.1} + test_callable.return_value = {"01": 0.2, "10": 0.7, "11": 0.1} abc_experiment.run_with_callable(test_callable, some="kwargs") @@ -567,15 +585,17 @@ def test_run_with_callable( assert sample_circuits[1].probabilities == {"00": 0.0, "01": 0.2, "10": 0.7, "11": 0.1} -def test_run_with_callable_missing_bitstring( +def test_run_with_callable_bad_bitstring( abc_experiment: BenchmarkingExperiment[ExampleResults], sample_circuits: list[Sample], ) -> None: abc_experiment._samples = sample_circuits test_callable = MagicMock() - test_callable.return_value = {"00": 0.0, "01": 0.2, "10": 0.8} + test_callable.return_value = {"000": 0.0, "01": 0.2, "10": 0.8} - with pytest.raises(RuntimeError, match="Returned probabilities are missing bitstrings."): + with pytest.raises( + RuntimeError, match="Returned probabilities include an incorrect number of bits." + ): abc_experiment.run_with_callable(test_callable, some="kwargs")