|  | 
| 1 | 1 | # Copyright (c) 2024 Airbyte, Inc., all rights reserved. | 
| 2 | 2 | 
 | 
| 3 |  | -from typing import Any, Hashable, Iterable, Mapping, Optional | 
|  | 3 | +from typing import Any, Iterable, Mapping, Optional | 
| 4 | 4 | 
 | 
| 5 | 5 | from airbyte_cdk.sources.declarative.retrievers import Retriever | 
| 6 | 6 | from airbyte_cdk.sources.message import MessageRepository | 
| @@ -89,21 +89,5 @@ def __init__( | 
| 89 | 89 |         self._stream_slicer = stream_slicer | 
| 90 | 90 | 
 | 
| 91 | 91 |     def generate(self) -> Iterable[Partition]: | 
| 92 |  | -        # Yield partitions for unique stream slices, avoiding duplicates | 
| 93 |  | -        seen_slices: set[Hashable] = set() | 
| 94 | 92 |         for stream_slice in self._stream_slicer.stream_slices(): | 
| 95 |  | -            slice_key = self._make_hashable(stream_slice) | 
| 96 |  | -            if slice_key in seen_slices: | 
| 97 |  | -                continue | 
| 98 |  | -            seen_slices.add(slice_key) | 
| 99 | 93 |             yield self._partition_factory.create(stream_slice) | 
| 100 |  | - | 
| 101 |  | -    @staticmethod | 
| 102 |  | -    def _make_hashable(obj: Any) -> Any: | 
| 103 |  | -        if isinstance(obj, dict): | 
| 104 |  | -            return frozenset( | 
| 105 |  | -                (k, StreamSlicerPartitionGenerator._make_hashable(v)) for k, v in obj.items() | 
| 106 |  | -            ) | 
| 107 |  | -        if isinstance(obj, list): | 
| 108 |  | -            return tuple(StreamSlicerPartitionGenerator._make_hashable(i) for i in obj) | 
| 109 |  | -        return obj | 
0 commit comments