Skip to content

fix(fivetran): missing setuptools dependency #14954

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom
Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

fix(fivetran): missing setuptools dependency #14954

wants to merge 1 commit into from

Conversation

sgomezvillamor
Copy link
Contributor

No description provided.

@github-actions github-actions bot added the ingestion PR or Issue related to the ingestion of metadata label Oct 8, 2025
@codecov Codecov
Copy link

codecov bot commented Oct 8, 2025 •
edited
Loading

❌ 12 Tests Failed:

Tests completed Failed Passed Skipped
3999 12 3987 47
View the top 3 failed test(s) by shortest run time 
tests.unit.test_iceberg::test_filtering
Stack Traces | 0.205s run time 
def test_filtering() -> None:
        source = with_iceberg_source(
            processing_threads=1,
            table_pattern=AllowDenyPattern(deny=[".*abcd.*"]),
            namespace_pattern=AllowDenyPattern(allow=["namespace1"]),
        )
        mock_catalog = MockCatalog(
            {
                "namespace1": {
                    "table_xyz": lambda catalog: Table(
                        identifier=("namespace1", "table_xyz"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespace1/table_xyz",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespace1/table_xyz",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                    "JKLtable": lambda catalog: Table(
                        identifier=("namespace1", "JKLtable"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespace1/JKLtable",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespace1/JKLtable",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                    "table_abcd": lambda catalog: Table(
                        identifier=("namespace1", "table_abcd"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespace1/table_abcd",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespace1/table_abcd",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                    "aaabcd": lambda catalog: Table(
                        identifier=("namespace1", "aaabcd"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespace1/aaabcd",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespace1/aaabcd",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                },
                "namespace2": {
                    "foo": lambda catalog: Table(
                        identifier=("namespace2", "foo"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespace2/foo",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespace2/foo",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                    "bar": lambda catalog: Table(
                        identifier=("namespace2", "bar"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespace2/bar",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespace2/bar",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                },
                "namespace3": {
                    "sales": lambda catalog: Table(
                        identifier=("namespace3", "sales"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespace3/sales",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespace3/sales",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                    "products": lambda catalog: Table(
                        identifier=("namespace2", "bar"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespace3/products",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespace3/products",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                },
            }
        )
        with patch(
            "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog"
        ) as get_catalog:
            get_catalog.return_value = mock_catalog
            wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()]
            # ingested 2 tables (6 MCPs each) and 1 namespace (4 MCPs)
            expected_wu_urns = [
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.table_xyz,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.JKLtable,PROD)",
            ] * MCPS_PER_TABLE + [
                "urn:li:container:075fc8fdac17b0eb3482e73052e875f1",
            ] * MCPS_PER_NAMESPACE
>           assert len(wu) == len(expected_wu_urns)
E           AssertionError: assert 5 == 19
E            +  where 5 = len([MetadataWorkUnit(id='urn:li:container:075fc8fdac17b0eb3482e73052e875f1-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:075fc8fdac17b0eb3482e73052e875f1', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929122613, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:075fc8fdac17b0eb3482e73052e875f1-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:075fc8fdac17b0eb3482e73052e875f1', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespace1', 'qualifiedName': 'namespace1', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929122613, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:075fc8fdac17b0eb3482e73052e875f1-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:075fc8fdac17b0eb3482e73052e875f1', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929122613, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:075fc8fdac17b0eb3482e73052e875f1-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:075fc8fdac17b0eb3482e73052e875f1', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929122613, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:075fc8fdac17b0eb3482e73052e875f1-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:075fc8fdac17b0eb3482e73052e875f1', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929122613, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True)])
E            +  and   19 = len(['urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.table_xyz,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.JKLtable,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.table_xyz,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.JKLtable,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.table_xyz,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.JKLtable,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.table_xyz,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.JKLtable,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.table_xyz,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.JKLtable,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.table_xyz,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.JKLtable,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.table_xyz,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace1.JKLtable,PROD)', 'urn:li:container:075fc8fdac17b0eb3482e73052e875f1', 'urn:li:container:075fc8fdac17b0eb3482e73052e875f1', 'urn:li:container:075fc8fdac17b0eb3482e73052e875f1', 'urn:li:container:075fc8fdac17b0eb3482e73052e875f1', 'urn:li:container:075fc8fdac17b0eb3482e73052e875f1'])

tests/unit/test_iceberg.py:1235: AssertionError
tests.unit.test_iceberg::test_proper_run_with_multiple_namespaces
Stack Traces | 0.206s run time 
def test_proper_run_with_multiple_namespaces() -> None:
        source = with_iceberg_source(processing_threads=3)
        mock_catalog = MockCatalog(
            {
                "namespaceA": {
                    "table1": lambda catalog: Table(
                        identifier=("namespaceA", "table1"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceA/table1",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceA/table1",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    )
                },
                "namespaceB": {},
            }
        )
        with patch(
            "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog"
        ) as get_catalog:
            get_catalog.return_value = mock_catalog
            wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()]
            # ingested 1 table (6 MCPs) and 2 namespaces (4 MCPs each), despite exception
            expected_wu_urns = [
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)",
            ] * MCPS_PER_TABLE + [
                "urn:li:container:74727446a56420d80ff3b1abf2a18087",
                "urn:li:container:390e031441265aae5b7b7ae8d51b0c1f",
            ] * MCPS_PER_NAMESPACE
>           assert len(wu) == len(expected_wu_urns)
E           AssertionError: assert 10 == 17
E            +  where 10 = len([MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123090, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceA', 'qualifiedName': 'namespaceA', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123090, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123090, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123090, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123090, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123090, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceB', 'qualifiedName': 'namespaceB', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123090, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123090, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123090, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123090, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True)])
E            +  and   17 = len(['urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f'])

tests/unit/test_iceberg.py:1100: AssertionError
tests.unit.test_iceberg::test_handle_expected_exceptions
Stack Traces | 0.207s run time 
def test_handle_expected_exceptions() -> None:
        source = with_iceberg_source(processing_threads=3)
    
        def _raise_no_such_property_exception(_: Catalog) -> Never:
            raise NoSuchPropertyException()
    
        def _raise_no_such_iceberg_table_exception(_: Catalog) -> Never:
            raise NoSuchIcebergTableError()
    
        def _raise_file_not_found_error(_: Catalog) -> Never:
            raise FileNotFoundError()
    
        def _raise_no_such_table_exception(_: Catalog) -> Never:
            raise NoSuchTableError()
    
        def _raise_server_error(_: Catalog) -> Never:
            raise ServerError()
    
        def _raise_rest_error(_: Catalog) -> Never:
            raise RESTError()
    
        def _raise_fileio_error(_: Catalog) -> Never:
            raise ValueError("Could not initialize FileIO: abc.dummy.fileio")
    
        mock_catalog = MockCatalog(
            {
                "namespaceA": {
                    "table1": lambda catalog: Table(
                        identifier=("namespaceA", "table1"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceA/table1",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceA/table1",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                    "table2": lambda catalog: Table(
                        identifier=("namespaceA", "table2"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceA/table2",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceA/table2",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                    "table3": lambda catalog: Table(
                        identifier=("namespaceA", "table3"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceA/table3",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceA/table3",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                    "table4": lambda catalog: Table(
                        identifier=("namespaceA", "table4"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceA/table4",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceA/table4",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                    "table5": _raise_no_such_property_exception,
                    "table6": _raise_no_such_table_exception,
                    "table7": _raise_file_not_found_error,
                    "table8": _raise_no_such_iceberg_table_exception,
                    "table9": _raise_server_error,
                    "table10": _raise_fileio_error,
                    "table11": _raise_rest_error,
                }
            }
        )
        with patch(
            "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog"
        ) as get_catalog:
            get_catalog.return_value = mock_catalog
            wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()]
            # ingested 4 tables (6 MCPs each) and 1 namespace (4 MCPs)
            expected_wu_urns = [
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table2,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table3,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table4,PROD)",
            ] * MCPS_PER_TABLE + [
                "urn:li:container:390e031441265aae5b7b7ae8d51b0c1f",
            ] * MCPS_PER_NAMESPACE
>           assert len(wu) == len(expected_wu_urns)
E           AssertionError: assert 5 == 33
E            +  where 5 = len([MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929122844, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceA', 'qualifiedName': 'namespaceA', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929122845, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929122845, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929122845, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929122845, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True)])
E            +  and   33 = len(['urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table4,PROD)', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f'])

tests/unit/test_iceberg.py:1346: AssertionError
tests.unit.test_iceberg::test_handle_unexpected_exceptions
Stack Traces | 0.207s run time 
def test_handle_unexpected_exceptions() -> None:
        source = with_iceberg_source(processing_threads=3)
    
        def _raise_exception(_: Catalog) -> Never:
            raise Exception()
    
        def _raise_other_value_error_exception(_: Catalog) -> Never:
            raise ValueError("Other value exception")
    
        mock_catalog = MockCatalog(
            {
                "namespaceA": {
                    "table1": lambda catalog: Table(
                        identifier=("namespaceA", "table1"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceA/table1",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceA/table1",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                    "table2": lambda catalog: Table(
                        identifier=("namespaceA", "table2"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceA/table2",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceA/table2",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                    "table3": lambda catalog: Table(
                        identifier=("namespaceA", "table3"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceA/table3",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceA/table3",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                    "table4": lambda catalog: Table(
                        identifier=("namespaceA", "table4"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceA/table4",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceA/table4",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                    "table5": _raise_exception,
                    "table6": _raise_other_value_error_exception,
                },
            }
        )
        with patch(
            "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog"
        ) as get_catalog:
            get_catalog.return_value = mock_catalog
            wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()]
            # ingested 4 tables (6 MCPs each) and 1 namespace (4 MCPs)
            expected_wu_urns = [
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table2,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table3,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table4,PROD)",
            ] * MCPS_PER_TABLE + [
                "urn:li:container:390e031441265aae5b7b7ae8d51b0c1f",
            ] * MCPS_PER_NAMESPACE
            # assert len(wu) == len(expected_wu_urns)
            urns = []
            for unit in wu:
                assert isinstance(unit.metadata, MetadataChangeProposalWrapper)
                urns.append(unit.metadata.entityUrn)
>           TestCase().assertCountEqual(
                urns,
                expected_wu_urns,
            )

tests/unit/test_iceberg.py:1446: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.../hostedtoolcache/Python/3.11.13....../x64/lib/python3.11/unittest/case.py:1233: in assertCountEqual
    self.fail(msg)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <unittest.case.TestCase testMethod=runTest>
msg = "Element counts were not equal:\nFirst has 0, Second has 7:  'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.t...paceA.table3,PROD)'\nFirst has 0, Second has 7:  'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table4,PROD)'"

    def fail(self, msg=None):
        """Fail immediately, with the given message."""
>       raise self.failureException(msg)
E       AssertionError: Element counts were not equal:
E       First has 0, Second has 7:  'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)'
E       First has 0, Second has 7:  'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table2,PROD)'
E       First has 0, Second has 7:  'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table3,PROD)'
E       First has 0, Second has 7:  'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table4,PROD)'

.../hostedtoolcache/Python/3.11.13....../x64/lib/python3.11/unittest/case.py:703: AssertionError
tests.unit.test_iceberg::test_known_exception_while_listing_tables
Stack Traces | 0.208s run time 
def test_known_exception_while_listing_tables() -> None:
        source = with_iceberg_source(processing_threads=2)
        mock_catalog = MockCatalogExceptionListingTables(
            {
                "namespaceA": {
                    "table1": lambda catalog: Table(
                        identifier=("namespaceA", "table1"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceA/table1",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceA/table1",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    )
                },
                "no_such_namespace": {},
                "rest_error": {},
                "namespaceB": {
                    "table2": lambda catalog: Table(
                        identifier=("namespaceB", "table2"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceB/table2",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceB/table2",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                    "table3": lambda catalog: Table(
                        identifier=("namespaceB", "table3"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceB/table3",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceB/table3",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                },
                "namespaceC": {
                    "table4": lambda catalog: Table(
                        identifier=("namespaceC", "table4"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceC/table4",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceC/table4",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    )
                },
                "namespaceD": {
                    "table5": lambda catalog: Table(
                        identifier=("namespaceD", "table5"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceA/table5",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceA/table5",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    )
                },
            }
        )
        with patch(
            "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog"
        ) as get_catalog:
            get_catalog.return_value = mock_catalog
            wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()]
            # ingested 5 tables (6 MCPs each) and 6 namespaces (4 MCPs each), despite exception
            expected_wu_urns = [
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)",
            ] * MCPS_PER_TABLE + [
                "urn:li:container:390e031441265aae5b7b7ae8d51b0c1f",
                "urn:li:container:9cb5e87ec392b231720f23bf00d6f6aa",
                "urn:li:container:74727446a56420d80ff3b1abf2a18087",
                "urn:li:container:3f9a24213cca64ab22e409d1b9a94789",
                "urn:li:container:38a0583b0305ec5066cb708199f6848c",
                "urn:li:container:7b510fcb61d4977da0b1707e533999d8",
            ] * MCPS_PER_NAMESPACE
>           assert len(wu) == len(expected_wu_urns)
E           AssertionError: assert 30 == 65
E            +  where 30 = len([MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123925, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceA', 'qualifiedName': 'namespaceA', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123925, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123925, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123925, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123926, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:9cb5e87ec392b231720f23bf00d6f6aa-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:9cb5e87ec392b231720f23bf00d6f6aa', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123926, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:9cb5e87ec392b231720f23bf00d6f6aa-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:9cb5e87ec392b231720f23bf00d6f6aa', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'no_such_namespace', 'qualifiedName': 'no_such_namespace', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123926, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:9cb5e87ec392b231720f23bf00d6f6aa-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:9cb5e87ec392b231720f23bf00d6f6aa', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123926, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:9cb5e87ec392b231720f23bf00d6f6aa-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:9cb5e87ec392b231720f23bf00d6f6aa', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123926, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:9cb5e87ec392b231720f23bf00d6f6aa-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:9cb5e87ec392b231720f23bf00d6f6aa', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123926, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:7b510fcb61d4977da0b1707e533999d8-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:7b510fcb61d4977da0b1707e533999d8', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123926, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:7b510fcb61d4977da0b1707e533999d8-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:7b510fcb61d4977da0b1707e533999d8', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'rest_error', 'qualifiedName': 'rest_error', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123926, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:7b510fcb61d4977da0b1707e533999d8-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:7b510fcb61d4977da0b1707e533999d8', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123926, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:7b510fcb61d4977da0b1707e533999d8-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:7b510fcb61d4977da0b1707e533999d8', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123926, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:7b510fcb61d4977da0b1707e533999d8-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:7b510fcb61d4977da0b1707e533999d8', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123926, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123926, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceB', 'qualifiedName': 'namespaceB', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123926, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123926, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123926, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123926, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123927, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceC', 'qualifiedName': 'namespaceC', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123927, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123927, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123927, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123927, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123927, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceD', 'qualifiedName': 'namespaceD', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123927, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123927, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123927, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123927, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True)])
E            +  and   65 = len(['urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:9cb5e87ec392b231720f23bf00d6f6aa', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c', 'urn:li:container:7b510fcb61d4977da0b1707e533999d8', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:9cb5e87ec392b231720f23bf00d6f6aa', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c', 'urn:li:container:7b510fcb61d4977da0b1707e533999d8', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:9cb5e87ec392b231720f23bf00d6f6aa', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c', 'urn:li:container:7b510fcb61d4977da0b1707e533999d8', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:9cb5e87ec392b231720f23bf00d6f6aa', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c', 'urn:li:container:7b510fcb61d4977da0b1707e533999d8', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:9cb5e87ec392b231720f23bf00d6f6aa', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c', 'urn:li:container:7b510fcb61d4977da0b1707e533999d8'])

tests/unit/test_iceberg.py:945: AssertionError
tests.unit.test_iceberg::test_known_exception_while_retrieving_namespace_properties
Stack Traces | 0.208s run time 
def test_known_exception_while_retrieving_namespace_properties() -> None:
        source = with_iceberg_source(processing_threads=2)
        mock_catalog = MockCatalogExceptionRetrievingNamespaceProperties(
            {
                "namespaceA": {
                    "table1": lambda catalog: Table(
                        identifier=("namespaceA", "table1"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceA/table1",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceA/table1",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    )
                },
                "no_such_namespace": {},
                "rest_error": {},
                "namespaceB": {
                    "table2": lambda catalog: Table(
                        identifier=("namespaceB", "table2"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceB/table2",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceB/table2",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                    "table3": lambda catalog: Table(
                        identifier=("namespaceB", "table3"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceB/table3",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceB/table3",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                },
                "namespaceC": {
                    "table4": lambda catalog: Table(
                        identifier=("namespaceC", "table4"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceC/table4",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceC/table4",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    )
                },
                "namespaceD": {
                    "table5": lambda catalog: Table(
                        identifier=("namespaceD", "table5"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceA/table5",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceA/table5",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    )
                },
            }
        )
        with patch(
            "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog"
        ) as get_catalog:
            get_catalog.return_value = mock_catalog
            wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()]
            # ingested 5 tables (6 MCPs each) and 4 namespaces (4 MCPs each), we will not ingest namespaces at all if we fail to get their properties
            expected_wu_urns = [
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)",
            ] * MCPS_PER_TABLE + [
                "urn:li:container:390e031441265aae5b7b7ae8d51b0c1f",
                "urn:li:container:74727446a56420d80ff3b1abf2a18087",
                "urn:li:container:3f9a24213cca64ab22e409d1b9a94789",
                "urn:li:container:38a0583b0305ec5066cb708199f6848c",
            ] * MCPS_PER_NAMESPACE
>           assert len(wu) == len(expected_wu_urns)
E           AssertionError: assert 20 == 55
E            +  where 20 = len([MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123596, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceA', 'qualifiedName': 'namespaceA', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123596, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123597, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123597, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123597, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123597, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceB', 'qualifiedName': 'namespaceB', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123597, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123597, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123597, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123598, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123598, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceC', 'qualifiedName': 'namespaceC', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123598, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123598, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123598, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123598, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123598, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceD', 'qualifiedName': 'namespaceD', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123598, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123598, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123598, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929123598, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True)])
E            +  and   55 = len(['urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c'])

tests/unit/test_iceberg.py:728: AssertionError
tests.unit.test_iceberg::test_unknown_exception_while_retrieving_namespace_properties
Stack Traces | 0.208s run time 
def test_unknown_exception_while_retrieving_namespace_properties() -> None:
        source = with_iceberg_source(processing_threads=2)
        mock_catalog = MockCatalogExceptionRetrievingNamespaceProperties(
            {
                "namespaceA": {
                    "table1": lambda catalog: Table(
                        identifier=("namespaceA", "table1"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceA/table1",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceA/table1",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    )
                },
                "generic_exception": {},
                "namespaceB": {
                    "table2": lambda catalog: Table(
                        identifier=("namespaceB", "table2"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceB/table2",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceB/table2",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                    "table3": lambda catalog: Table(
                        identifier=("namespaceB", "table3"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceB/table3",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceB/table3",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                },
                "namespaceC": {
                    "table4": lambda catalog: Table(
                        identifier=("namespaceC", "table4"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceC/table4",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceC/table4",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    )
                },
                "namespaceD": {
                    "table5": lambda catalog: Table(
                        identifier=("namespaceD", "table5"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceA/table5",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceA/table5",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    )
                },
            }
        )
        with patch(
            "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog"
        ) as get_catalog:
            get_catalog.return_value = mock_catalog
            wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()]
            # ingested 5 tables (6 MCPs each) and 4 namespaces (4 MCPs each), despite exception
            expected_wu_urns = [
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)",
            ] * MCPS_PER_TABLE + [
                "urn:li:container:390e031441265aae5b7b7ae8d51b0c1f",
                "urn:li:container:74727446a56420d80ff3b1abf2a18087",
                "urn:li:container:3f9a24213cca64ab22e409d1b9a94789",
                "urn:li:container:38a0583b0305ec5066cb708199f6848c",
            ] * MCPS_PER_NAMESPACE
>           assert len(wu) == len(expected_wu_urns)
E           AssertionError: assert 20 == 55
E            +  where 20 = len([MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124158, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceA', 'qualifiedName': 'namespaceA', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124158, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124158, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124158, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124158, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124159, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceB', 'qualifiedName': 'namespaceB', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124159, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124159, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124159, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124159, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124159, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceC', 'qualifiedName': 'namespaceC', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124159, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124159, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124159, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124159, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124159, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceD', 'qualifiedName': 'namespaceD', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124159, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124159, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124159, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929124159, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True)])
E            +  and   55 = len(['urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c'])

tests/unit/test_iceberg.py:835: AssertionError
tests.unit.test_iceberg::test_unknown_exception_while_listing_tables
Stack Traces | 0.209s run time 
def test_unknown_exception_while_listing_tables() -> None:
        source = with_iceberg_source(processing_threads=2)
        mock_catalog = MockCatalogExceptionListingTables(
            {
                "namespaceA": {
                    "table1": lambda catalog: Table(
                        identifier=("namespaceA", "table1"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceA/table1",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceA/table1",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    )
                },
                "generic_exception": {},
                "namespaceB": {
                    "table2": lambda catalog: Table(
                        identifier=("namespaceB", "table2"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceB/table2",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceB/table2",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                    "table3": lambda catalog: Table(
                        identifier=("namespaceB", "table3"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceB/table3",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceB/table3",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    ),
                },
                "namespaceC": {
                    "table4": lambda catalog: Table(
                        identifier=("namespaceC", "table4"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceC/table4",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceC/table4",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    )
                },
                "namespaceD": {
                    "table5": lambda catalog: Table(
                        identifier=("namespaceD", "table5"),
                        metadata=TableMetadataV2(
                            partition_specs=[PartitionSpec(spec_id=0)],
                            location="s3:.../abcdefg/namespaceA/table5",
                            last_column_id=0,
                            schemas=[Schema(schema_id=0)],
                        ),
                        metadata_location="s3:.../abcdefg/namespaceA/table5",
                        io=PyArrowFileIO(),
                        catalog=catalog,
                    )
                },
            }
        )
        with patch(
            "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog"
        ) as get_catalog:
            get_catalog.return_value = mock_catalog
            wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()]
            # ingested 5 tables (6 MCPs each) and 5 namespaces (4 MCPs each), despite exception
            expected_wu_urns = [
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)",
            ] * MCPS_PER_TABLE + [
                "urn:li:container:390e031441265aae5b7b7ae8d51b0c1f",
                "urn:li:container:be99158f9640329f4394315e1d8dacf3",
                "urn:li:container:74727446a56420d80ff3b1abf2a18087",
                "urn:li:container:3f9a24213cca64ab22e409d1b9a94789",
                "urn:li:container:38a0583b0305ec5066cb708199f6848c",
            ] * MCPS_PER_NAMESPACE
>           assert len(wu) == len(expected_wu_urns)
E           AssertionError: assert 25 == 60
E            +  where 25 = len([MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121531, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceA', 'qualifiedName': 'namespaceA', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121531, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121532, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121532, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121532, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:be99158f9640329f4394315e1d8dacf3-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:be99158f9640329f4394315e1d8dacf3', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121532, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:be99158f9640329f4394315e1d8dacf3-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:be99158f9640329f4394315e1d8dacf3', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'generic_exception', 'qualifiedName': 'generic_exception', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121532, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:be99158f9640329f4394315e1d8dacf3-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:be99158f9640329f4394315e1d8dacf3', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121532, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:be99158f9640329f4394315e1d8dacf3-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:be99158f9640329f4394315e1d8dacf3', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121532, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:be99158f9640329f4394315e1d8dacf3-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:be99158f9640329f4394315e1d8dacf3', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121532, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121532, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceB', 'qualifiedName': 'namespaceB', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121532, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121532, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121532, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:74727446a56420d80ff3b1abf2a18087-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:74727446a56420d80ff3b1abf2a18087', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121532, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121532, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceC', 'qualifiedName': 'namespaceC', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121532, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121532, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121532, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:3f9a24213cca64ab22e409d1b9a94789-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:3f9a24213cca64ab22e409d1b9a94789', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121533, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-status', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='status', aspect=StatusClass({'removed': False}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121533, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-containerProperties', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='containerProperties', aspect=ContainerPropertiesClass({'customProperties': {}, 'externalUrl': None, 'name': 'namespaceD', 'qualifiedName': 'namespaceD', 'description': None, 'env': 'PROD', 'created': None, 'lastModified': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121533, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-subTypes', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='subTypes', aspect=SubTypesClass({'typeNames': [<DatasetContainerSubTypes.NAMESPACE: 'Namespace'>]}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121533, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-dataPlatformInstance', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='dataPlatformInstance', aspect=DataPlatformInstanceClass({'platform': 'urn:li:dataPlatform:iceberg', 'instance': None}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121533, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True), MetadataWorkUnit(id='urn:li:container:38a0583b0305ec5066cb708199f6848c-browsePathsV2', metadata=MetadataChangeProposalWrapper(entityType='container', changeType='UPSERT', entityUrn='urn:li:container:38a0583b0305ec5066cb708199f6848c', entityKeyAspect=None, auditHeader=None, aspectName='browsePathsV2', aspect=BrowsePathsV2Class({'path': []}), systemMetadata=SystemMetadataClass({'lastObserved': 1759929121533, 'runId': 'iceberg-source-test', 'lastRunId': 'no-run-id-provided', 'pipelineName': None, 'registryName': None, 'registryVersion': None, 'properties': None, 'version': None, 'aspectCreated': None, 'aspectModified': None}), headers=None), treat_errors_as_warnings=False, is_primary_source=True)])
E            +  and   60 = len(['urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:be99158f9640329f4394315e1d8dacf3', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:be99158f9640329f4394315e1d8dacf3', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:be99158f9640329f4394315e1d8dacf3', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:be99158f9640329f4394315e1d8dacf3', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c', 'urn:li:container:390e031441265aae5b7b7ae8d51b0c1f', 'urn:li:container:be99158f9640329f4394315e1d8dacf3', 'urn:li:container:74727446a56420d80ff3b1abf2a18087', 'urn:li:container:3f9a24213cca64ab22e409d1b9a94789', 'urn:li:container:38a0583b0305ec5066cb708199f6848c'])

tests/unit/test_iceberg.py:1053: AssertionError
tests.integration.iceberg.test_iceberg::test_iceberg_ingest
Stack Traces | 35.9s run time 
docker_compose_runner = <function docker_compose_runner.<locals>.run at 0x7fe82e0d99e0>
pytestconfig = <_pytest.config.Config object at 0x7fe90d137fd0>
tmp_path = PosixPath('.../pytest-of-runner/pytest-0/test_iceberg_ingest0')
mock_time = None

    @freeze_time(FROZEN_TIME)
    def test_iceberg_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time):
        test_resources_dir = pytestconfig.rootpath / "tests/integration/iceberg/"
    
        with docker_compose_runner(
            test_resources_dir / "docker-compose.yml", "iceberg"
        ) as docker_services:
            wait_for_port(docker_services, "spark-iceberg", 8888, timeout=120)
    
            # Run the create.py pyspark file to populate the table.
            spark_submit(".../iceberg/setup/create.py", "nyc.taxis")
    
            # Run the metadata ingestion pipeline.
            config_file = (test_resources_dir / "iceberg_to_file.yml").resolve()
>           run_datahub_cmd(
                ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path
            )

.../integration/iceberg/test_iceberg.py:91: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_helpers/click_helpers.py:63: in run_datahub_cmd
    assert_result_ok(result)
tests/test_helpers/click_helpers.py:17: in assert_result_ok
    raise result.exception
venv/lib/python3.11.../site-packages/click/testing.py:504: in invoke
    return_value = cli.main(args=args or (), prog_name=prog_name, **extra)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
venv/lib/python3.11.............../site-packages/click/core.py:1383: in main
    rv = self.invoke(ctx)
         ^^^^^^^^^^^^^^^^
venv/lib/python3.11.............../site-packages/click/core.py:1850: in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
venv/lib/python3.11.............../site-packages/click/core.py:1850: in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
venv/lib/python3.11.............../site-packages/click/core.py:1246: in invoke
    return ctx.invoke(self.callback, **ctx.params)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
venv/lib/python3.11.............../site-packages/click/core.py:814: in invoke
    return callback(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^
.../datahub/telemetry/telemetry.py:464: in wrapper
    raise e
.../datahub/telemetry/telemetry.py:433: in wrapper
    res = func(*args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^
.../datahub/telemetry/telemetry.py:464: in wrapper
    raise e
.../datahub/telemetry/telemetry.py:433: in wrapper
    res = func(*args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^
.../datahub/upgrade/upgrade.py:491: in async_wrapper
    ret = func(*args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

config = '.../integration/iceberg/iceberg_to_file.yml'
dry_run = False, preview = False, strict_warnings = True, preview_workunits = 10
test_source_connection = False, report_to = 'datahub', no_default_report = False
no_spinner = False, no_progress = False

    @ingest.command()
    @click.option(
        "-c",
        "--config",
        type=click.Path(dir_okay=False),
        help="Config file in .toml or .yaml format.",
        required=True,
    )
    @click.option(
        "-n",
        "--dry-run",
        type=bool,
        is_flag=True,
        default=False,
        help="Perform a dry run of the ingestion, essentially skipping writing to sink.",
    )
    @click.option(
        "--preview",
        type=bool,
        is_flag=True,
        default=False,
        help="Perform limited ingestion from the source to the sink to get a quick preview.",
    )
    @click.option(
        "--preview-workunits",
        type=int,
        default=10,
        help="The number of workunits to produce for preview.",
    )
    @click.option(
        "--strict-warnings/--no-strict-warnings",
        default=False,
        help="If enabled, ingestion runs with warnings will yield a non-zero error code",
    )
    @click.option(
        "--test-source-connection",
        type=bool,
        is_flag=True,
        default=False,
        help="When set, ingestion will only test the source connection details from the recipe",
    )
    @click.option(
        "--report-to",
        type=str,
        default="datahub",
        help="Provide an destination to send a structured report from the run. The default is 'datahub' and sends the report directly to the datahub server (using the sink configured in your recipe). Use the --no-default-report flag to turn off this default feature. Any other parameter passed to this argument is currently assumed to be a file that you want to write the report to. Supplements the reporting configuration in the recipe",
    )
    @click.option(
        "--no-default-report",
        type=bool,
        is_flag=True,
        default=False,
        help="Turn off default reporting of ingestion results to DataHub",
    )
    @click.option(
        "--no-spinner", type=bool, is_flag=True, default=False, help="Turn off spinner"
    )
    @click.option(
        "--no-progress",
        type=bool,
        is_flag=True,
        default=False,
        help="If enabled, mute intermediate progress ingestion reports",
    )
    @telemetry.with_telemetry(
        capture_kwargs=[
            "dry_run",
            "preview",
            "strict_warnings",
            "test_source_connection",
            "no_default_report",
            "no_spinner",
            "no_progress",
        ]
    )
    @upgrade.check_upgrade
    def run(
        config: str,
        dry_run: bool,
        preview: bool,
        strict_warnings: bool,
        preview_workunits: int,
        test_source_connection: bool,
        report_to: Optional[str],
        no_default_report: bool,
        no_spinner: bool,
        no_progress: bool,
    ) -> None:
        """Ingest metadata into DataHub."""
    
        def run_pipeline_to_completion(pipeline: Pipeline) -> int:
            logger.info("Starting metadata ingestion")
            with click_spinner.spinner(disable=no_spinner or no_progress):
                try:
                    pipeline.run()
                except Exception as e:
                    logger.info(
                        f"Source ({pipeline.source_type}) report:\n{pipeline.source.get_report().as_string()}"
                    )
                    logger.info(
                        f"Sink ({pipeline.sink_type}) report:\n{pipeline.sink.get_report().as_string()}"
                    )
                    raise e
                else:
                    logger.info("Finished metadata ingestion")
                    pipeline.log_ingestion_stats()
                    ret = pipeline.pretty_print_summary(warnings_as_failure=strict_warnings)
                    return ret
    
        # main function begins
        logger.info("DataHub CLI version: %s", nice_version_name())
    
        pipeline_config = load_config_file(
            config,
            squirrel_original_config=True,
            squirrel_field="__raw_config",
            allow_stdin=True,
            allow_remote=True,
            process_directives=True,
            resolve_env_vars=True,
        )
        raw_pipeline_config = pipeline_config.pop("__raw_config")
    
        if test_source_connection:
            sys.exit(_test_source_connection(report_to, pipeline_config))
    
        if no_default_report:
            # The default is "datahub" reporting. The extra flag will disable it.
            report_to = None
    
        # logger.debug(f"Using config: {pipeline_config}")
        pipeline = Pipeline.create(
            pipeline_config,
            dry_run=dry_run,
            preview_mode=preview,
            preview_workunits=preview_workunits,
            report_to=report_to,
            no_progress=no_progress,
            raw_config=raw_pipeline_config,
        )
        ret = run_pipeline_to_completion(pipeline)
    
        if ret:
>           sys.exit(ret)
E           SystemExit: 1

.../datahub/cli/ingest_cli.py:184: SystemExit
tests.integration.iceberg.test_iceberg::test_iceberg_profiling
Stack Traces | 37s run time 
docker_compose_runner = <function docker_compose_runner.<locals>.run at 0x7fe82e0d99e0>
pytestconfig = <_pytest.config.Config object at 0x7fe90d137fd0>
tmp_path = PosixPath('.../pytest-of-runner/pytest-0/test_iceberg_profiling0')
mock_time = None

    @freeze_time(FROZEN_TIME)
    def test_iceberg_profiling(docker_compose_runner, pytestconfig, tmp_path, mock_time):
        test_resources_dir = pytestconfig.rootpath / "tests/integration/iceberg/"
    
        with docker_compose_runner(
            test_resources_dir / "docker-compose.yml", "iceberg"
        ) as docker_services:
            wait_for_port(docker_services, "spark-iceberg", 8888, timeout=120)
    
            # Run the create.py pyspark file to populate the table.
            spark_submit(".../iceberg/setup/create.py", "nyc.taxis")
    
            # Run the metadata ingestion pipeline.
            config_file = (test_resources_dir / "iceberg_profile_to_file.yml").resolve()
>           run_datahub_cmd(
                ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path
            )

.../integration/iceberg/test_iceberg.py:232: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_helpers/click_helpers.py:63: in run_datahub_cmd
    assert_result_ok(result)
tests/test_helpers/click_helpers.py:17: in assert_result_ok
    raise result.exception
venv/lib/python3.11.../site-packages/click/testing.py:504: in invoke
    return_value = cli.main(args=args or (), prog_name=prog_name, **extra)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
venv/lib/python3.11.............../site-packages/click/core.py:1383: in main
    rv = self.invoke(ctx)
         ^^^^^^^^^^^^^^^^
venv/lib/python3.11.............../site-packages/click/core.py:1850: in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
venv/lib/python3.11.............../site-packages/click/core.py:1850: in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
venv/lib/python3.11.............../site-packages/click/core.py:1246: in invoke
    return ctx.invoke(self.callback, **ctx.params)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
venv/lib/python3.11.............../site-packages/click/core.py:814: in invoke
    return callback(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^
.../datahub/telemetry/telemetry.py:464: in wrapper
    raise e
.../datahub/telemetry/telemetry.py:433: in wrapper
    res = func(*args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^
.../datahub/telemetry/telemetry.py:464: in wrapper
    raise e
.../datahub/telemetry/telemetry.py:433: in wrapper
    res = func(*args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^
.../datahub/upgrade/upgrade.py:491: in async_wrapper
    ret = func(*args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

config = '.../integration/iceberg/iceberg_profile_to_file.yml'
dry_run = False, preview = False, strict_warnings = True, preview_workunits = 10
test_source_connection = False, report_to = 'datahub', no_default_report = False
no_spinner = False, no_progress = False

    @ingest.command()
    @click.option(
        "-c",
        "--config",
        type=click.Path(dir_okay=False),
        help="Config file in .toml or .yaml format.",
        required=True,
    )
    @click.option(
        "-n",
        "--dry-run",
        type=bool,
        is_flag=True,
        default=False,
        help="Perform a dry run of the ingestion, essentially skipping writing to sink.",
    )
    @click.option(
        "--preview",
        type=bool,
        is_flag=True,
        default=False,
        help="Perform limited ingestion from the source to the sink to get a quick preview.",
    )
    @click.option(
        "--preview-workunits",
        type=int,
        default=10,
        help="The number of workunits to produce for preview.",
    )
    @click.option(
        "--strict-warnings/--no-strict-warnings",
        default=False,
        help="If enabled, ingestion runs with warnings will yield a non-zero error code",
    )
    @click.option(
        "--test-source-connection",
        type=bool,
        is_flag=True,
        default=False,
        help="When set, ingestion will only test the source connection details from the recipe",
    )
    @click.option(
        "--report-to",
        type=str,
        default="datahub",
        help="Provide an destination to send a structured report from the run. The default is 'datahub' and sends the report directly to the datahub server (using the sink configured in your recipe). Use the --no-default-report flag to turn off this default feature. Any other parameter passed to this argument is currently assumed to be a file that you want to write the report to. Supplements the reporting configuration in the recipe",
    )
    @click.option(
        "--no-default-report",
        type=bool,
        is_flag=True,
        default=False,
        help="Turn off default reporting of ingestion results to DataHub",
    )
    @click.option(
        "--no-spinner", type=bool, is_flag=True, default=False, help="Turn off spinner"
    )
    @click.option(
        "--no-progress",
        type=bool,
        is_flag=True,
        default=False,
        help="If enabled, mute intermediate progress ingestion reports",
    )
    @telemetry.with_telemetry(
        capture_kwargs=[
            "dry_run",
            "preview",
            "strict_warnings",
            "test_source_connection",
            "no_default_report",
            "no_spinner",
            "no_progress",
        ]
    )
    @upgrade.check_upgrade
    def run(
        config: str,
        dry_run: bool,
        preview: bool,
        strict_warnings: bool,
        preview_workunits: int,
        test_source_connection: bool,
        report_to: Optional[str],
        no_default_report: bool,
        no_spinner: bool,
        no_progress: bool,
    ) -> None:
        """Ingest metadata into DataHub."""
    
        def run_pipeline_to_completion(pipeline: Pipeline) -> int:
            logger.info("Starting metadata ingestion")
            with click_spinner.spinner(disable=no_spinner or no_progress):
                try:
                    pipeline.run()
                except Exception as e:
                    logger.info(
                        f"Source ({pipeline.source_type}) report:\n{pipeline.source.get_report().as_string()}"
                    )
                    logger.info(
                        f"Sink ({pipeline.sink_type}) report:\n{pipeline.sink.get_report().as_string()}"
                    )
                    raise e
                else:
                    logger.info("Finished metadata ingestion")
                    pipeline.log_ingestion_stats()
                    ret = pipeline.pretty_print_summary(warnings_as_failure=strict_warnings)
                    return ret
    
        # main function begins
        logger.info("DataHub CLI version: %s", nice_version_name())
    
        pipeline_config = load_config_file(
            config,
            squirrel_original_config=True,
            squirrel_field="__raw_config",
            allow_stdin=True,
            allow_remote=True,
            process_directives=True,
            resolve_env_vars=True,
        )
        raw_pipeline_config = pipeline_config.pop("__raw_config")
    
        if test_source_connection:
            sys.exit(_test_source_connection(report_to, pipeline_config))
    
        if no_default_report:
            # The default is "datahub" reporting. The extra flag will disable it.
            report_to = None
    
        # logger.debug(f"Using config: {pipeline_config}")
        pipeline = Pipeline.create(
            pipeline_config,
            dry_run=dry_run,
            preview_mode=preview,
            preview_workunits=preview_workunits,
            report_to=report_to,
            no_progress=no_progress,
            raw_config=raw_pipeline_config,
        )
        ret = run_pipeline_to_completion(pipeline)
    
        if ret:
>           sys.exit(ret)
E           SystemExit: 1

.../datahub/cli/ingest_cli.py:184: SystemExit
tests.integration.iceberg.test_iceberg::test_iceberg_stateful_ingest
Stack Traces | 45.7s run time 
docker_compose_runner = <function docker_compose_runner.<locals>.run at 0x7fe82e0d99e0>
pytestconfig = <_pytest.config.Config object at 0x7fe90d137fd0>
tmp_path = PosixPath('.../pytest-of-runner/pytest-0/test_iceberg_stateful_ingest0')
mock_time = None
mock_datahub_graph = <MagicMock spec='DataHubGraph' id='140635144495760'>

    @freeze_time(FROZEN_TIME)
    def test_iceberg_stateful_ingest(
        docker_compose_runner, pytestconfig, tmp_path, mock_time, mock_datahub_graph
    ):
        test_resources_dir = pytestconfig.rootpath / "tests/integration/iceberg"
        platform_instance = "test_platform_instance"
    
        pipeline_config_dict: Dict[str, Any] = {
            "source": {
                "type": "iceberg",
                "config": {
                    "catalog": {
                        "default": {
                            "type": "rest",
                            "uri": "http://localhost:8181",
                            "s3.access-key-id": "admin",
                            "s3.secret-access-key": "password",
                            "s3.region": "us-east-1",
                            "warehouse": "s3a:.../warehouse/wh/",
                            "s3.endpoint": "http://localhost:9000",
                        },
                    },
                    "user_ownership_property": "owner",
                    "group_ownership_property": "owner",
                    "platform_instance": f"{platform_instance}",
                    # enable stateful ingestion
                    "stateful_ingestion": {
                        "enabled": True,
                        "remove_stale_metadata": True,
                        "fail_safe_threshold": 100.0,
                        "state_provider": {
                            "type": "datahub",
                            "config": {"datahub_api": {"server": GMS_SERVER}},
                        },
                    },
                },
            },
            "sink": {
                # we are not really interested in the resulting events for this test
                "type": "console"
            },
            "pipeline_name": "test_pipeline",
        }
    
        with (
            docker_compose_runner(
                test_resources_dir / "docker-compose.yml", "iceberg"
            ) as docker_services,
            patch(
                "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph",
                mock_datahub_graph,
            ) as mock_checkpoint,
        ):
            wait_for_port(docker_services, "spark-iceberg", 8888, timeout=120)
    
            # Run the create.py pyspark file to populate two tables.
            spark_submit(".../iceberg/setup/create.py", "nyc.taxis")
            spark_submit(".../iceberg/setup/create.py", "nyc.another_taxis")
    
            # Both checkpoint and reporting will use the same mocked graph instance.
            mock_checkpoint.return_value = mock_datahub_graph
    
            # Do the first run of the pipeline and get the default job's checkpoint.
>           pipeline_run1 = run_and_get_pipeline(pipeline_config_dict)
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

.../integration/iceberg/test_iceberg.py:166: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_helpers/state_helpers.py:45: in run_and_get_pipeline
    pipeline.raise_from_status()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <datahub.ingestion.run.pipeline.Pipeline object at 0x7fe82ba57cd0>
raise_warnings = False

    def raise_from_status(self, raise_warnings: bool = False) -> None:
        if self.source.get_report().failures:
>           raise PipelineExecutionError(
                "Source reported errors", self.source.get_report().failures
            )
E           datahub.configuration.common.PipelineExecutionError: ('Source reported errors', [{'title': 'Error when processing a table', 'message': 'Skipping the table due to errors when processing it.', 'context': ["('nyc', 'another_taxis') <class 'AttributeError'>: 'pydantic_core._pydantic_core.ValidationInfo' object has no attribute 'current_snapshot_id'", "('nyc', 'taxis') <class 'AttributeError'>: 'pydantic_core._pydantic_core.ValidationInfo' object has no attribute 'current_snapshot_id'"]}])

.../ingestion/run/pipeline.py:621: PipelineExecutionError
tests.integration.iceberg.test_iceberg::test_multiprocessing_iceberg_ingest
Stack Traces | 76.5s run time 
docker_compose_runner = <function docker_compose_runner.<locals>.run at 0x7fe82e0d99e0>
pytestconfig = <_pytest.config.Config object at 0x7fe90d137fd0>
tmp_path = PosixPath('.../pytest-of-runner/pytest-0/test_multiprocessing_iceberg_i0')
mock_time = None

    @freeze_time(FROZEN_TIME)
    def test_multiprocessing_iceberg_ingest(
        docker_compose_runner, pytestconfig, tmp_path, mock_time
    ):
        test_resources_dir = pytestconfig.rootpath / "tests/integration/iceberg/"
    
        with docker_compose_runner(
            test_resources_dir / "docker-compose.yml", "iceberg"
        ) as docker_services:
            wait_for_port(docker_services, "spark-iceberg", 8888, timeout=120)
    
            # Run the create.py pyspark file to populate the table.
            spark_submit(".../iceberg/setup/create.py", "nyc.taxis")
    
            # Run the metadata ingestion pipeline.
            config_file = (
                test_resources_dir / "iceberg_multiprocessing_to_file.yml"
            ).resolve()
>           run_datahub_cmd(
                ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path
            )

.../integration/iceberg/test_iceberg.py:65: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_helpers/click_helpers.py:63: in run_datahub_cmd
    assert_result_ok(result)
tests/test_helpers/click_helpers.py:17: in assert_result_ok
    raise result.exception
venv/lib/python3.11.../site-packages/click/testing.py:504: in invoke
    return_value = cli.main(args=args or (), prog_name=prog_name, **extra)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
venv/lib/python3.11.............../site-packages/click/core.py:1383: in main
    rv = self.invoke(ctx)
         ^^^^^^^^^^^^^^^^
venv/lib/python3.11.............../site-packages/click/core.py:1850: in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
venv/lib/python3.11.............../site-packages/click/core.py:1850: in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
venv/lib/python3.11.............../site-packages/click/core.py:1246: in invoke
    return ctx.invoke(self.callback, **ctx.params)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
venv/lib/python3.11.............../site-packages/click/core.py:814: in invoke
    return callback(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^
.../datahub/telemetry/telemetry.py:464: in wrapper
    raise e
.../datahub/telemetry/telemetry.py:433: in wrapper
    res = func(*args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^
.../datahub/telemetry/telemetry.py:464: in wrapper
    raise e
.../datahub/telemetry/telemetry.py:433: in wrapper
    res = func(*args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^
.../datahub/upgrade/upgrade.py:491: in async_wrapper
    ret = func(*args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

config = '.../integration/iceberg/iceberg_multiprocessing_to_file.yml'
dry_run = False, preview = False, strict_warnings = True, preview_workunits = 10
test_source_connection = False, report_to = 'datahub', no_default_report = False
no_spinner = False, no_progress = False

    @ingest.command()
    @click.option(
        "-c",
        "--config",
        type=click.Path(dir_okay=False),
        help="Config file in .toml or .yaml format.",
        required=True,
    )
    @click.option(
        "-n",
        "--dry-run",
        type=bool,
        is_flag=True,
        default=False,
        help="Perform a dry run of the ingestion, essentially skipping writing to sink.",
    )
    @click.option(
        "--preview",
        type=bool,
        is_flag=True,
        default=False,
        help="Perform limited ingestion from the source to the sink to get a quick preview.",
    )
    @click.option(
        "--preview-workunits",
        type=int,
        default=10,
        help="The number of workunits to produce for preview.",
    )
    @click.option(
        "--strict-warnings/--no-strict-warnings",
        default=False,
        help="If enabled, ingestion runs with warnings will yield a non-zero error code",
    )
    @click.option(
        "--test-source-connection",
        type=bool,
        is_flag=True,
        default=False,
        help="When set, ingestion will only test the source connection details from the recipe",
    )
    @click.option(
        "--report-to",
        type=str,
        default="datahub",
        help="Provide an destination to send a structured report from the run. The default is 'datahub' and sends the report directly to the datahub server (using the sink configured in your recipe). Use the --no-default-report flag to turn off this default feature. Any other parameter passed to this argument is currently assumed to be a file that you want to write the report to. Supplements the reporting configuration in the recipe",
    )
    @click.option(
        "--no-default-report",
        type=bool,
        is_flag=True,
        default=False,
        help="Turn off default reporting of ingestion results to DataHub",
    )
    @click.option(
        "--no-spinner", type=bool, is_flag=True, default=False, help="Turn off spinner"
    )
    @click.option(
        "--no-progress",
        type=bool,
        is_flag=True,
        default=False,
        help="If enabled, mute intermediate progress ingestion reports",
    )
    @telemetry.with_telemetry(
        capture_kwargs=[
            "dry_run",
            "preview",
            "strict_warnings",
            "test_source_connection",
            "no_default_report",
            "no_spinner",
            "no_progress",
        ]
    )
    @upgrade.check_upgrade
    def run(
        config: str,
        dry_run: bool,
        preview: bool,
        strict_warnings: bool,
        preview_workunits: int,
        test_source_connection: bool,
        report_to: Optional[str],
        no_default_report: bool,
        no_spinner: bool,
        no_progress: bool,
    ) -> None:
        """Ingest metadata into DataHub."""
    
        def run_pipeline_to_completion(pipeline: Pipeline) -> int:
            logger.info("Starting metadata ingestion")
            with click_spinner.spinner(disable=no_spinner or no_progress):
                try:
                    pipeline.run()
                except Exception as e:
                    logger.info(
                        f"Source ({pipeline.source_type}) report:\n{pipeline.source.get_report().as_string()}"
                    )
                    logger.info(
                        f"Sink ({pipeline.sink_type}) report:\n{pipeline.sink.get_report().as_string()}"
                    )
                    raise e
                else:
                    logger.info("Finished metadata ingestion")
                    pipeline.log_ingestion_stats()
                    ret = pipeline.pretty_print_summary(warnings_as_failure=strict_warnings)
                    return ret
    
        # main function begins
        logger.info("DataHub CLI version: %s", nice_version_name())
    
        pipeline_config = load_config_file(
            config,
            squirrel_original_config=True,
            squirrel_field="__raw_config",
            allow_stdin=True,
            allow_remote=True,
            process_directives=True,
            resolve_env_vars=True,
        )
        raw_pipeline_config = pipeline_config.pop("__raw_config")
    
        if test_source_connection:
            sys.exit(_test_source_connection(report_to, pipeline_config))
    
        if no_default_report:
            # The default is "datahub" reporting. The extra flag will disable it.
            report_to = None
    
        # logger.debug(f"Using config: {pipeline_config}")
        pipeline = Pipeline.create(
            pipeline_config,
            dry_run=dry_run,
            preview_mode=preview,
            preview_workunits=preview_workunits,
            report_to=report_to,
            no_progress=no_progress,
            raw_config=raw_pipeline_config,
        )
        ret = run_pipeline_to_completion(pipeline)
    
        if ret:
>           sys.exit(ret)
E           SystemExit: 1

.../datahub/cli/ingest_cli.py:184: SystemExit

To view more test analytics, go to the Test Analytics Dashboard
📋 Got 3 mins? Take this short survey to help us improve Test Analytics.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

No reviews

Assignees

No one assigned

Labels

ingestion PR or Issue related to the ingestion of metadata

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

1 participant

@sgomezvillamor