fix: update DeltaTableDataset.py updating the from data catalog metho… (#1202)

* fix: update DeltaTableDataset.py updating the from data catalog method for deltalake v1.0.0

Signed-off-by: SRIKAR-8-77 <[email protected]>

* testing

Signed-off-by: SRIKAR-8-77 <[email protected]>

* Fix-Signing_off_DCO

Signed-off-by: SRIKAR-8-77 <[email protected]>

* Unpin deltalake

Signed-off-by: Ankita Katiyar <[email protected]>

* lint

Signed-off-by: Ankita Katiyar <[email protected]>

* Tests

Signed-off-by: Ankita Katiyar <[email protected]>

* Update release notes

Signed-off-by: Ankita Katiyar <[email protected]>

* Fix release notes

Signed-off-by: Merel Theisen <[email protected]>

---------

Signed-off-by: SRIKAR-8-77 <[email protected]>
Signed-off-by: Ankita Katiyar <[email protected]>
Signed-off-by: Merel Theisen <[email protected]>
Signed-off-by: Merel Theisen <[email protected]>
Co-authored-by: Sreekar Reddy <[email protected]>
Co-authored-by: Ankita Katiyar <[email protected]>
Co-authored-by: Ankita Katiyar <[email protected]>
Co-authored-by: Merel Theisen <[email protected]>
Co-authored-by: Merel Theisen <[email protected]>

Author: 4 people

kedro-datasets/RELEASE.md

@@ -15,9 +15,12 @@
 | `chromadb.ChromaDBDataset` | A dataset for loading and saving data to ChromaDB vector database collections | `kedro_datasets_experimental.chromadb` |
 
 ## Bug fixes and other changes
+- Updated `pandas.DeltaTableDataset` to be compatible with `deltalake` version 1.x.
+
 ## Community contributions
 
 - [Armand Masseau](https://github.com/armandmasseaugit)
+- [SRIKAR-8-77](https://github.com/SRIKAR-8-77)
 
 # Release 9.0.0
 
@@ -64,7 +67,6 @@
 Many thanks to the following Kedroids for contributing PRs to this release:
 - [Guillaume Tauzin](https://github.com/gtauzin)
 - [gitgud5000](https://github.com/gitgud5000)
-- [Armand Masseau](https://github.com/armandmasseaugit)
 
 # Release 8.1.0
 

kedro-datasets/kedro_datasets/pandas/deltatable_dataset.py

@@ -2,13 +2,14 @@
 S3, GCS), Databricks unity catalog and AWS Glue catalog respectively. It handles
 load and save using a pandas dataframe.
 """
+
 from __future__ import annotations
 
 from copy import deepcopy
 from typing import Any
 
 import pandas as pd
-from deltalake import DataCatalog, DeltaTable, Metadata
+from deltalake import DeltaTable, Metadata
 from deltalake.exceptions import TableNotFoundError
 from deltalake.writer import write_deltalake
 from kedro.io.core import AbstractDataset, DatasetError
@@ -84,7 +85,7 @@ def __init__(  # noqa: PLR0913
         self,
         *,
         filepath: str | None = None,
-        catalog_type: DataCatalog | None = None,
+        catalog_type: str | None = None,
         catalog_name: str | None = None,
         database: str | None = None,
         table: str | None = None,
@@ -169,12 +170,20 @@ def __init__(  # noqa: PLR0913
                 )
             except TableNotFoundError:
                 self.is_empty_dir = True
-        else:
-            self._delta_table = DeltaTable.from_data_catalog(
-                data_catalog=DataCatalog[self._catalog_type],  # type: ignore[misc]
-                data_catalog_id=self._catalog_name,
-                database_name=self._database or "",
-                table_name=self._table or "",
+        elif self._catalog_type:
+            if self._catalog_type.upper() == "AWS":
+                table_uri = f"glue:///{self._database}/{self._table}"
+            elif self._catalog_type.upper() == "UNITY":
+                table_uri = (
+                    f"unity://{self._catalog_name}/{self._database}/{self._table}"
+                )
+            else:
+                raise ValueError(f"Unsupported catalog type: {self._catalog_type}")
+
+            self._delta_table = DeltaTable(
+                table_uri=table_uri,
+                storage_options=self.fs_args,
+                version=self._version,
             )
 
     @property

kedro-datasets/pyproject.toml

@@ -134,7 +134,7 @@ optuna-studydataset = ["optuna"]
 optuna = ["kedro-datasets[optuna-studydataset]"]
 
 pandas-csvdataset = ["kedro-datasets[pandas-base]"]
-pandas-deltatabledataset = ["kedro-datasets[pandas-base]", "deltalake>=0.10.0, <1.0.0"]
+pandas-deltatabledataset = ["kedro-datasets[pandas-base]", "deltalake>=0.10.0"]
 pandas-exceldataset = ["kedro-datasets[pandas-base]", "openpyxl>=3.0.6, <4.0"]
 pandas-featherdataset = ["kedro-datasets[pandas-base]"]
 pandas-gbqtabledataset = ["kedro-datasets[pandas-base]", "pandas-gbq>=0.12.0"]
@@ -174,8 +174,8 @@ plotly-plotlydataset = ["kedro-datasets[pandas-base,plotly-base]"]
 plotly = ["kedro-datasets[plotly-htmldataset,plotly-jsondataset,plotly-plotlydataset]"]
 
 polars-csvdataset = ["kedro-datasets[polars-base]"]
-polars-eagerpolarsdataset = ["kedro-datasets[polars-base]", "pyarrow>=4.0", "xlsx2csv>=0.8.0", "deltalake >= 0.6.2, <1.0.0"]
-polars-lazypolarsdataset = ["kedro-datasets[polars-base]", "pyarrow>=4.0", "deltalake >= 0.6.2, <1.0.0"]
+polars-eagerpolarsdataset = ["kedro-datasets[polars-base]", "pyarrow>=4.0", "xlsx2csv>=0.8.0", "deltalake >= 0.6.2"]
+polars-lazypolarsdataset = ["kedro-datasets[polars-base]", "pyarrow>=4.0", "deltalake >= 0.6.2"]
 polars = [
     """kedro-datasets[polars-csvdataset,\
     polars-eagerpolarsdataset,\
@@ -263,9 +263,9 @@ test = [
     "compress-pickle[lz4]~=2.1.0",
     "coverage>=7.2.0",
     "dask[complete]>=2021.10",
+    "deltalake>=0.10.0",
     "delta-spark>=1.0, <3.0; python_version <= '3.11'",
     "delta-spark>=4.0; python_version >= '3.12'",
-    "deltalake>=0.10.0, <1.0.0",
     "dill~=0.3.1",
     "filelock>=3.4.0, <4.0",
     "fiona >=1.8, <2.0",

kedro-datasets/tests/pandas/test_deltatable_dataset.py

@@ -2,7 +2,7 @@
 
 import pandas as pd
 import pytest
-from deltalake import DataCatalog, Metadata
+from deltalake import Metadata
 from kedro.io.core import DatasetError
 from pandas.testing import assert_frame_equal
 
@@ -121,12 +121,11 @@ def test_from_aws_glue_catalog(self, mocker):
             "kedro_datasets.pandas.deltatable_dataset.DeltaTable"
         )
         _ = DeltaTableDataset(catalog_type="AWS", database="db", table="tbl")
-        mock_delta_table.from_data_catalog.assert_called_once()
-        mock_delta_table.from_data_catalog.assert_called_with(
-            data_catalog=DataCatalog.AWS,
-            data_catalog_id=None,
-            database_name="db",
-            table_name="tbl",
+        mock_delta_table.assert_called_once()
+        mock_delta_table.assert_called_with(
+            table_uri="glue:///db/tbl",
+            storage_options={},
+            version=None,
         )
 
     def test_from_databricks_unity_catalog(self, mocker):
@@ -137,17 +136,16 @@ def test_from_databricks_unity_catalog(self, mocker):
         _ = DeltaTableDataset(
             catalog_type="UNITY", catalog_name="id", database="db", table="tbl"
         )
-        mock_delta_table.from_data_catalog.assert_called_once()
-        mock_delta_table.from_data_catalog.assert_called_with(
-            data_catalog=DataCatalog.UNITY,
-            data_catalog_id="id",
-            database_name="db",
-            table_name="tbl",
+        mock_delta_table.assert_called_once()
+        mock_delta_table.assert_called_with(
+            table_uri="unity://id/db/tbl",
+            storage_options={},
+            version=None,
         )
 
     def test_from_unsupported_catalog(self):
         """Test dataset creation from unsupported catalog."""
-        with pytest.raises(KeyError):
+        with pytest.raises(ValueError):
             DeltaTableDataset(catalog_type="unsupported", database="db", table="tbl")
 
     def test_unsupported_write_mode(self, filepath):