auto convert pandas to spark dataframe

Signed-off-by: Sajid Alam <[email protected]>

Author: SajidAlamQB

kedro-datasets/kedro_datasets/spark/spark_dataset_v2.py

@@ -261,6 +261,8 @@ def _save(self, data: DataFrame | pd.DataFrame) -> None:
             data: PySpark DataFrame or Pandas DataFrame to save.
                   Pandas DataFrames will be automatically converted to Spark.
         """
+        import pandas as pd  # noqa: PLC0415
+
         spark_session = get_spark_with_remote_support()
 
         # Convert Pandas DataFrame to Spark DataFrame if needed

kedro-datasets/tests/spark/test_spark_dataset_v2.py

@@ -208,6 +208,20 @@ def test_relative_path(self, sample_spark_df):
             finally:
                 os.chdir(original_cwd)
 
+    def test_save_pandas_dataframe(self, tmp_path, sample_pandas_df):
+        """Test saving a Pandas DataFrame directly (auto-converts to Spark)."""
+        filepath = str(tmp_path / "test_pandas.parquet")
+        dataset = SparkDatasetV2(filepath=filepath)
+
+        # Save Pandas DataFrame directly
+        dataset.save(sample_pandas_df)
+        assert Path(filepath).exists()
+
+        # Load and verify
+        loaded_df = dataset.load()
+        assert loaded_df.count() == len(sample_pandas_df)
+        assert set(loaded_df.columns) == set(sample_pandas_df.columns)
+
 
 class TestSparkDatasetV2Schema:
     """Test schema handling in SparkDatasetV2."""