@@ -75,46 +75,46 @@ class ExternalTableDataset(BaseTableDataset):
7575
7676 ### Example usage for the [Python API](https://docs.kedro.org/en/stable/catalog-data/advanced_data_catalog_usage/):
7777
78- ```python
79- from kedro_datasets.databricks import ExternalTableDataset
80- from pyspark.sql import SparkSession
81- from pyspark.sql.types import IntegerType, Row, StringType, StructField, StructType
82- import importlib.metadata
83-
84- DELTA_VERSION = importlib.metadata.version("delta-spark")
85- major_version = int(DELTA_VERSION.split(".")[0])
86- delta_package = (
87- f"io.delta:delta-spark_2.13:{DELTA_VERSION}"
88- if major_version >= 4
89- else f"io.delta:delta-core_2.12:{DELTA_VERSION}"
90- )
91-
92- schema = StructType(
93- [StructField("name", StringType(), True), StructField("age", IntegerType(), True)]
94- )
95- data = [("Alex", 31), ("Bob", 12), ("Clarke", 65), ("Dave", 29)]
96-
97- spark_df = (
98- SparkSession.builder.config("spark.jars.packages", delta_package)
99- .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
100- .config(
101- "spark.sql.catalog.spark_catalog",
102- "org.apache.spark.sql.delta.catalog.DeltaCatalog",
78+ ```python
79+ from kedro_datasets.databricks import ExternalTableDataset
80+ from pyspark.sql import SparkSession
81+ from pyspark.sql.types import IntegerType, Row, StringType, StructField, StructType
82+ import importlib.metadata
83+
84+ DELTA_VERSION = importlib.metadata.version("delta-spark")
85+ major_version = int(DELTA_VERSION.split(".")[0])
86+ delta_package = (
87+ f"io.delta:delta-spark_2.13:{DELTA_VERSION}"
88+ if major_version >= 4
89+ else f"io.delta:delta-core_2.12:{DELTA_VERSION}"
10390 )
104- .getOrCreate()
105- .createDataFrame(data, schema)
106- )
107-
108- dataset = ExternalTableDataset(
109- table="names_and_ages",
110- write_mode="overwrite",
111- location="abfss://[email protected] /depts/cust", 112- )
113-
114- dataset.save(spark_df)
115- reloaded = dataset.load()
116- assert Row(name="Bob", age=12) in reloaded.take(4)
117- ```
91+
92+ schema = StructType(
93+ [StructField("name", StringType(), True), StructField("age", IntegerType(), True)]
94+ )
95+ data = [("Alex", 31), ("Bob", 12), ("Clarke", 65), ("Dave", 29)]
96+
97+ spark_df = (
98+ SparkSession.builder.config("spark.jars.packages", delta_package)
99+ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
100+ .config(
101+ "spark.sql.catalog.spark_catalog",
102+ "org.apache.spark.sql.delta.catalog.DeltaCatalog",
103+ )
104+ .getOrCreate()
105+ .createDataFrame(data, schema)
106+ )
107+
108+ dataset = ExternalTableDataset(
109+ table="names_and_ages",
110+ write_mode="overwrite",
111+ location="abfss://[email protected] /depts/cust", 112+ )
113+
114+ dataset.save(spark_df)
115+ reloaded = dataset.load()
116+ assert Row(name="Bob", age=12) in reloaded.take(4)
117+ ```
118118
119119 """
120120
0 commit comments