Skip to content

Commit 28ed5f5

Browse files
committed
Fix test
Signed-off-by: Merel Theisen <[email protected]>
1 parent 83f09a4 commit 28ed5f5

File tree

1 file changed

+39
-39
lines changed

1 file changed

+39
-39
lines changed

kedro-datasets/kedro_datasets_experimental/databricks/external_table_dataset.py

Lines changed: 39 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -75,46 +75,46 @@ class ExternalTableDataset(BaseTableDataset):
7575
7676
### Example usage for the [Python API](https://docs.kedro.org/en/stable/catalog-data/advanced_data_catalog_usage/):
7777
78-
```python
79-
from kedro_datasets.databricks import ExternalTableDataset
80-
from pyspark.sql import SparkSession
81-
from pyspark.sql.types import IntegerType, Row, StringType, StructField, StructType
82-
import importlib.metadata
83-
84-
DELTA_VERSION = importlib.metadata.version("delta-spark")
85-
major_version = int(DELTA_VERSION.split(".")[0])
86-
delta_package = (
87-
f"io.delta:delta-spark_2.13:{DELTA_VERSION}"
88-
if major_version >= 4
89-
else f"io.delta:delta-core_2.12:{DELTA_VERSION}"
90-
)
91-
92-
schema = StructType(
93-
[StructField("name", StringType(), True), StructField("age", IntegerType(), True)]
94-
)
95-
data = [("Alex", 31), ("Bob", 12), ("Clarke", 65), ("Dave", 29)]
96-
97-
spark_df = (
98-
SparkSession.builder.config("spark.jars.packages", delta_package)
99-
.config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
100-
.config(
101-
"spark.sql.catalog.spark_catalog",
102-
"org.apache.spark.sql.delta.catalog.DeltaCatalog",
78+
```python
79+
from kedro_datasets.databricks import ExternalTableDataset
80+
from pyspark.sql import SparkSession
81+
from pyspark.sql.types import IntegerType, Row, StringType, StructField, StructType
82+
import importlib.metadata
83+
84+
DELTA_VERSION = importlib.metadata.version("delta-spark")
85+
major_version = int(DELTA_VERSION.split(".")[0])
86+
delta_package = (
87+
f"io.delta:delta-spark_2.13:{DELTA_VERSION}"
88+
if major_version >= 4
89+
else f"io.delta:delta-core_2.12:{DELTA_VERSION}"
10390
)
104-
.getOrCreate()
105-
.createDataFrame(data, schema)
106-
)
107-
108-
dataset = ExternalTableDataset(
109-
table="names_and_ages",
110-
write_mode="overwrite",
111-
location="abfss://[email protected]/depts/cust",
112-
)
113-
114-
dataset.save(spark_df)
115-
reloaded = dataset.load()
116-
assert Row(name="Bob", age=12) in reloaded.take(4)
117-
```
91+
92+
schema = StructType(
93+
[StructField("name", StringType(), True), StructField("age", IntegerType(), True)]
94+
)
95+
data = [("Alex", 31), ("Bob", 12), ("Clarke", 65), ("Dave", 29)]
96+
97+
spark_df = (
98+
SparkSession.builder.config("spark.jars.packages", delta_package)
99+
.config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
100+
.config(
101+
"spark.sql.catalog.spark_catalog",
102+
"org.apache.spark.sql.delta.catalog.DeltaCatalog",
103+
)
104+
.getOrCreate()
105+
.createDataFrame(data, schema)
106+
)
107+
108+
dataset = ExternalTableDataset(
109+
table="names_and_ages",
110+
write_mode="overwrite",
111+
location="abfss://[email protected]/depts/cust",
112+
)
113+
114+
dataset.save(spark_df)
115+
reloaded = dataset.load()
116+
assert Row(name="Bob", age=12) in reloaded.take(4)
117+
```
118118
119119
"""
120120

0 commit comments

Comments
 (0)