Upgrade to DataFusion 53 (#1402)

* Upgrade to DataFusion 53

* Fix fmt

* Fix fmt

* Fix docs

* Bump datafusion rev to 53.0.0

* Bump ffi example datafusion commit to the same as main repo

---------

Co-authored-by: Tim Saucer <timsaucer@gmail.com>

Author: nuno-faria

Cargo.lock

@@ -48,19 +48,19 @@ tokio = { version = "1.49", features = [
   "rt-multi-thread",
   "sync",
 ] }
-pyo3 = { version = "0.26", features = [
+pyo3 = { version = "0.28", features = [
   "extension-module",
   "abi3",
   "abi3-py310",
 ] }
-pyo3-async-runtimes = { version = "0.26", features = ["tokio-runtime"] }
+pyo3-async-runtimes = { version = "0.28", features = ["tokio-runtime"] }
 pyo3-log = "0.13.3"
-arrow = { version = "57", features = ["pyarrow"] }
-arrow-select = { version = "57" }
-datafusion = { version = "52", features = ["avro", "unicode_expressions"] }
-datafusion-substrait = { version = "52", optional = true }
-datafusion-proto = { version = "52" }
-datafusion-ffi = { version = "52" }
+arrow = { version = "58", features = ["pyarrow"] }
+arrow-select = { version = "58" }
+datafusion = { version = "53", features = ["avro", "unicode_expressions"] }
+datafusion-substrait = { version = "53", optional = true }
+datafusion-proto = { version = "53" }
+datafusion-ffi = { version = "53" }
 prost = "0.14.3" # keep in line with `datafusion-substrait`
 serde_json = "1"
 uuid = { version = "1.21", features = ["v4"] }
@@ -70,7 +70,7 @@ mimalloc = { version = "0.1", optional = true, default-features = false, feature
 async-trait = "0.1.89"
 futures = "0.3"
 cstr = "0.2"
-object_store = { version = "0.12.4", features = [
+object_store = { version = "0.13.1", features = [
   "aws",
   "gcp",
   "azure",
@@ -82,7 +82,7 @@ parking_lot = "0.12"
 
 [build-dependencies]
 prost-types = "0.14.3"     # keep in line with `datafusion-substrait`
-pyo3-build-config = "0.26"
+pyo3-build-config = "0.28"
 
 [lib]
 name = "datafusion_python"
@@ -95,4 +95,7 @@ codegen-units = 1
 # We cannot publish to crates.io with any patches in the below section. Developers
 # must remove any entries in this section before creating a release candidate.
 [patch.crates-io]
-# datafusion = { git = "https://github.com/apache/datafusion.git", rev = "6713439497561fa74a94177e5b8632322fb7cea5" }
+datafusion = { git = "https://github.com/apache/datafusion.git", rev = "518560246e87d489eba6d511fa167aa429b06728" }
+datafusion-substrait = { git = "https://github.com/apache/datafusion.git", rev = "518560246e87d489eba6d511fa167aa429b06728" }
+datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "518560246e87d489eba6d511fa167aa429b06728" }
+datafusion-ffi = { git = "https://github.com/apache/datafusion.git", rev = "518560246e87d489eba6d511fa167aa429b06728" }

Cargo.toml

@@ -156,7 +156,7 @@ instead of mutating the container directly:
 
 .. code-block:: rust
 
-    #[pyclass(name = "Config", module = "datafusion", subclass, frozen)]
+    #[pyclass(from_py_object, name = "Config", module = "datafusion", subclass, frozen)]
     #[derive(Clone)]
     pub(crate) struct PyConfig {
         config: Arc<RwLock<ConfigOptions>>,
@@ -170,7 +170,7 @@ existing instance in place:
 
 .. code-block:: rust
 
-    #[pyclass(frozen, name = "SessionContext", module = "datafusion", subclass)]
+    #[pyclass(from_py_object, frozen, name = "SessionContext", module = "datafusion", subclass)]
     #[derive(Clone)]
     pub struct PySessionContext {
         pub ctx: SessionContext,
@@ -186,7 +186,7 @@ field updates:
 
     // TODO: This looks like this needs pyo3 tracking so leaving unfrozen for now
     #[derive(Debug, Clone)]
-    #[pyclass(name = "DataTypeMap", module = "datafusion.common", subclass)]
+    #[pyclass(from_py_object, name = "DataTypeMap", module = "datafusion.common", subclass)]
     pub struct DataTypeMap {
         #[pyo3(get, set)]
         pub arrow_type: PyDataType,
@@ -232,8 +232,11 @@ can then be turned into a ``ForeignTableProvider`` the associated code is:
 
 .. code-block:: rust
 
-    let capsule = capsule.downcast::<PyCapsule>()?;
-    let provider = unsafe { capsule.reference::<FFI_TableProvider>() };
+    let capsule = capsule.cast::<PyCapsule>()?;
+    let data: NonNull<FFI_TableProvider> = capsule
+        .pointer_checked(Some(name))?
+        .cast();
+    let codec = unsafe { data.as_ref() };
 
 By convention the ``datafusion-python`` library expects a Python object that has a
 ``TableProvider`` PyCapsule to have this capsule accessible by calling a function named

docs/source/contributor-guide/ffi.rst

@@ -18,6 +18,27 @@
 Upgrade Guides
 ==============
 
+DataFusion 53.0.0
+-----------------
+
+This version includes an upgraded version of ``pyo3``, which changed the way to extract an FFI
+object. Example:
+
+Before:
+
+.. code-block:: rust
+
+    let codec = unsafe { capsule.reference::<FFI_LogicalExtensionCodec>() };
+
+Now:
+
+.. code-block:: rust
+
+    let data: NonNull<FFI_LogicalExtensionCodec> = capsule
+        .pointer_checked(Some(c_str!("datafusion_logical_extension_codec")))?
+        .cast();
+    let codec = unsafe { data.as_ref() };
+
 DataFusion 52.0.0
 -----------------