Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
dad6f8c
refactor(sedona-schema): canonical N-D raster schema
james-willis May 4, 2026
d69069a
feat(sedona-raster): N-D trait surface and BandRef::is_2d
james-willis May 4, 2026
5bebc6e
refactor(sedona-raster, sedona-raster-functions, sedona-testing): N-D…
james-willis May 4, 2026
7a1c7e7
feat(raster-gdal): port loader to canonical N-D schema
james-willis May 4, 2026
f8da8e0
fix(raster-gdal): keep Cow::Owned band bytes alive for GDAL MEM dataset
james-willis May 6, 2026
9e396ce
feat(sedona-raster): error on OutDb byte access via BandRef accessors
james-willis May 11, 2026
0825294
refactor(raster): add and adopt RasterMetadata/BandMetadata compatibi…
james-willis May 12, 2026
5b12a07
fix(raster-gdal): restore utils.rs (GDAL loader) and gate source_uri …
james-willis May 12, 2026
3fbce40
refactor(raster): drop RasterRefBandsExt and RasterStructArray lifeti…
james-willis May 12, 2026
af1773b
refactor(raster): restore main's builder + array tests via the shim
james-willis May 12, 2026
c195ed1
refactor(raster): trim leftover cosmetic divergence flagged by review
james-willis May 12, 2026
c37408b
chore: fix codespell hits (implementor → implementer)
james-willis May 12, 2026
fa32b34
fix(raster): address Dewey's review of PR-749
james-willis May 13, 2026
a97def9
refactor(raster): format band view assert via sedona_internal_datafus…
james-willis May 13, 2026
cfca0e6
feat: reintroduce view machinery for non-identity band views
james-willis May 5, 2026
2b4185d
refactor(raster-gdal): restore _owned_band_bytes plumbing for view-ma…
james-willis May 12, 2026
c44c160
refactor(raster): make validate_view and visible_shape_from_view pub(…
james-willis May 13, 2026
f11db86
refactor(raster): dedupe strided copy, lazy-materialize data() via On…
james-willis May 13, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion rust/sedona-raster-functions/src/rs_pixel_functions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ impl SedonaScalarKernel for RsPixelAsCentroid {
let grid_x = (col_x - 1) as f64 + 0.5;
let grid_y = (row_y - 1) as f64 + 0.5;

let affine = AffineMatrix::from_metadata(raster.metadata());
let affine = AffineMatrix::from_metadata(&raster.metadata());
let (wx, wy) = affine.transform(grid_x, grid_y);

write_wkb_point(&mut builder, (wx, wy))
Expand Down
97 changes: 85 additions & 12 deletions rust/sedona-raster-gdal/src/gdal_common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ use sedona_gdal::mem::MemDatasetBuilder;
use sedona_gdal::raster::types::DatasetOptions;
use sedona_gdal::raster::types::GdalDataType;

use std::borrow::Cow;

use sedona_raster::traits::{MetadataRef, RasterMetadata, RasterRef};
use sedona_schema::raster::{BandDataType, StorageType};

Expand Down Expand Up @@ -182,27 +184,35 @@ pub(crate) fn convert_gdal_err(e: GdalError) -> DataFusionError {
DataFusionError::External(Box::new(e))
}

/// This function creates a GDAL dataset backed by the MEM driver that directly
/// references the band data stored in the [RasterRef]. No data copying occurs -
/// the GDAL bands point to the same memory as the data buffer held by [RasterRef].
/// Build a GDAL MEM dataset whose bands point at the bytes held by `raster`.
///
/// Each band's bytes come from `BandRef::contiguous_data()`. When that returns
/// `Cow::Borrowed`, the GDAL band points directly at the StructArray's
/// backing buffer (zero-copy); the caller must keep `raster` alive for the
/// dataset's lifetime. When it returns `Cow::Owned` (e.g. a sliced or
/// permuted view materialized by the reader), the moved `Vec<u8>` is
/// returned alongside the dataset and the caller must keep it alive too.
///
/// # Arguments
/// * `raster` - The RasterRef value
/// * `band_indices` - The indices of the bands to include in the GDAL dataset (1-based)
///
/// # Returns
/// A [`Dataset`] that provides access to the GDAL dataset.
/// A pair `(Dataset, Vec<Vec<u8>>)`. The second element holds any
/// reader-allocated band bytes that GDAL pointers may reference; it must
/// outlive the dataset.
///
/// # Errors
/// Returns an error if:
/// - Any band is N-D (not the legacy `["y","x"]` 2-D shape with identity view)
/// - Any band uses OutDb storage
/// - GDAL driver operations fail
/// - Accessing RasterRef fails
pub unsafe fn raster_ref_to_gdal_mem<R: RasterRef + ?Sized>(
gdal: &Gdal,
raster: &R,
band_indices: &[usize],
) -> Result<Dataset> {
) -> Result<(Dataset, Vec<Vec<u8>>)> {
let metadata = raster.metadata();
let bands = raster.bands();

Expand All @@ -212,7 +222,12 @@ pub unsafe fn raster_ref_to_gdal_mem<R: RasterRef + ?Sized>(
// Create internal MEM dataset via sedona-gdal shim to avoid open dataset list contention.
let mut mem_ds_builder = MemDatasetBuilder::new(width, height);

// Add bands with DATAPOINTER option (zero-copy)
// Reader-allocated band bytes (Cow::Owned). Each entry is moved out of
// the Cow without a copy and must outlive the dataset, since GDAL holds
// a raw pointer into it.
let mut owned_band_bytes: Vec<Vec<u8>> = Vec::new();

// Add bands with DATAPOINTER option.
//
// Note: GDALAddBand always appends a new band, so the destination band index
// is sequential (1..=band_indices.len()), even if the source band indices are
Expand All @@ -222,6 +237,13 @@ pub unsafe fn raster_ref_to_gdal_mem<R: RasterRef + ?Sized>(
.band(src_band_index)
.map_err(|e| arrow_datafusion_err!(e))?;

if !band.is_2d() {
return exec_err!(
"GDAL backend requires a 2-dim band; got dim_names={:?}",
band.dim_names()
);
}

if band.metadata().storage_type()? != StorageType::InDb {
return Err(DataFusionError::NotImplemented(
"OutDb bands are not supported in raster_to_mem_dataset".to_string(),
Expand All @@ -231,8 +253,20 @@ pub unsafe fn raster_ref_to_gdal_mem<R: RasterRef + ?Sized>(
let band_metadata = band.metadata();
let band_type = band_metadata.data_type()?;
let gdal_type = band_data_type_to_gdal(&band_type);
let band_data = band.data();
let data_ptr = band_data.as_ptr();
let band_data = band
.contiguous_data()
.map_err(|e| arrow_datafusion_err!(e))?;
// For Cow::Borrowed the pointer is into the StructArray (caller keeps
// it alive). For Cow::Owned we move the Vec into `owned_band_bytes`
// — no extra copy of the reader's materialization — and point GDAL
// at it; the Vec is kept alive alongside the returned Dataset.
let data_ptr: *const u8 = match band_data {
Cow::Borrowed(b) => b.as_ptr(),
Cow::Owned(v) => {
owned_band_bytes.push(v);
owned_band_bytes.last().unwrap().as_ptr()
}
};
unsafe {
mem_ds_builder = mem_ds_builder.add_band(gdal_type, data_ptr as *mut u8);
}
Expand Down Expand Up @@ -295,14 +329,17 @@ pub unsafe fn raster_ref_to_gdal_mem<R: RasterRef + ?Sized>(
}
}

Ok(dataset)
Ok((dataset, owned_band_bytes))
}

pub fn raster_ref_to_gdal_empty<R: RasterRef + ?Sized>(gdal: &Gdal, raster: &R) -> Result<Dataset> {
unsafe {
// SAFETY: raster_ref_to_gdal_mem is safe to call with an empty band list. The
// returned dataset will have zero bands and references no external memory.
raster_ref_to_gdal_mem(gdal, raster, &[])
// returned dataset has zero bands, references no external memory, and the
// owned-bytes Vec is necessarily empty.
let (dataset, owned) = raster_ref_to_gdal_mem(gdal, raster, &[])?;
debug_assert!(owned.is_empty());
Ok(dataset)
}
}

Expand Down Expand Up @@ -770,7 +807,7 @@ mod tests {
let raster = single_raster(&raster_array);

with_gdal(|gdal| {
let dataset = unsafe { raster_ref_to_gdal_mem(gdal, &raster, &[3, 1])? };
let (dataset, _owned) = unsafe { raster_ref_to_gdal_mem(gdal, &raster, &[3, 1])? };
assert_eq!(dataset.raster_size(), (2, 2));
assert_eq!(dataset.raster_count(), 2);
assert_eq!(
Expand Down Expand Up @@ -825,4 +862,40 @@ mod tests {
.unwrap();
assert!(err.to_string().contains("OutDb bands are not supported"));
}

#[test]
fn test_raster_ref_to_gdal_mem_rejects_nd_bands() {
// Build a 3-D in-db band shaped ["time","y","x"] over a 2-D raster.
// The N-D guard should fire before any GDAL call.
let mut builder = RasterBuilder::new(1);
builder
.start_raster_2d(2, 2, 0.0, 2.0, 1.0, -1.0, 0.0, 0.0, None)
.unwrap();
builder
.start_band_nd(
None,
&["time", "y", "x"],
&[3, 2, 2],
BandDataType::UInt8,
None,
None,
None,
)
.unwrap();
builder
.band_data_writer()
.append_value(vec![0u8; 3 * 2 * 2]);
builder.finish_band().unwrap();
builder.finish_raster().unwrap();
let raster_array = builder.finish().unwrap();
let raster = single_raster(&raster_array);

let err = with_gdal(|gdal| unsafe { raster_ref_to_gdal_mem(gdal, &raster, &[1]) })
.err()
.unwrap();
assert!(
err.to_string().contains("requires a 2-dim band"),
"got: {err}"
);
}
}
Loading
Loading