diff --git a/Cargo.lock b/Cargo.lock index 491e8ed..905364f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -203,7 +203,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 2.1.1", + "rustc-hash 2.1.2", "shlex", "syn", ] @@ -235,9 +235,9 @@ checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" [[package]] name = "cc" -version = "1.2.57" +version = "1.2.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" +checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" dependencies = [ "find-msvc-tools", "jobserver", @@ -560,9 +560,9 @@ dependencies = [ [[package]] name = "inventory" -version = "0.3.22" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "009ae045c87e7082cb72dab0ccd01ae075dd00141ddc108f43a0ea150a9e7227" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" dependencies = [ "rustversion", ] @@ -608,9 +608,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jobserver" @@ -624,10 +624,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.91" +version = "0.3.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -649,9 +651,9 @@ checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" [[package]] name = "libc" -version = "0.2.183" +version = "0.2.184" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" [[package]] name = "libloading" @@ -761,9 +763,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" [[package]] name = "num-integer" @@ -796,13 +798,13 @@ dependencies = [ "num-traits", "pyo3", "pyo3-build-config", - "rustc-hash 2.1.1", + "rustc-hash 2.1.2", ] [[package]] name = "om-file-format-sys" version = "0.0.0" -source = "git+https://github.com/open-meteo/om-file-format?rev=4a9ecdcc28e6fc762cf3542a5d7ae3832b7d1f3c#4a9ecdcc28e6fc762cf3542a5d7ae3832b7d1f3c" +source = "git+https://github.com/open-meteo/om-file-format?rev=e42baa2edeb8e77e32540c7134edc77b3975a5bb#e42baa2edeb8e77e32540c7134edc77b3975a5bb" dependencies = [ "bindgen", "cc", @@ -812,7 +814,7 @@ dependencies = [ [[package]] name = "omfiles" version = "0.1.0" -source = "git+https://github.com/open-meteo/rust-omfiles?rev=5019cd968f50b489c6c192693f0da4a0ef1d9658#5019cd968f50b489c6c192693f0da4a0ef1d9658" +source = "git+https://github.com/open-meteo/rust-omfiles?rev=4954f78dcba4b2451cd31110859dcf715da4e09f#4954f78dcba4b2451cd31110859dcf715da4e09f" dependencies = [ "async-executor", "async-lock", @@ -831,9 +833,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "ordered-float" -version = "5.1.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4779c6901a562440c3786d08192c6fbda7c1c2060edd10006b05ee35d10f2d" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" dependencies = [ "num-traits", ] @@ -1184,9 +1186,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "rustix" @@ -1309,9 +1311,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "1.0.4" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" dependencies = [ "serde_core", ] @@ -1417,9 +1419,9 @@ dependencies = [ [[package]] name = "toml" -version = "1.0.7+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd28d57d8a6f6e458bc0b8784f8fdcc4b99a437936056fa122cb234f18656a96" +checksum = "994b95d9e7bae62b34bab0e2a4510b801fa466066a6a8b2b57361fa1eba068ee" dependencies = [ "indexmap", "serde_core", @@ -1432,27 +1434,27 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.0.1+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b320e741db58cac564e26c607d3cc1fdc4a88fd36c879568c07856ed83ff3e9" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ "serde_core", ] [[package]] name = "toml_parser" -version = "1.0.10+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df25b4befd31c4816df190124375d5a20c6b6921e2cad937316de3fccd63420" +checksum = "39ca317ebc49f06bd748bfba29533eac9485569dc9bf80b849024b025e814fb9" dependencies = [ "winnow", ] [[package]] name = "toml_writer" -version = "1.0.7+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17aaa1c6e3dc22b1da4b6bba97d066e354c7945cac2f7852d4e4e7ca7a6b56d" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" [[package]] name = "unic-char-property" @@ -1563,9 +1565,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.114" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" dependencies = [ "cfg-if", "once_cell", @@ -1576,23 +1578,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.64" +version = "0.4.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" +checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e" dependencies = [ - "cfg-if", - "futures-util", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.114" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1600,9 +1598,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.114" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" dependencies = [ "bumpalo", "proc-macro2", @@ -1613,23 +1611,13 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.114" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" dependencies = [ "unicode-ident", ] -[[package]] -name = "web-sys" -version = "0.3.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "windows-core" version = "0.62.2" @@ -1700,9 +1688,9 @@ dependencies = [ [[package]] name = "winnow" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" +checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" [[package]] name = "wit-bindgen" @@ -1712,18 +1700,18 @@ checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" [[package]] name = "zerocopy" -version = "0.8.42" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.42" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index c191c74..90febb5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,8 +24,8 @@ async-lock = "3.4" numpy = "0.28" num-traits = "0.2" delegate = "0.13" -omfiles-rs = { git = "https://github.com/open-meteo/rust-omfiles", rev = "5019cd968f50b489c6c192693f0da4a0ef1d9658", package = "omfiles", features = ["metadata-tree"] } -om-file-format-sys = { git = "https://github.com/open-meteo/om-file-format", rev = "4a9ecdcc28e6fc762cf3542a5d7ae3832b7d1f3c" } +omfiles-rs = { git = "https://github.com/open-meteo/rust-omfiles", rev = "4954f78dcba4b2451cd31110859dcf715da4e09f", package = "omfiles", features = ["metadata-tree"] } +om-file-format-sys = { git = "https://github.com/open-meteo/om-file-format", rev = "e42baa2edeb8e77e32540c7134edc77b3975a5bb" } thiserror = "2.0" [features] diff --git a/pyproject.toml b/pyproject.toml index 7229ae4..16a14b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,6 +112,9 @@ ignore = [ [project.entry-points."xarray.backends"] om = "omfiles.xarray:OmXarrayEntrypoint" +[project.entry-points."numcodecs.codecs"] +turbo_pfor = "omfiles._numcodecs:TurboPfor" + [project.entry-points."zarr.codecs"] "omfiles.pfor" = "omfiles._zarr3:PforCodec" "omfiles.pfor_serializer" = "omfiles._zarr3:PforSerializer" diff --git a/python/omfiles/_numcodecs.py b/python/omfiles/_numcodecs.py index 3889950..7e1a3bf 100644 --- a/python/omfiles/_numcodecs.py +++ b/python/omfiles/_numcodecs.py @@ -49,6 +49,3 @@ def decode(self, buf, out=None): # type: ignore buf = buf return self._impl.decode_array(buf, np.dtype(self.dtype), self.chunk_elements) - - -numcodecs.register_codec(TurboPfor) diff --git a/python/omfiles/_rust/__init__.pyi b/python/omfiles/_rust/__init__.pyi index 17e60f1..58c72fb 100644 --- a/python/omfiles/_rust/__init__.pyi +++ b/python/omfiles/_rust/__init__.pyi @@ -193,6 +193,16 @@ class OmFileReader: It is safe to call this method multiple times. """ + def get_complete_lut(self) -> builtins.list[builtins.int]: + r""" + Retrieve the complete lookup table for the variable. + + The lookup table is a monotonically increasing array of u64 values containing + n+1 elements, where n is the number of chunks. Each value represents the absolute + offset in the file of the end of the chunk, and the first value is the start offset + of the first chunk. The size of chunk j can be calculated as lut[j+1] - lut[j] + using zero-based indexing. + """ def get_child_by_index(self, index: builtins.int) -> OmFileReader: r""" Get a child reader at the specified index. diff --git a/src/codecs.rs b/src/codecs.rs index b2acb80..2125936 100644 --- a/src/codecs.rs +++ b/src/codecs.rs @@ -2,6 +2,7 @@ use numpy::{ ndarray, IntoPyArray, PyArray1, PyArrayDescr, PyArrayDescrMethods, PyArrayDyn, PyArrayMethods, PyUntypedArray, PyUntypedArrayMethods, }; +use omfiles_rs::_om_file_format_sys as om_file_format_sys; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::PyBytes; diff --git a/src/reader.rs b/src/reader.rs index b2c1934..d790ba8 100644 --- a/src/reader.rs +++ b/src/reader.rs @@ -369,6 +369,24 @@ impl OmFileReader { self.with_reader(|reader| Ok(reader.data_type().is_scalar())) } + /// Retrieve the complete lookup table for the variable. + /// + /// The lookup table is a monotonically increasing array of u64 values containing + /// n+1 elements, where n is the number of chunks. Each value represents the absolute + /// offset in the file of the end of the chunk, and the first value is the start offset + /// of the first chunk. The size of chunk j can be calculated as lut[j+1] - lut[j] + /// using zero-based indexing. + fn get_complete_lut(&self) -> PyResult> { + let lut = self.with_reader(|reader| { + reader + .expect_array() + .map_err(convert_omfilesrs_error)? + .get_complete_lut() + .map_err(convert_omfilesrs_error) + })?; + Ok(lut) + } + /// Check if the variable is a group (a variable with data type None). /// /// Returns: