From 53c3107cd5c8be7afcb2e86fc36069ad38bb51cb Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 5 Jun 2026 09:35:52 -0400 Subject: [PATCH 01/36] allow script to create single file with multiframes, additional arguments --- scripts/sentinelDownload/README.md | 34 +++- scripts/sentinelDownload/sentinel2Download.py | 179 +++++++++++++----- 2 files changed, 167 insertions(+), 46 deletions(-) diff --git a/scripts/sentinelDownload/README.md b/scripts/sentinelDownload/README.md index e6033f97b..12c758a3a 100644 --- a/scripts/sentinelDownload/README.md +++ b/scripts/sentinelDownload/README.md @@ -27,13 +27,41 @@ The script accepts command-line options via `click`: - `--start-date` _(str, default `2025-01-01`)_ - Start date in `YYYY-MM-DD` format. - `--end-date` _(str, default = today)_ - End date in `YYYY-MM-DD` format. - `--max-results` _(int, default `5`)_ - Maximum number of images to download. -- `--output-dir` _(path, default `sequentialTestRasters`)_ - Directory to save clipped files and JSON file. +- `--output-name` _(str, default `sequentialTestRasters`)_ - Base name for output. Rasters are saved under `downloads//`; ingest JSON is written as `.json` next to the script. - `--cloud-cover` _(float, default `30.0`)_ - Maximum allowed cloud cover percentage of files found. - `--size-km` _(float, default `10.0`)_ - Size of square window (in kilometers) to clip around the point. +- `--single-file` _(flag, default off)_ - Combine all downloaded frames into one multiframe GeoTIFF instead of writing separate files per date. The generated ingest JSON uses `frame_property: "frame"` for multiframe ingest. + +### `--single-file` GDAL requirement + +The `--single-file` option shells out to `gdal_translate` to append each clipped frame as a subdataset in one multiframe GeoTIFF. GDAL must be installed and `gdal_translate` must be on your `PATH`. + +If GDAL is not available, run without `--single-file` (the default writes one GeoTIFF per frame). --- ## Outputs -- **GeoTIFF files** - Clipped Sentinel-2 visual images (RGB). -- **`sample.json`** - JSON metadata describing datasets, layers, and frames, useful for ingestion into GeoDatalytics. +Files are written relative to this script directory: + +``` +scripts/sentinelDownload/ + sentinel2Download.py + .json # ingest manifest (sibling to the script) + downloads/ + / + *.tif # clipped GeoTIFF(s) +``` + +- **GeoTIFF files** - Clipped Sentinel-2 visual images (RGB). With `--single-file`, one multiframe GeoTIFF is written instead of separate per-date files. +- **`.json`** - Ingest manifest describing the project, dataset, layers, and frames. + +## Ingesting into GeoDatalytics + +Then ingest the sequential data from the project root: + +```bash +./manage.py ingest .json --replace +``` + +Use `--replace` if you have previously ingested the same project or dataset and need to refresh it. diff --git a/scripts/sentinelDownload/sentinel2Download.py b/scripts/sentinelDownload/sentinel2Download.py index c0d1ca120..4999192de 100644 --- a/scripts/sentinelDownload/sentinel2Download.py +++ b/scripts/sentinelDownload/sentinel2Download.py @@ -14,6 +14,9 @@ from datetime import UTC, datetime import json from pathlib import Path +import shutil +import subprocess +import tempfile import click import numpy as np @@ -26,6 +29,8 @@ # STAC API from AWS Earth Search STAC_API_URL = "https://earth-search.aws.element84.com/v1" +SCRIPT_DIR = Path(__file__).resolve().parent +DOWNLOADS_DIR = SCRIPT_DIR / "downloads" def default_end_date(): @@ -101,6 +106,44 @@ def read_cog_window_rgb(cog_url, lon, lat, size_km=10): return data, meta +def combine_frames_to_multiframe(frame_paths, output_path): + """ + Combine single-frame GeoTIFFs into one multiframe GeoTIFF. + + Each appended page becomes a scrubbable frame when imported with + frame_property: "frame". + """ + if not frame_paths: + return + + gdal_translate = shutil.which("gdal_translate") + if gdal_translate is None: + msg = ( + "gdal_translate is required for --single-file but was not found on PATH. " + "Install GDAL or run without --single-file." + ) + raise RuntimeError(msg) + + output_path = Path(output_path) + creation_options = ["-co", "COMPRESS=LZW"] + subprocess.run( + [gdal_translate, *creation_options, str(frame_paths[0]), str(output_path)], + check=True, + ) + for frame_path in frame_paths[1:]: + subprocess.run( + [ + gdal_translate, + *creation_options, + "-co", + "APPEND_SUBDATASET=YES", + str(frame_path), + str(output_path), + ], + check=True, + ) + + @click.command() @click.option( "--lat", default=43.135763, type=float, required=True, help="Latitude of the location." @@ -130,11 +173,14 @@ def read_cog_window_rgb(cog_url, lon, lat, size_km=10): help="Maximum number of images to download.", ) @click.option( - "--output-dir", - type=click.Path(), + "--output-name", + type=str, default="sequentialTestRasters", show_default=True, - help="Directory to save the downloaded files.", + help=( + "Base name for output: writes rasters to downloads// and " + "a sibling ingest JSON named .json next to this script." + ), ) @click.option( "--cloud-cover", type=float, default=30.0, show_default=True, help="Max cloud cover percentage." @@ -146,11 +192,21 @@ def read_cog_window_rgb(cog_url, lon, lat, size_km=10): show_default=True, help="Size of square window to clip around the point in kilometers.", ) +@click.option( + "--single-file", + is_flag=True, + default=False, + help=( + "Write all frames into one multiframe GeoTIFF instead of separate files. " + "The generated output JSON will use frame_property: \"frame\"." + ), +) def download_stac_sentinel( # noqa: PLR0913, PLR0915 - lat, lon, start_date, end_date, max_results, output_dir, cloud_cover, size_km + lat, lon, start_date, end_date, max_results, output_name, cloud_cover, size_km, single_file ): """Download clipped Sentinel-2 L1C visual images from AWS via STAC API.""" - Path(output_dir).mkdir(parents=True, exist_ok=True) + output_dir = DOWNLOADS_DIR / output_name + output_dir.mkdir(parents=True, exist_ok=True) catalog = Client.open(STAC_API_URL) @@ -166,7 +222,7 @@ def download_stac_sentinel( # noqa: PLR0913, PLR0915 limit=max_results, ) - items = list(search.get_items()) + items = list(search.items()) if not items: click.echo("⚠️ No Sentinel-2 images found.") @@ -187,40 +243,59 @@ def download_stac_sentinel( # noqa: PLR0913, PLR0915 ) downloaded_files = [] - - for i, item in enumerate(items): - if i >= max_results: - break - date_str = item.datetime.strftime("%Y-%m-%d") - item_id = item.id - click.echo(f"[{i + 1}/{len(items)}] {item_id} from {date_str}") - - visual_asset = item.assets.get("visual") - if visual_asset: - url = visual_asset.href - filename = f"{item_id}_visual_clip_{int(size_km)}km.tif" - filepath = Path(output_dir) / filename - - click.echo(f" - Reading {size_km}km x {size_km}km window around point") - try: - data, meta = read_cog_window_rgb(url, lon, lat, size_km=size_km) - with rasterio.open(filepath, "w", **meta) as dst: - dst.write(data) - except (RasterioError, RasterioIOError) as e: - click.echo(f" - ⚠️ Failed to read or save clipped image: {e}") + downloaded_frame_paths = [] + multiframe_filename = None + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + for i, item in enumerate(items): + if i >= max_results: + break + date_str = item.datetime.strftime("%Y-%m-%d") + item_id = item.id + click.echo(f"[{i + 1}/{len(items)}] {item_id} from {date_str}") + + visual_asset = item.assets.get("visual") + if visual_asset: + url = visual_asset.href + filename = f"{item_id}_visual_clip_{int(size_km)}km.tif" + filepath = output_dir / filename + write_path = temp_path / filename if single_file else filepath + + click.echo(f" - Reading {size_km}km x {size_km}km window around point") + try: + data, meta = read_cog_window_rgb(url, lon, lat, size_km=size_km) + with rasterio.open(write_path, "w", **meta) as dst: + dst.write(data) + except (RasterioError, RasterioIOError) as e: + click.echo(f" - ⚠️ Failed to read or save clipped image: {e}") + else: + if single_file: + click.echo(f" - Buffered frame {len(downloaded_frame_paths)}") + downloaded_frame_paths.append(write_path) + else: + click.echo(f" - Saved clipped image to {filename}") + downloaded_files.append(filename) else: - click.echo(f" - Saved clipped image to {filename}") - downloaded_files.append(filename) - else: - click.echo(f" - ⚠️ Visual asset not available in item {item_id}") + click.echo(f" - ⚠️ Visual asset not available in item {item_id}") + + if single_file and downloaded_frame_paths: + multiframe_filename = f"sentinel_visual_clip_{int(size_km)}km_multiframe.tif" + multiframe_path = output_dir / multiframe_filename + click.echo( + f"Combining {len(downloaded_frame_paths)} frames into {multiframe_filename}..." + ) + combine_frames_to_multiframe(downloaded_frame_paths, multiframe_path) + click.echo(f" - Saved multiframe image to {multiframe_filename}") click.echo("✅ Download complete.") - # Generate dataset.json dataset_json = { "type": "Dataset", "name": "Sequential Test Rasters", "description": "Clipped Sentinel-2 images downloaded and clipped around point", "category": "imagery", + "tags": ["sentinel-2", "imagery", "sequential"], "files": [], "layers": [], } @@ -229,22 +304,40 @@ def download_stac_sentinel( # noqa: PLR0913, PLR0915 "type": "Project", "name": "Sentinel-2 Clipped Images", "datasets": ["Sequential Test Rasters"], - "default_map_center": [lat, lon], + "default_map_center": [lon, lat], "default_map_zoom": 11, } - # Add each file as its own layer - layer_frames = [] - for idx, f in enumerate(downloaded_files): - dataset_json["files"].append({"path": f"{output_dir}/{f}", "name": f"Frame {idx}"}) - layer_frames.append({"name": f"Sequential Layer {idx}", "index": idx, "data": f}) - - layer = {"name": "Sequential Test Layers", "frames": layer_frames} - dataset_json["layers"].append(layer) - - json_path = Path(output_dir, "sample.json") + if single_file and multiframe_filename: + dataset_json["files"].append( + {"path": f"{output_name}/{multiframe_filename}", "name": multiframe_filename} + ) + dataset_json["layers"].append( + { + "name": "Sequential Test Layers", + "frame_property": "frame", + "data": multiframe_filename, + } + ) + else: + layer_frames = [] + for idx, f in enumerate(downloaded_files): + dataset_json["files"].append({"path": f"{output_name}/{f}", "name": f"Frame {idx}"}) + layer_frames.append( + { + "name": f"Sequential Layer {idx}", + "index": idx, + "data": f, + } + ) + + dataset_json["layers"].append({"name": "Sequential Test Layers", "frames": layer_frames}) + + json_path = SCRIPT_DIR / f"{output_name}.json" with json_path.open("w") as jf: json.dump([project_json, dataset_json], jf, indent=4) + click.echo(f" - Wrote ingest JSON to {json_path}") + click.echo(f" - Rasters saved under {output_dir}") if __name__ == "__main__": From 4d475421990ea4be4f439cfe82e056dc3ef34d2a Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 5 Jun 2026 09:45:07 -0400 Subject: [PATCH 02/36] don't force a band when the source_filters are empty --- uvdat/core/tasks/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uvdat/core/tasks/dataset.py b/uvdat/core/tasks/dataset.py index 2724079ca..435042786 100644 --- a/uvdat/core/tasks/dataset.py +++ b/uvdat/core/tasks/dataset.py @@ -136,7 +136,7 @@ def create_layers_and_frames(dataset, layer_options=None): # noqa: C901, PLR091 index=index, vector=vector, raster=raster, - source_filters=frame_info.get("source_filters", {"band": 1}), + source_filters=frame_info.get("source_filters", {}), ) From aa64e065e7407524c7251b5a16843244706db95b Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 5 Jun 2026 09:50:29 -0400 Subject: [PATCH 03/36] prevent frame or band from being a style property --- scripts/sentinelDownload/sentinel2Download.py | 14 +-- web/src/store/map.ts | 23 ++--- web/src/store/style.ts | 94 ++++++++++++++++--- 3 files changed, 95 insertions(+), 36 deletions(-) diff --git a/scripts/sentinelDownload/sentinel2Download.py b/scripts/sentinelDownload/sentinel2Download.py index 4999192de..dc90d6666 100644 --- a/scripts/sentinelDownload/sentinel2Download.py +++ b/scripts/sentinelDownload/sentinel2Download.py @@ -106,6 +106,10 @@ def read_cog_window_rgb(cog_url, lon, lat, size_km=10): return data, meta +def _run_checked_command(cmd: list[str]) -> None: + subprocess.run(cmd, check=True) # noqa: S603 + + def combine_frames_to_multiframe(frame_paths, output_path): """ Combine single-frame GeoTIFFs into one multiframe GeoTIFF. @@ -126,12 +130,11 @@ def combine_frames_to_multiframe(frame_paths, output_path): output_path = Path(output_path) creation_options = ["-co", "COMPRESS=LZW"] - subprocess.run( + _run_checked_command( [gdal_translate, *creation_options, str(frame_paths[0]), str(output_path)], - check=True, ) for frame_path in frame_paths[1:]: - subprocess.run( + _run_checked_command( [ gdal_translate, *creation_options, @@ -140,7 +143,6 @@ def combine_frames_to_multiframe(frame_paths, output_path): str(frame_path), str(output_path), ], - check=True, ) @@ -198,10 +200,10 @@ def combine_frames_to_multiframe(frame_paths, output_path): default=False, help=( "Write all frames into one multiframe GeoTIFF instead of separate files. " - "The generated output JSON will use frame_property: \"frame\"." + 'The generated output JSON will use frame_property: "frame".' ), ) -def download_stac_sentinel( # noqa: PLR0913, PLR0915 +def download_stac_sentinel( # noqa: C901, PLR0912, PLR0913, PLR0915 lat, lon, start_date, end_date, max_results, output_name, cloud_cover, size_km, single_file ): """Download clipped Sentinel-2 L1C visual images from AWS via STAC API.""" diff --git a/web/src/store/map.ts b/web/src/store/map.ts index d22204ace..a59464dd5 100644 --- a/web/src/store/map.ts +++ b/web/src/store/map.ts @@ -524,9 +524,6 @@ export const useMapStore = defineStore("map", () => { ): Source | undefined { const map = getMap(); - const queryParams: { projection: string; style?: string } = { - projection: "epsg:3857", - }; const { layerId, layerCopyId } = parseSourceString(sourceId); const styleSpec = styleStore.selectedLayerStyles[`${layerId}.${layerCopyId}`].style_spec; @@ -540,22 +537,14 @@ export const useMapStore = defineStore("map", () => { (f: LayerFrame) => f.index === layer.current_frame_index, ); if (frame?.source_filters) { - filters = Object.entries(frame.source_filters).map(([k, v]) => ({ - filter_by: k, - list: [v], - include: true, - transparency: true, - apply: true, - })); + filters = styleStore.sourceFiltersToStyleFilters(frame.source_filters); } } - if (styleSpec) { - const styleParams = styleStore.getRasterTilesQuery( - { ...styleSpec, filters }, - styleStore.colormaps, - ); - if (styleParams) queryParams.style = JSON.stringify(styleParams); - } + const queryParams = styleStore.buildRasterTileQueryParams( + styleSpec ?? styleStore.getDefaultStyleSpec(raster, layerId), + filters, + styleStore.colormaps, + ); const query = new URLSearchParams(queryParams); rasterSourceTileURLs.value[sourceId] = `${baseURL}rasters/${raster.id}/tiles/{z}/{x}/{y}.png/?${query}`; diff --git a/web/src/store/style.ts b/web/src/store/style.ts index e2e54fc38..bee2dd72a 100644 --- a/web/src/store/style.ts +++ b/web/src/store/style.ts @@ -89,6 +89,51 @@ export function colormapMarkersSubsample( return markers; } +// frame/band select which slice to read; they must be flat query params, not in style JSON +const RASTER_SOURCE_FILTER_KEYS = new Set(["frame", "band"]); + +// Ingest defaults missing source_filters to { band: 1 }; that is not a real band selection. +function isDefaultBandSourceFilter( + sourceFilters: Record, +): boolean { + const keys = Object.keys(sourceFilters); + return ( + keys.length === 1 && + keys[0] === "band" && + (sourceFilters.band === 1 || sourceFilters.band === "1") + ); +} + +function sourceFiltersToStyleFilters( + sourceFilters: Record | undefined, +): StyleFilter[] { + if (!sourceFilters || !Object.keys(sourceFilters).length) return []; + if (isDefaultBandSourceFilter(sourceFilters)) return []; + return Object.entries(sourceFilters).map(([k, v]) => ({ + filter_by: k, + list: [v], + include: true, + transparency: true, + apply: true, + })); +} + +function getRasterSourceFilterParams(filters: StyleFilter[]) { + const params: Record = {}; + filters.forEach((f) => { + if ( + f.apply && + f.filter_by && + f.include && + f.list?.length === 1 && + RASTER_SOURCE_FILTER_KEYS.has(f.filter_by) + ) { + params[f.filter_by] = f.list[0]; + } + }); + return params; +} + function getRasterTilesQuery(styleSpec: StyleSpec, colormaps: Colormap[]) { let query: Record = {}; const colorSpecs = styleSpec.colors || []; @@ -126,13 +171,37 @@ function getRasterTilesQuery(styleSpec: StyleSpec, colormaps: Colormap[]) { } }); styleSpec.filters.forEach((f) => { - if (f.apply && f.filter_by && f.include && f.list?.length === 1) { + if ( + f.apply && + f.filter_by && + f.include && + f.list?.length === 1 && + !RASTER_SOURCE_FILTER_KEYS.has(f.filter_by) + ) { query[f.filter_by] = f.list[0]; } }); return query; } +function buildRasterTileQueryParams( + styleSpec: StyleSpec, + filters: StyleFilter[], + colormaps: Colormap[], +) { + const params: Record = { projection: "epsg:3857" }; + Object.entries(getRasterSourceFilterParams(filters)).forEach( + ([key, value]) => { + params[key] = String(value); + }, + ); + const styleQuery = getRasterTilesQuery({ ...styleSpec, filters }, colormaps); + if (Object.keys(styleQuery).length) { + params.style = JSON.stringify(styleQuery); + } + return params; +} + function getVectorColorPaintProperty( styleSpec: StyleSpec, groupName: string, @@ -441,13 +510,7 @@ export const useStyleStore = defineStore("style", () => { if (frame?.source_filters) { filters = [ ...filters, - ...Object.entries(frame.source_filters).map(([k, v]) => ({ - filter_by: k, - list: [v], - include: true, - transparency: true, - apply: true, - })), + ...sourceFiltersToStyleFilters(frame.source_filters), ]; } const mapLayer = map.getLayer(mapLayerId) as @@ -550,11 +613,13 @@ export const useStyleStore = defineStore("style", () => { const source = map.getSource(mapLayer.source) as RasterTileSource; const sourceURL = mapStore.rasterSourceTileURLs[mapLayer.source]; if (source && sourceURL) { - const newQueryParams: { projection: string; style?: string } = { - projection: "epsg:3857", - }; - newQueryParams.style = JSON.stringify(rasterTilesQuery); - const newQuery = new URLSearchParams(newQueryParams); + const newQuery = new URLSearchParams( + buildRasterTileQueryParams( + { ...styleSpec, filters }, + filters, + colormaps.value, + ), + ); tileURL = sourceURL.split("?")[0] + "?" + newQuery.toString(); return { paint, tileURL }; } @@ -605,6 +670,9 @@ export const useStyleStore = defineStore("style", () => { selectedLayerStyles, fetchColormaps, getRasterTilesQuery, + getRasterSourceFilterParams, + buildRasterTileQueryParams, + sourceFiltersToStyleFilters, colormapMarkersSubsample, getDefaultColor, getDefaultStyleSpec, From 2856796e7eebc6e5b8c69596964d4077dff1c6fb Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 12 Jun 2026 13:39:08 -0400 Subject: [PATCH 04/36] add RasterFramePreview model --- uvdat/core/admin.py | 7 ++ .../migrations/0024_raster_frame_preview.py | 66 +++++++++++++++++++ uvdat/core/models/__init__.py | 2 + uvdat/core/models/frame_preview.py | 45 +++++++++++++ 4 files changed, 120 insertions(+) create mode 100644 uvdat/core/migrations/0024_raster_frame_preview.py create mode 100644 uvdat/core/models/frame_preview.py diff --git a/uvdat/core/admin.py b/uvdat/core/admin.py index 02b1e859d..e98e1c1c5 100644 --- a/uvdat/core/admin.py +++ b/uvdat/core/admin.py @@ -19,6 +19,7 @@ NetworkNode, Project, RasterData, + RasterFramePreview, Region, SizeConfig, SizeRangeConfig, @@ -69,6 +70,12 @@ class LayerStyleAdmin(admin.ModelAdmin): list_display = ["id", "name", "layer"] +@admin.register(RasterFramePreview) +class RasterFramePreviewAdmin(admin.ModelAdmin): + list_display = ["id", "layer_style", "layer_frame", "status", "width", "height"] + list_select_related = ["layer_style", "layer_frame"] + + @admin.register(Colormap) class ColormapAdmin(admin.ModelAdmin): list_display = ["id", "name"] diff --git a/uvdat/core/migrations/0024_raster_frame_preview.py b/uvdat/core/migrations/0024_raster_frame_preview.py new file mode 100644 index 000000000..260540508 --- /dev/null +++ b/uvdat/core/migrations/0024_raster_frame_preview.py @@ -0,0 +1,66 @@ +# Generated by Django 5.2.9 on 2026-06-12 +from __future__ import annotations + +from django.db import migrations, models +import django.db.models.deletion +import s3_file_field.fields + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0023_task_subscribers"), + ] + + operations = [ + migrations.CreateModel( + name="RasterFramePreview", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("image", s3_file_field.fields.S3FileField()), + ("width", models.PositiveIntegerField()), + ("height", models.PositiveIntegerField()), + ("bounds", models.JSONField()), + ( + "status", + models.CharField( + choices=[ + ("pending", "Pending"), + ("ready", "Ready"), + ("failed", "Failed"), + ], + default="pending", + max_length=10, + ), + ), + ( + "layer_frame", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="style_previews", + to="core.layerframe", + ), + ), + ( + "layer_style", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="frame_previews", + to="core.layerstyle", + ), + ), + ], + options={ + "constraints": [ + models.UniqueConstraint( + fields=("layer_style", "layer_frame"), + name="unique_layer_style_layer_frame_preview", + ) + ], + }, + ), + ] diff --git a/uvdat/core/models/__init__.py b/uvdat/core/models/__init__.py index 83ff77636..e3fcf6305 100644 --- a/uvdat/core/models/__init__.py +++ b/uvdat/core/models/__init__.py @@ -6,6 +6,7 @@ from .data import RasterData, VectorData, VectorFeature from .dataset import Dataset, DatasetTag from .file_item import FileItem +from .frame_preview import RasterFramePreview from .layer import Layer, LayerFrame from .networks import Network, NetworkEdge, NetworkNode from .project import Project @@ -34,6 +35,7 @@ "Layer", "LayerFrame", "LayerStyle", + "RasterFramePreview", "Network", "NetworkEdge", "NetworkNode", diff --git a/uvdat/core/models/frame_preview.py b/uvdat/core/models/frame_preview.py new file mode 100644 index 000000000..d28a00890 --- /dev/null +++ b/uvdat/core/models/frame_preview.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from django.db import models +from s3_file_field import S3FileField + +from .layer import LayerFrame +from .styles import LayerStyle + + +class RasterFramePreview(models.Model): + class Status(models.TextChoices): + PENDING = "pending", "Pending" + READY = "ready", "Ready" + FAILED = "failed", "Failed" + + layer_style = models.ForeignKey( + LayerStyle, + related_name="frame_previews", + on_delete=models.CASCADE, + ) + layer_frame = models.ForeignKey( + LayerFrame, + related_name="style_previews", + on_delete=models.CASCADE, + ) + image = S3FileField() + width = models.PositiveIntegerField() + height = models.PositiveIntegerField() + bounds = models.JSONField() + status = models.CharField( + max_length=10, + choices=Status.choices, + default=Status.PENDING, + ) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["layer_style", "layer_frame"], + name="unique_layer_style_layer_frame_preview", + ) + ] + + def __str__(self): + return f"Preview style={self.layer_style_id} frame={self.layer_frame_id} ({self.id})" From 9bfb6bceac734d1d4812c32d2a35ccd3c3a639c2 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 12 Jun 2026 13:44:58 -0400 Subject: [PATCH 05/36] add deletion of s3/minIO preview imageobject when layer/layer_style deleted --- uvdat/core/models/frame_preview.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/uvdat/core/models/frame_preview.py b/uvdat/core/models/frame_preview.py index d28a00890..15bf2dc7d 100644 --- a/uvdat/core/models/frame_preview.py +++ b/uvdat/core/models/frame_preview.py @@ -1,6 +1,7 @@ from __future__ import annotations from django.db import models +from django.dispatch import receiver from s3_file_field import S3FileField from .layer import LayerFrame @@ -43,3 +44,9 @@ class Meta: def __str__(self): return f"Preview style={self.layer_style_id} frame={self.layer_frame_id} ({self.id})" + + +@receiver(models.signals.post_delete, sender=RasterFramePreview) +def delete_preview_image(sender, instance, **kwargs): + if instance.image: + instance.image.delete(save=False) From f781a3f2b329ef370ecd9df11036b7729eaccd18 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 12 Jun 2026 14:40:31 -0400 Subject: [PATCH 06/36] create style for thumbnail generation, make sure frame is not in style --- uvdat/core/raster_style.py | 164 +++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 uvdat/core/raster_style.py diff --git a/uvdat/core/raster_style.py b/uvdat/core/raster_style.py new file mode 100644 index 000000000..b37f62179 --- /dev/null +++ b/uvdat/core/raster_style.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from uvdat.core.models import Colormap + +RASTER_SOURCE_FILTER_KEYS = frozenset({"frame", "band"}) + + +def _hex_to_rgb(color: str) -> tuple[int, int, int]: + color = color.lstrip("#") + return int(color[0:2], 16), int(color[2:4], 16), int(color[4:6], 16) + + +def _rgb_to_hex(rgb: tuple[int, int, int]) -> str: + return "#{:02X}{:02X}{:02X}".format(*rgb) + + +def _mix_marker_colors( + marker_a: dict[str, Any], + marker_b: dict[str, Any], +) -> dict[str, Any]: + rgb_a = _hex_to_rgb(marker_a["color"]) + rgb_b = _hex_to_rgb(marker_b["color"]) + mixed = tuple((a + b) // 2 for a, b in zip(rgb_a, rgb_b, strict=True)) + return { + "color": _rgb_to_hex(mixed), + "value": (marker_a["value"] + marker_b["value"]) / 2, + } + + +def colormap_markers_subsample( + colormap: Colormap, + applied_colormap: dict[str, Any], + n: int | None = None, +) -> list[dict[str, Any]]: + markers = list(colormap.markers) + if ( + n is None + and applied_colormap.get("discrete") + and applied_colormap.get("n_colors") + ): + n = applied_colormap["n_colors"] + if n and markers: + while n > len(markers): + expanded: list[dict[str, Any]] = [] + for i in range(len(markers) - 1): + expanded.append(markers[i]) + expanded.append(_mix_marker_colors(markers[i], markers[i + 1])) + expanded.append(markers[-1]) + markers = expanded + total_items = len(markers) - 1 + interval = total_items // (n - 1) if n > 1 else 0 + elements = [markers[0]] + elements.extend(markers[i * interval] for i in range(1, n - 1)) + elements.append(markers[-1]) + return elements + return markers + + +def is_default_band_source_filter(source_filters: dict[str, Any] | None) -> bool: + if not source_filters or len(source_filters) != 1: + return False + if "band" not in source_filters: + return False + return source_filters["band"] in (1, "1") + + +def _build_color_query_from_spec( + color_spec: dict[str, Any], + colormaps_by_id: dict[int, Colormap], +) -> dict[str, Any]: + color_query: dict[str, Any] = {} + colormap_spec = color_spec.get("colormap") + if colormap_spec: + if colormap_spec.get("range"): + color_query["min"] = colormap_spec["range"][0] + color_query["max"] = colormap_spec["range"][1] + if colormap_spec.get("discrete"): + color_query["scheme"] = "discrete" + if colormap_spec.get("clamp") is False: + color_query["clamp"] = False + colormap = colormaps_by_id.get(colormap_spec.get("id")) + if colormap and colormap.markers: + markers = colormap_markers_subsample(colormap, colormap_spec) + color_query["palette"] = [marker["color"] for marker in markers] + elif color_spec.get("single_color"): + color_query["palette"] = color_spec["single_color"] + return color_query + + +def _apply_color_spec_to_query( + query: dict[str, Any], + color_spec: dict[str, Any], + color_query: dict[str, Any], +) -> dict[str, Any]: + if not color_spec.get("visible"): + return query + if color_spec.get("name") == "all": + return color_query + query.setdefault("bands", []) + color_query["band"] = color_spec["name"].replace("Band ", "") + query["bands"].append(color_query) + return query + + +def _apply_non_source_filters( + query: dict[str, Any], + style_spec: dict[str, Any], +) -> None: + for filter_spec in style_spec.get("filters", []): + if ( + filter_spec.get("include") + and filter_spec.get("filter_by") + and filter_spec.get("list") + and len(filter_spec["list"]) == 1 + and filter_spec["filter_by"] not in RASTER_SOURCE_FILTER_KEYS + ): + query[filter_spec["filter_by"]] = filter_spec["list"][0] + + +def build_raster_tiles_style_query( + style_spec: dict[str, Any], + colormaps_by_id: dict[int, Colormap], +) -> dict[str, Any]: + query: dict[str, Any] = {} + for color_spec in style_spec.get("colors", []): + color_query = _build_color_query_from_spec(color_spec, colormaps_by_id) + query = _apply_color_spec_to_query(query, color_spec, color_query) + _apply_non_source_filters(query, style_spec) + return query + + +def apply_source_filters_to_style_query( + base_query: dict[str, Any], + source_filters: dict[str, Any] | None, +) -> dict[str, Any]: + query = dict(base_query) + if ( + source_filters + and not is_default_band_source_filter(source_filters) + and "band" in source_filters + ): + query["band"] = source_filters["band"] + return query + + +def raster_source_filter_kwargs(source_filters: dict[str, Any] | None) -> dict[str, Any]: + """Return large_image kwargs that must not be embedded in style JSON.""" + if not source_filters or "frame" not in source_filters: + return {} + return {"frame": source_filters["frame"]} + + +def build_thumbnail_style_query( + style_spec: dict[str, Any], + source_filters: dict[str, Any] | None, + colormaps_by_id: dict[int, Colormap], +) -> dict[str, Any]: + return apply_source_filters_to_style_query( + build_raster_tiles_style_query(style_spec, colormaps_by_id), + source_filters, + ) From 9ae568877e7c62df9384ed3d55b9697ce721542a Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 12 Jun 2026 14:51:18 -0400 Subject: [PATCH 07/36] generate preview task --- uvdat/core/tasks/__init__.py | 2 + uvdat/core/tasks/frame_preview.py | 222 ++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 uvdat/core/tasks/frame_preview.py diff --git a/uvdat/core/tasks/__init__.py b/uvdat/core/tasks/__init__.py index 9e431834c..f1656a793 100644 --- a/uvdat/core/tasks/__init__.py +++ b/uvdat/core/tasks/__init__.py @@ -2,8 +2,10 @@ from .chart import convert_chart from .dataset import convert_dataset +from .frame_preview import generate_layer_style_previews __all__ = [ "convert_chart", "convert_dataset", + "generate_layer_style_previews", ] diff --git a/uvdat/core/tasks/frame_preview.py b/uvdat/core/tasks/frame_preview.py new file mode 100644 index 000000000..263d21b44 --- /dev/null +++ b/uvdat/core/tasks/frame_preview.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +import io +import json +import logging +import time +from typing import Any + +from celery import shared_task +from django.core.files.base import ContentFile +from django_large_image import tilesource, utilities +from PIL import Image + +from uvdat.core.models import ( + Colormap, + Layer, + LayerFrame, + LayerStyle, + Project, + RasterData, + RasterFramePreview, +) +from uvdat.core.raster_style import ( + apply_source_filters_to_style_query, + build_raster_tiles_style_query, + raster_source_filter_kwargs, +) + +logger = logging.getLogger(__name__) + +FRAME_PREVIEW_MAX_PX = 4096 +FRAME_PREVIEW_MIN_PX = 1024 +FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION = 1 / 8 + +DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC = { + "default_frame": 0, + "opacity": 1, + "colors": [{"name": "all", "visible": True, "use_feature_props": True}], + "sizes": [{"name": "all", "zoom_scaling": True, "single_size": 5}], + "filters": [], +} + + +def resolve_resolution_fraction(resolution_fraction: float | None = None) -> float: + if resolution_fraction is None: + return FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION + return float(resolution_fraction) + + +def _raster_max_dimension(metadata: dict[str, Any]) -> int: + size_x = metadata.get("sizeX") or metadata.get("width") or 0 + size_y = metadata.get("sizeY") or metadata.get("height") or 0 + return max(int(size_x), int(size_y)) + + +def resolve_preview_max_dimension( + resolution_fraction: float | None = None, + raster_max_dimension: int | None = None, +) -> int: + fraction = resolve_resolution_fraction(resolution_fraction) + fractional = round(FRAME_PREVIEW_MAX_PX * fraction) + if not raster_max_dimension or raster_max_dimension < 2: + return max(2, fractional) + floor = min(raster_max_dimension, FRAME_PREVIEW_MIN_PX) + return max(2, min(max(fractional, floor), raster_max_dimension)) + + +def _colormaps_for_style(layer_style: LayerStyle) -> dict[int, Colormap]: + colormaps = Colormap.objects.filter(project=layer_style.project) | Colormap.objects.filter( + project__isnull=True + ) + return {colormap.id: colormap for colormap in colormaps} + + +def _thumbnail_png_bytes(thumb_data: Any) -> bytes: + if isinstance(thumb_data, bytes): + return thumb_data + if isinstance(thumb_data, Image.Image): + buffer = io.BytesIO() + thumb_data.save(buffer, format="PNG") + return buffer.getvalue() + msg = f"Unsupported thumbnail data type: {type(thumb_data)!r}" + raise TypeError(msg) + + +def _preview_bounds(source) -> dict[str, Any] | None: + bounds = tilesource.get_bounds(source, projection="EPSG:4326") + if not bounds: + return None + return { + "srs": "EPSG:4326", + "xmin": bounds["xmin"], + "xmax": bounds["xmax"], + "ymin": bounds["ymin"], + "ymax": bounds["ymax"], + } + + +def generate_frame_preview_png( + raster: RasterData, + source_filters: dict[str, Any] | None, + base_style_query: dict[str, Any], + resolution_fraction: float | None = None, +) -> tuple[bytes, int, int, dict[str, Any] | None]: + style_query = apply_source_filters_to_style_query(base_style_query, source_filters) + style = json.dumps(style_query) if style_query else None + source_kwargs = raster_source_filter_kwargs(source_filters) + raster_path = utilities.field_file_to_local_path(raster.cloud_optimized_geotiff) + source = tilesource.get_tilesource_from_path( + raster_path, + encoding="PNG", + style=style, + ) + max_dimension = resolve_preview_max_dimension( + resolution_fraction, + _raster_max_dimension(source.getMetadata()), + ) + thumb_data, _mime_type = source.getThumbnail( + encoding="PNG", + width=max_dimension, + height=max_dimension, + **source_kwargs, + ) + png_bytes = _thumbnail_png_bytes(thumb_data) + image = Image.open(io.BytesIO(png_bytes)) + return png_bytes, image.width, image.height, _preview_bounds(source) + + +def is_multiframe_raster_layer(layer: Layer) -> bool: + return LayerFrame.objects.filter(layer=layer, raster__isnull=False).count() > 1 + + +def _multiframe_raster_frames(layer_style: LayerStyle): + return ( + LayerFrame.objects.filter(layer=layer_style.layer, raster__isnull=False) + .select_related("raster") + .order_by("index") + ) + + +def ensure_default_layer_style(layer: Layer, project: Project) -> LayerStyle: + style = LayerStyle.objects.filter(layer=layer, project=project, name="Default").first() + if style is None: + style = LayerStyle.objects.create(name="Default", layer=layer, project=project) + style.save_style_configs(DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC) + if layer.default_style_id is None: + layer.default_style = style + layer.save(update_fields=["default_style"]) + return style + + +@shared_task +def generate_layer_style_previews( + layer_style_id: int, + resolution_fraction: float | None = None, +): + started = time.perf_counter() + layer_style = LayerStyle.objects.select_related("layer").get(id=layer_style_id) + frames = list(_multiframe_raster_frames(layer_style)) + if len(frames) <= 1: + return + + logger.info( + "Generating %d multiframe raster previews for style=%s layer=%r", + len(frames), + layer_style_id, + layer_style.layer.name, + ) + + style_spec = layer_style.repr_style_configs() + colormaps_by_id = _colormaps_for_style(layer_style) + base_style_query = build_raster_tiles_style_query(style_spec, colormaps_by_id) + + ready_count = 0 + failed_count = 0 + for frame in frames: + existing = RasterFramePreview.objects.filter( + layer_style=layer_style, + layer_frame=frame, + ).first() + try: + png_bytes, width, height, bounds = generate_frame_preview_png( + frame.raster, + frame.source_filters, + base_style_query, + resolution_fraction, + ) + preview, _created = RasterFramePreview.objects.update_or_create( + layer_style=layer_style, + layer_frame=frame, + defaults={ + "width": width, + "height": height, + "bounds": bounds or {}, + "status": RasterFramePreview.Status.READY, + }, + ) + preview.image.save( + f"frame-previews/{layer_style_id}/{frame.index}.png", + ContentFile(png_bytes), + save=True, + ) + ready_count += 1 + except Exception: + failed_count += 1 + logger.exception( + "Failed to generate frame preview for style=%s frame=%s", + layer_style_id, + frame.id, + ) + if existing is not None: + existing.status = RasterFramePreview.Status.FAILED + existing.save(update_fields=["status"]) + + logger.info( + "Multiframe raster previews for style=%s layer=%r: %d ready, %d failed in %.2fs", + layer_style_id, + layer_style.layer.name, + ready_count, + failed_count, + time.perf_counter() - started, + ) From af880c80594ce70a091ca85043bb7bf99e68afac Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 12 Jun 2026 14:51:36 -0400 Subject: [PATCH 08/36] generate previews of multiframe raster data on ingest --- uvdat/core/management/commands/ingest.py | 39 ++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/uvdat/core/management/commands/ingest.py b/uvdat/core/management/commands/ingest.py index bcfe0519f..69637404e 100644 --- a/uvdat/core/management/commands/ingest.py +++ b/uvdat/core/management/commands/ingest.py @@ -17,7 +17,12 @@ import djclick as click import pooch -from uvdat.core.models import Chart, Dataset, FileItem, Project +from uvdat.core.models import Chart, Dataset, FileItem, Layer, Project +from uvdat.core.tasks.frame_preview import ( + ensure_default_layer_style, + generate_layer_style_previews, + is_multiframe_raster_layer, +) DATA_FOLDER = Path(os.environ.get("INGEST_BIND_MOUNT_POINT", "sample_data")) DOWNLOADS_FOLDER = DATA_FOLDER / "downloads" @@ -152,6 +157,24 @@ def ingest_file(file_info, *, index=0, dataset=None, chart=None, replace=False, new_file_item.file.save(file_path, File(f)) +def generate_ingest_multiframe_previews(converted_dataset_names: set[str]) -> int: + if not converted_dataset_names: + return 0 + style_count = 0 + for project in Project.objects.filter(datasets__name__in=converted_dataset_names).distinct(): + layers = Layer.objects.filter( + dataset__name__in=converted_dataset_names, + dataset__in=project.datasets.all(), + ) + for layer in layers: + if not is_multiframe_raster_layer(layer): + continue + style = ensure_default_layer_style(layer, project) + generate_layer_style_previews(style.id) + style_count += 1 + return style_count + + def ingest_projects(data: list[ProjectItem], *, replace=False) -> None: for project in data: click.echo(f"\t- {project['name']}") @@ -264,7 +287,8 @@ def default_conversion_process(dataset: Dataset, options: DatasetItem): def ingest_datasets( data: list[DatasetItem], json_file_path: Path, *, replace=False, skip_cache=False -) -> None: +) -> set[str]: + converted_dataset_names: set[str] = set() superuser = User.objects.filter(is_superuser=True).first() if superuser is None: raise click.ClickException("Please create at least one superuser") @@ -320,6 +344,7 @@ def ingest_datasets( f"\t\t Dataset {dataset_for_conversion.name} converted.", fg="green", ) + converted_dataset_names.add(dataset["name"]) else: click.secho( f"\t\t Dataset {dataset['name']} already exists, not importing/converting", @@ -329,6 +354,8 @@ def ingest_datasets( dataset_for_conversion.set_tags(dataset.get("tags")) dataset_for_conversion.set_owner(superuser) + return converted_dataset_names + @click.command() @click.argument("file_path") @@ -385,10 +412,16 @@ def ingest(*, file_path, replace, clear, skip_cache): elif item["type"] == "Chart": charts.append(item) click.echo("Ingesting Datasets:") - ingest_datasets(datasets, file_path, replace=replace, skip_cache=skip_cache) + converted_dataset_names = ingest_datasets( + datasets, file_path, replace=replace, skip_cache=skip_cache + ) click.echo("Ingesting Projects:") ingest_projects(projects, replace=replace) click.echo("Ingesting Charts:") ingest_charts(charts, replace=replace, skip_cache=skip_cache) + preview_count = generate_ingest_multiframe_previews(converted_dataset_names) + if preview_count: + click.echo(f"Generated multiframe raster previews for {preview_count} layer style(s).") + click.secho("Ingestion complete.", fg="green") From 15c20e79a5c5699c7c7d3f2d706a43c088ce299d Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 12 Jun 2026 15:05:28 -0400 Subject: [PATCH 09/36] linting --- uvdat/core/models/__init__.py | 2 +- uvdat/core/raster_style.py | 6 +- uvdat/core/tests/test_frame_preview.py | 77 ++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 6 deletions(-) create mode 100644 uvdat/core/tests/test_frame_preview.py diff --git a/uvdat/core/models/__init__.py b/uvdat/core/models/__init__.py index e3fcf6305..0e4672d76 100644 --- a/uvdat/core/models/__init__.py +++ b/uvdat/core/models/__init__.py @@ -35,12 +35,12 @@ "Layer", "LayerFrame", "LayerStyle", - "RasterFramePreview", "Network", "NetworkEdge", "NetworkNode", "Project", "RasterData", + "RasterFramePreview", "Region", "SizeConfig", "SizeRangeConfig", diff --git a/uvdat/core/raster_style.py b/uvdat/core/raster_style.py index b37f62179..0b1dc3118 100644 --- a/uvdat/core/raster_style.py +++ b/uvdat/core/raster_style.py @@ -36,11 +36,7 @@ def colormap_markers_subsample( n: int | None = None, ) -> list[dict[str, Any]]: markers = list(colormap.markers) - if ( - n is None - and applied_colormap.get("discrete") - and applied_colormap.get("n_colors") - ): + if n is None and applied_colormap.get("discrete") and applied_colormap.get("n_colors"): n = applied_colormap["n_colors"] if n and markers: while n > len(markers): diff --git a/uvdat/core/tests/test_frame_preview.py b/uvdat/core/tests/test_frame_preview.py new file mode 100644 index 000000000..bb1522be6 --- /dev/null +++ b/uvdat/core/tests/test_frame_preview.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import pytest + +from uvdat.core.raster_style import ( + apply_source_filters_to_style_query, + build_thumbnail_style_query, + is_default_band_source_filter, + raster_source_filter_kwargs, +) +from uvdat.core.tasks.frame_preview import ( + FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION, + FRAME_PREVIEW_MAX_PX, + FRAME_PREVIEW_MIN_PX, + resolve_preview_max_dimension, +) + + +@pytest.mark.parametrize( + ("source_filters", "expected"), + [ + ({}, False), + ({"band": 1}, True), + ({"band": "1"}, True), + ({"frame": 0}, False), + ({"band": 2}, False), + ], +) +def test_is_default_band_source_filter(source_filters, expected): + assert is_default_band_source_filter(source_filters) is expected + + +def test_build_thumbnail_style_query_does_not_embed_frame(): + style_spec = { + "colors": [{"name": "all", "visible": True, "single_color": "#ffffff"}], + "filters": [], + } + query = build_thumbnail_style_query(style_spec, {"frame": 3}, {}) + assert "frame" not in query + assert query == {"palette": "#ffffff"} + + +def test_raster_source_filter_kwargs_extracts_frame(): + assert raster_source_filter_kwargs({"frame": 3}) == {"frame": 3} + assert raster_source_filter_kwargs({"band": 2}) == {} + assert raster_source_filter_kwargs({"frame": 1, "band": 2}) == {"frame": 1} + + +def test_apply_source_filters_to_style_query_embeds_band_not_frame(): + query = apply_source_filters_to_style_query({}, {"frame": 3, "band": 2}) + assert query == {"band": 2} + assert apply_source_filters_to_style_query({}, {"frame": 3, "band": 1}) == {} + + +@pytest.mark.parametrize( + ("resolution_fraction", "raster_max_dimension", "expected_max_px"), + [ + (None, None, round(FRAME_PREVIEW_MAX_PX * FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION)), + (0.25, None, 1024), + (0.5, None, 2048), + (FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION, None, 512), + (None, 4096, FRAME_PREVIEW_MIN_PX), + (0.25, 4096, FRAME_PREVIEW_MIN_PX), + (0.5, 4096, 2048), + (None, 800, 800), + (0.5, 500, 500), + (0.75, None, 3072), + ], +) +def test_resolve_preview_max_dimension( + resolution_fraction, + raster_max_dimension, + expected_max_px, +): + assert ( + resolve_preview_max_dimension(resolution_fraction, raster_max_dimension) == expected_max_px + ) From 9736bca0d171e99f578181c32cfbcd28c0662afb Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 12 Jun 2026 15:18:24 -0400 Subject: [PATCH 10/36] allow band: 1 when explicitly defined --- uvdat/core/raster_style.py | 14 +------------- uvdat/core/tests/test_frame_preview.py | 18 ++---------------- 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/uvdat/core/raster_style.py b/uvdat/core/raster_style.py index 0b1dc3118..4a2a5d317 100644 --- a/uvdat/core/raster_style.py +++ b/uvdat/core/raster_style.py @@ -55,14 +55,6 @@ def colormap_markers_subsample( return markers -def is_default_band_source_filter(source_filters: dict[str, Any] | None) -> bool: - if not source_filters or len(source_filters) != 1: - return False - if "band" not in source_filters: - return False - return source_filters["band"] in (1, "1") - - def _build_color_query_from_spec( color_spec: dict[str, Any], colormaps_by_id: dict[int, Colormap], @@ -133,11 +125,7 @@ def apply_source_filters_to_style_query( source_filters: dict[str, Any] | None, ) -> dict[str, Any]: query = dict(base_query) - if ( - source_filters - and not is_default_band_source_filter(source_filters) - and "band" in source_filters - ): + if source_filters and "band" in source_filters: query["band"] = source_filters["band"] return query diff --git a/uvdat/core/tests/test_frame_preview.py b/uvdat/core/tests/test_frame_preview.py index bb1522be6..2625c8785 100644 --- a/uvdat/core/tests/test_frame_preview.py +++ b/uvdat/core/tests/test_frame_preview.py @@ -5,7 +5,6 @@ from uvdat.core.raster_style import ( apply_source_filters_to_style_query, build_thumbnail_style_query, - is_default_band_source_filter, raster_source_filter_kwargs, ) from uvdat.core.tasks.frame_preview import ( @@ -16,20 +15,6 @@ ) -@pytest.mark.parametrize( - ("source_filters", "expected"), - [ - ({}, False), - ({"band": 1}, True), - ({"band": "1"}, True), - ({"frame": 0}, False), - ({"band": 2}, False), - ], -) -def test_is_default_band_source_filter(source_filters, expected): - assert is_default_band_source_filter(source_filters) is expected - - def test_build_thumbnail_style_query_does_not_embed_frame(): style_spec = { "colors": [{"name": "all", "visible": True, "single_color": "#ffffff"}], @@ -49,7 +34,8 @@ def test_raster_source_filter_kwargs_extracts_frame(): def test_apply_source_filters_to_style_query_embeds_band_not_frame(): query = apply_source_filters_to_style_query({}, {"frame": 3, "band": 2}) assert query == {"band": 2} - assert apply_source_filters_to_style_query({}, {"frame": 3, "band": 1}) == {} + assert apply_source_filters_to_style_query({}, {"frame": 3, "band": 1}) == {"band": 1} + assert apply_source_filters_to_style_query({}, {"band": 1}) == {"band": 1} @pytest.mark.parametrize( From d71eee93baa799acca46848e0f8ea83fd99dfe8e Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Sun, 14 Jun 2026 16:24:57 -0400 Subject: [PATCH 11/36] remove status from RasterFramePreview model and update migration --- uvdat/core/admin.py | 2 +- uvdat/core/migrations/0024_raster_frame_preview.py | 12 ------------ uvdat/core/models/frame_preview.py | 10 ---------- uvdat/core/tasks/frame_preview.py | 9 --------- 4 files changed, 1 insertion(+), 32 deletions(-) diff --git a/uvdat/core/admin.py b/uvdat/core/admin.py index e98e1c1c5..7a049c0d4 100644 --- a/uvdat/core/admin.py +++ b/uvdat/core/admin.py @@ -72,7 +72,7 @@ class LayerStyleAdmin(admin.ModelAdmin): @admin.register(RasterFramePreview) class RasterFramePreviewAdmin(admin.ModelAdmin): - list_display = ["id", "layer_style", "layer_frame", "status", "width", "height"] + list_display = ["id", "layer_style", "layer_frame", "width", "height"] list_select_related = ["layer_style", "layer_frame"] diff --git a/uvdat/core/migrations/0024_raster_frame_preview.py b/uvdat/core/migrations/0024_raster_frame_preview.py index 260540508..ced146b13 100644 --- a/uvdat/core/migrations/0024_raster_frame_preview.py +++ b/uvdat/core/migrations/0024_raster_frame_preview.py @@ -25,18 +25,6 @@ class Migration(migrations.Migration): ("width", models.PositiveIntegerField()), ("height", models.PositiveIntegerField()), ("bounds", models.JSONField()), - ( - "status", - models.CharField( - choices=[ - ("pending", "Pending"), - ("ready", "Ready"), - ("failed", "Failed"), - ], - default="pending", - max_length=10, - ), - ), ( "layer_frame", models.ForeignKey( diff --git a/uvdat/core/models/frame_preview.py b/uvdat/core/models/frame_preview.py index 15bf2dc7d..f0051ab37 100644 --- a/uvdat/core/models/frame_preview.py +++ b/uvdat/core/models/frame_preview.py @@ -9,11 +9,6 @@ class RasterFramePreview(models.Model): - class Status(models.TextChoices): - PENDING = "pending", "Pending" - READY = "ready", "Ready" - FAILED = "failed", "Failed" - layer_style = models.ForeignKey( LayerStyle, related_name="frame_previews", @@ -28,11 +23,6 @@ class Status(models.TextChoices): width = models.PositiveIntegerField() height = models.PositiveIntegerField() bounds = models.JSONField() - status = models.CharField( - max_length=10, - choices=Status.choices, - default=Status.PENDING, - ) class Meta: constraints = [ diff --git a/uvdat/core/tasks/frame_preview.py b/uvdat/core/tasks/frame_preview.py index 263d21b44..c4359aaab 100644 --- a/uvdat/core/tasks/frame_preview.py +++ b/uvdat/core/tasks/frame_preview.py @@ -174,10 +174,6 @@ def generate_layer_style_previews( ready_count = 0 failed_count = 0 for frame in frames: - existing = RasterFramePreview.objects.filter( - layer_style=layer_style, - layer_frame=frame, - ).first() try: png_bytes, width, height, bounds = generate_frame_preview_png( frame.raster, @@ -192,7 +188,6 @@ def generate_layer_style_previews( "width": width, "height": height, "bounds": bounds or {}, - "status": RasterFramePreview.Status.READY, }, ) preview.image.save( @@ -208,10 +203,6 @@ def generate_layer_style_previews( layer_style_id, frame.id, ) - if existing is not None: - existing.status = RasterFramePreview.Status.FAILED - existing.save(update_fields=["status"]) - logger.info( "Multiframe raster previews for style=%s layer=%r: %d ready, %d failed in %.2fs", layer_style_id, From 4afe7f51bdd300275d7174b76b7701687445e900 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Sun, 14 Jun 2026 17:06:39 -0400 Subject: [PATCH 12/36] move checks for multiframe and layers into models --- uvdat/core/management/commands/ingest.py | 3 +-- uvdat/core/models/layer.py | 14 ++++++++++++++ uvdat/core/models/styles.py | 16 ++++++++++++++++ uvdat/core/tasks/frame_preview.py | 15 +-------------- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/uvdat/core/management/commands/ingest.py b/uvdat/core/management/commands/ingest.py index 69637404e..f3fe30f68 100644 --- a/uvdat/core/management/commands/ingest.py +++ b/uvdat/core/management/commands/ingest.py @@ -21,7 +21,6 @@ from uvdat.core.tasks.frame_preview import ( ensure_default_layer_style, generate_layer_style_previews, - is_multiframe_raster_layer, ) DATA_FOLDER = Path(os.environ.get("INGEST_BIND_MOUNT_POINT", "sample_data")) @@ -167,7 +166,7 @@ def generate_ingest_multiframe_previews(converted_dataset_names: set[str]) -> in dataset__in=project.datasets.all(), ) for layer in layers: - if not is_multiframe_raster_layer(layer): + if not layer.is_multiframe_raster(): continue style = ensure_default_layer_style(layer, project) generate_layer_style_previews(style.id) diff --git a/uvdat/core/models/layer.py b/uvdat/core/models/layer.py index 88489d9b1..ed8960fe7 100644 --- a/uvdat/core/models/layer.py +++ b/uvdat/core/models/layer.py @@ -25,6 +25,20 @@ class Layer(models.Model): def __str__(self): return f"{self.name} ({self.id})" + def multiframe_raster_frames(self): + prefetched = getattr(self, "raster_frames", None) + if prefetched is not None: + return prefetched + return list( + self.frames.filter(raster__isnull=False).select_related("raster").order_by("index") + ) + + def is_multiframe_raster(self) -> bool: + prefetched = getattr(self, "raster_frames", None) + if prefetched is not None: + return len(prefetched) > 1 + return self.frames.filter(raster__isnull=False).count() > 1 + class LayerFrame(models.Model): name = models.CharField(max_length=255, default="Layer Frame") diff --git a/uvdat/core/models/styles.py b/uvdat/core/models/styles.py index 385d1cabb..fc57835df 100644 --- a/uvdat/core/models/styles.py +++ b/uvdat/core/models/styles.py @@ -224,6 +224,22 @@ def repr_style_configs(self): "filters": filters, } + def multiframe_preview_urls(self, layer=None) -> list[str | None] | None: + layer = layer or self.layer + frames = layer.multiframe_raster_frames() + if len(frames) <= 1: + return None + + previews_by_frame_id = { + preview.layer_frame_id: preview for preview in self.frame_previews.all() + } + return [ + preview.image.url + if (preview := previews_by_frame_id.get(frame.id)) and preview.image + else None + for frame in frames + ] + def get_default_colormap(): return Colormap.objects.filter(project__isnull=True).first() diff --git a/uvdat/core/tasks/frame_preview.py b/uvdat/core/tasks/frame_preview.py index c4359aaab..050a32e32 100644 --- a/uvdat/core/tasks/frame_preview.py +++ b/uvdat/core/tasks/frame_preview.py @@ -14,7 +14,6 @@ from uvdat.core.models import ( Colormap, Layer, - LayerFrame, LayerStyle, Project, RasterData, @@ -126,18 +125,6 @@ def generate_frame_preview_png( return png_bytes, image.width, image.height, _preview_bounds(source) -def is_multiframe_raster_layer(layer: Layer) -> bool: - return LayerFrame.objects.filter(layer=layer, raster__isnull=False).count() > 1 - - -def _multiframe_raster_frames(layer_style: LayerStyle): - return ( - LayerFrame.objects.filter(layer=layer_style.layer, raster__isnull=False) - .select_related("raster") - .order_by("index") - ) - - def ensure_default_layer_style(layer: Layer, project: Project) -> LayerStyle: style = LayerStyle.objects.filter(layer=layer, project=project, name="Default").first() if style is None: @@ -156,7 +143,7 @@ def generate_layer_style_previews( ): started = time.perf_counter() layer_style = LayerStyle.objects.select_related("layer").get(id=layer_style_id) - frames = list(_multiframe_raster_frames(layer_style)) + frames = layer_style.layer.multiframe_raster_frames() if len(frames) <= 1: return From 3cb20018807e9868a1556bb2d0ab777a5f72442c Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Sun, 14 Jun 2026 17:44:17 -0400 Subject: [PATCH 13/36] layer/layerstyle serializer multiframe_preview_urls data --- uvdat/core/models/layer.py | 5 ++++ uvdat/core/rest/dataset.py | 3 +- uvdat/core/rest/layer.py | 15 ++++++---- uvdat/core/rest/querysets.py | 50 ++++++++++++++++++++++++++++++++++ uvdat/core/rest/serializers.py | 31 ++++++++++++++++++++- 5 files changed, 96 insertions(+), 8 deletions(-) create mode 100644 uvdat/core/rest/querysets.py diff --git a/uvdat/core/models/layer.py b/uvdat/core/models/layer.py index ed8960fe7..e87d83031 100644 --- a/uvdat/core/models/layer.py +++ b/uvdat/core/models/layer.py @@ -39,6 +39,11 @@ def is_multiframe_raster(self) -> bool: return len(prefetched) > 1 return self.frames.filter(raster__isnull=False).count() > 1 + def default_multiframe_preview_urls(self) -> list[str | None] | None: + if self.default_style_id is None: + return None + return self.default_style.multiframe_preview_urls(layer=self) + class LayerFrame(models.Model): name = models.CharField(max_length=255, default="Layer Frame") diff --git a/uvdat/core/rest/dataset.py b/uvdat/core/rest/dataset.py index 65d5a1adb..f0f2ea2d5 100644 --- a/uvdat/core/rest/dataset.py +++ b/uvdat/core/rest/dataset.py @@ -6,6 +6,7 @@ from uvdat.core.access_control import DatasetGuardianPermission from uvdat.core.models import Dataset, DatasetTag, Network +from uvdat.core.rest.querysets import layer_queryset_with_previews from uvdat.core.rest.serializers import ( DatasetSerializer, FileItemSerializer, @@ -43,7 +44,7 @@ def tags(self, request, **kwargs): @action(detail=True, methods=["get"]) def layers(self, request, **kwargs): dataset: Dataset = self.get_object() - layers = list(dataset.layers.all()) + layers = list(layer_queryset_with_previews().filter(dataset=dataset)) serializer = LayerSerializer(layers, many=True) return Response(serializer.data, status=200) diff --git a/uvdat/core/rest/layer.py b/uvdat/core/rest/layer.py index 95e12c217..33ff4f076 100644 --- a/uvdat/core/rest/layer.py +++ b/uvdat/core/rest/layer.py @@ -7,17 +7,20 @@ from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet from uvdat.core.models import Layer, LayerFrame, LayerStyle +from uvdat.core.rest.querysets import layer_queryset_with_previews from uvdat.core.rest.serializers import ( LayerFrameSerializer, LayerSerializer, - LayerStyleSerializer, + LayerStyleWithPreviewsSerializer, ) class LayerViewSet(ReadOnlyModelViewSet): - queryset = Layer.objects.select_related("dataset").all() serializer_class = LayerSerializer + def get_queryset(self): + return layer_queryset_with_previews() + @action(detail=True, methods=["get"]) def frames(self, request, **kwargs): layer: Layer = self.get_object() @@ -33,7 +36,7 @@ class LayerFrameViewSet(ReadOnlyModelViewSet): class LayerStyleViewSet(ModelViewSet): queryset = LayerStyle.objects.all() - serializer_class = LayerStyleSerializer + serializer_class = LayerStyleWithPreviewsSerializer def get_queryset(self): qs = super().get_queryset() @@ -43,11 +46,11 @@ def get_queryset(self): layer_id = int(self.request.query_params.get("layer", -1)) if layer_id > -1: qs = qs.filter(layer=int(layer_id)) - return qs + return layer_queryset_with_previews(qs, for_layer_style=True) def create(self, request, **kwargs): is_default = request.data.pop("is_default", False) - serializer = LayerStyleSerializer(data=request.data) + serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) with transaction.atomic(): try: @@ -62,7 +65,7 @@ def create(self, request, **kwargs): def partial_update(self, request, **kwargs): instance = self.get_object() is_default = request.data.pop("is_default", False) - serializer = LayerStyleSerializer(instance, data=request.data, partial=True) + serializer = self.get_serializer(instance, data=request.data, partial=True) serializer.is_valid(raise_exception=True) with transaction.atomic(): try: diff --git a/uvdat/core/rest/querysets.py b/uvdat/core/rest/querysets.py new file mode 100644 index 000000000..bf567cd48 --- /dev/null +++ b/uvdat/core/rest/querysets.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from django.db.models import Prefetch, QuerySet + +from uvdat.core.models import Layer, LayerFrame, LayerStyle, RasterFramePreview + +RASTER_FRAMES_QUERYSET = ( + LayerFrame.objects.filter(raster__isnull=False).select_related("raster").order_by("index") +) + +FRAME_PREVIEWS_PREFETCH = Prefetch( + "frame_previews", + queryset=RasterFramePreview.objects.select_related("layer_frame"), +) + +LAYER_RASTER_FRAMES_PREFETCH = Prefetch( + "frames", + queryset=RASTER_FRAMES_QUERYSET, + to_attr="raster_frames", +) + +STYLE_LAYER_RASTER_FRAMES_PREFETCH = Prefetch( + "layer__frames", + queryset=RASTER_FRAMES_QUERYSET, + to_attr="raster_frames", +) + + +def layer_queryset_with_previews( + queryset: QuerySet | None = None, + *, + for_layer_style: bool = False, +) -> QuerySet: + # This is for /layer-styles/?layer={layer_id} so we need to prefetch the frames for all styles + if for_layer_style: + qs = queryset if queryset is not None else LayerStyle.objects.all() + return qs.select_related("layer", "layer__default_style").prefetch_related( + FRAME_PREVIEWS_PREFETCH, + STYLE_LAYER_RASTER_FRAMES_PREFETCH, + ) + + qs = queryset if queryset is not None else Layer.objects.all() + # This is for /layers/{layer_id} so we need to prefetch the frames and the default style + return qs.select_related("dataset", "default_style").prefetch_related( + LAYER_RASTER_FRAMES_PREFETCH, + Prefetch( + "default_style__frame_previews", + queryset=RasterFramePreview.objects.select_related("layer_frame"), + ), + ) diff --git a/uvdat/core/rest/serializers.py b/uvdat/core/rest/serializers.py index d255d1e14..89cfe32f8 100644 --- a/uvdat/core/rest/serializers.py +++ b/uvdat/core/rest/serializers.py @@ -149,6 +149,11 @@ class Meta: fields = "__all__" +def _omit_null_field(data: dict, field: str) -> None: + if data.get(field) is None: + data.pop(field, None) + + class LayerStyleSerializer(serializers.ModelSerializer): is_default = serializers.SerializerMethodField("get_is_default") @@ -178,12 +183,36 @@ class Meta: exclude = ["default_frame", "opacity"] +class LayerStyleWithPreviewsSerializer(LayerStyleSerializer): + multiframe_preview_urls = serializers.SerializerMethodField() + + def get_multiframe_preview_urls(self, obj): + preview_layer = self.context.get("preview_layer") + if preview_layer is not None: + return obj.multiframe_preview_urls(layer=preview_layer) + return obj.multiframe_preview_urls() + + def to_representation(self, instance): + data = super().to_representation(instance) + _omit_null_field(data, "multiframe_preview_urls") + return data + + class LayerSerializer(serializers.ModelSerializer): default_style = LayerStyleSerializer() + multiframe_preview_urls = serializers.SerializerMethodField() + + def get_multiframe_preview_urls(self, obj): + return obj.default_multiframe_preview_urls() + + def to_representation(self, instance): + data = super().to_representation(instance) + _omit_null_field(data, "multiframe_preview_urls") + return data class Meta: model = Layer - fields = ["id", "name", "metadata", "dataset", "default_style"] + fields = ["id", "name", "metadata", "dataset", "default_style", "multiframe_preview_urls"] class VectorDataSerializer(serializers.ModelSerializer): From 17f359895b38082aa3d0bb0854b45bec1842841d Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 19 Jun 2026 08:41:44 -0400 Subject: [PATCH 14/36] store georeferenced bounds for previews --- uvdat/core/tasks/frame_preview.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/uvdat/core/tasks/frame_preview.py b/uvdat/core/tasks/frame_preview.py index 050a32e32..ef9d5802a 100644 --- a/uvdat/core/tasks/frame_preview.py +++ b/uvdat/core/tasks/frame_preview.py @@ -86,13 +86,18 @@ def _preview_bounds(source) -> dict[str, Any] | None: bounds = tilesource.get_bounds(source, projection="EPSG:4326") if not bounds: return None - return { + result: dict[str, Any] = { "srs": "EPSG:4326", "xmin": bounds["xmin"], "xmax": bounds["xmax"], "ymin": bounds["ymin"], "ymax": bounds["ymax"], } + for corner in ("ul", "ur", "lr", "ll"): + corner_bounds = bounds.get(corner) + if corner_bounds: + result[corner] = {"x": corner_bounds["x"], "y": corner_bounds["y"]} + return result def generate_frame_preview_png( From af7e7cd010c9ab0d3ed86d55e1385deacfc01db1 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 19 Jun 2026 09:00:57 -0400 Subject: [PATCH 15/36] restructure for preview objects with bounds and urls --- uvdat/core/frame_preview_types.py | 27 ++++ uvdat/core/models/layer.py | 9 +- uvdat/core/models/styles.py | 32 ++++- uvdat/core/rest/serializers.py | 31 +++-- uvdat/core/tasks/frame_preview.py | 11 +- uvdat/core/tests/test_frame_preview.py | 168 +++++++++++++++++++++++++ 6 files changed, 255 insertions(+), 23 deletions(-) create mode 100644 uvdat/core/frame_preview_types.py diff --git a/uvdat/core/frame_preview_types.py b/uvdat/core/frame_preview_types.py new file mode 100644 index 000000000..632f892ad --- /dev/null +++ b/uvdat/core/frame_preview_types.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import TypedDict + + +class FramePreviewCorner(TypedDict): + x: float + y: float + + +class FramePreviewBounds(TypedDict, total=False): + srs: str # EPSG:4326 + xmin: float + xmax: float + ymin: float + ymax: float + ul: FramePreviewCorner + ur: FramePreviewCorner + lr: FramePreviewCorner + ll: FramePreviewCorner + + +class FramePreviewData(TypedDict): + url: str # presigned URL of the preview image + width: int + height: int + bounds: FramePreviewBounds diff --git a/uvdat/core/models/layer.py b/uvdat/core/models/layer.py index e87d83031..1eeb6ddb1 100644 --- a/uvdat/core/models/layer.py +++ b/uvdat/core/models/layer.py @@ -1,11 +1,16 @@ from __future__ import annotations +from typing import TYPE_CHECKING + from django.db import models from .data import RasterData, VectorData from .dataset import Dataset from .querysets import ProjectQuerySet +if TYPE_CHECKING: + from uvdat.core.frame_preview_types import FramePreviewData + def default_source_filters(): return {} @@ -39,10 +44,10 @@ def is_multiframe_raster(self) -> bool: return len(prefetched) > 1 return self.frames.filter(raster__isnull=False).count() > 1 - def default_multiframe_preview_urls(self) -> list[str | None] | None: + def default_multiframe_previews(self) -> list[FramePreviewData | None] | None: if self.default_style_id is None: return None - return self.default_style.multiframe_preview_urls(layer=self) + return self.default_style.multiframe_previews(layer=self) class LayerFrame(models.Model): diff --git a/uvdat/core/models/styles.py b/uvdat/core/models/styles.py index fc57835df..e129a1929 100644 --- a/uvdat/core/models/styles.py +++ b/uvdat/core/models/styles.py @@ -1,16 +1,21 @@ from __future__ import annotations import contextlib -from typing import Any +from typing import TYPE_CHECKING, Any from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models +from uvdat.core.frame_preview_types import FramePreviewData + from .colormap import Colormap from .layer import Layer from .project import Project from .querysets import ProjectQuerySet +if TYPE_CHECKING: + from .frame_preview import RasterFramePreview + class LayerStyle(models.Model): name = models.CharField(max_length=255, default="Layer Style") @@ -224,7 +229,7 @@ def repr_style_configs(self): "filters": filters, } - def multiframe_preview_urls(self, layer=None) -> list[str | None] | None: + def _multiframe_previews_by_frame(self, layer=None) -> list[RasterFramePreview | None] | None: layer = layer or self.layer frames = layer.multiframe_raster_frames() if len(frames) <= 1: @@ -234,12 +239,29 @@ def multiframe_preview_urls(self, layer=None) -> list[str | None] | None: preview.layer_frame_id: preview for preview in self.frame_previews.all() } return [ - preview.image.url - if (preview := previews_by_frame_id.get(frame.id)) and preview.image - else None + previews_by_frame_id.get(frame.id) if frame.id in previews_by_frame_id else None for frame in frames ] + @staticmethod + def serialize_frame_preview( + preview: RasterFramePreview | None, + ) -> FramePreviewData | None: + if preview is None or not preview.image: + return None + return FramePreviewData( + url=preview.image.url, + width=preview.width, + height=preview.height, + bounds=preview.bounds, + ) + + def multiframe_previews(self, layer=None) -> list[FramePreviewData | None] | None: + previews = self._multiframe_previews_by_frame(layer=layer) + if previews is None: + return None + return [self.serialize_frame_preview(preview) for preview in previews] + def get_default_colormap(): return Colormap.objects.filter(project__isnull=True).first() diff --git a/uvdat/core/rest/serializers.py b/uvdat/core/rest/serializers.py index 89cfe32f8..8a6aa165e 100644 --- a/uvdat/core/rest/serializers.py +++ b/uvdat/core/rest/serializers.py @@ -184,35 +184,42 @@ class Meta: class LayerStyleWithPreviewsSerializer(LayerStyleSerializer): - multiframe_preview_urls = serializers.SerializerMethodField() + multiframe_previews = serializers.SerializerMethodField() - def get_multiframe_preview_urls(self, obj): - preview_layer = self.context.get("preview_layer") - if preview_layer is not None: - return obj.multiframe_preview_urls(layer=preview_layer) - return obj.multiframe_preview_urls() + def _preview_layer(self, obj): + return self.context.get("preview_layer") or obj.layer + + def get_multiframe_previews(self, obj): + return obj.multiframe_previews(layer=self._preview_layer(obj)) def to_representation(self, instance): data = super().to_representation(instance) - _omit_null_field(data, "multiframe_preview_urls") + _omit_null_field(data, "multiframe_previews") return data class LayerSerializer(serializers.ModelSerializer): default_style = LayerStyleSerializer() - multiframe_preview_urls = serializers.SerializerMethodField() + multiframe_previews = serializers.SerializerMethodField() - def get_multiframe_preview_urls(self, obj): - return obj.default_multiframe_preview_urls() + def get_multiframe_previews(self, obj): + return obj.default_multiframe_previews() def to_representation(self, instance): data = super().to_representation(instance) - _omit_null_field(data, "multiframe_preview_urls") + _omit_null_field(data, "multiframe_previews") return data class Meta: model = Layer - fields = ["id", "name", "metadata", "dataset", "default_style", "multiframe_preview_urls"] + fields = [ + "id", + "name", + "metadata", + "dataset", + "default_style", + "multiframe_previews", + ] class VectorDataSerializer(serializers.ModelSerializer): diff --git a/uvdat/core/tasks/frame_preview.py b/uvdat/core/tasks/frame_preview.py index ef9d5802a..64917f9c4 100644 --- a/uvdat/core/tasks/frame_preview.py +++ b/uvdat/core/tasks/frame_preview.py @@ -4,7 +4,7 @@ import json import logging import time -from typing import Any +from typing import TYPE_CHECKING, Any from celery import shared_task from django.core.files.base import ContentFile @@ -25,6 +25,9 @@ raster_source_filter_kwargs, ) +if TYPE_CHECKING: + from uvdat.core.frame_preview_types import FramePreviewBounds + logger = logging.getLogger(__name__) FRAME_PREVIEW_MAX_PX = 4096 @@ -82,11 +85,11 @@ def _thumbnail_png_bytes(thumb_data: Any) -> bytes: raise TypeError(msg) -def _preview_bounds(source) -> dict[str, Any] | None: +def _preview_bounds(source) -> FramePreviewBounds | None: bounds = tilesource.get_bounds(source, projection="EPSG:4326") if not bounds: return None - result: dict[str, Any] = { + result: FramePreviewBounds = { "srs": "EPSG:4326", "xmin": bounds["xmin"], "xmax": bounds["xmax"], @@ -105,7 +108,7 @@ def generate_frame_preview_png( source_filters: dict[str, Any] | None, base_style_query: dict[str, Any], resolution_fraction: float | None = None, -) -> tuple[bytes, int, int, dict[str, Any] | None]: +) -> tuple[bytes, int, int, FramePreviewBounds | None]: style_query = apply_source_filters_to_style_query(base_style_query, source_filters) style = json.dumps(style_query) if style_query else None source_kwargs = raster_source_filter_kwargs(source_filters) diff --git a/uvdat/core/tests/test_frame_preview.py b/uvdat/core/tests/test_frame_preview.py index 2625c8785..b84b4b078 100644 --- a/uvdat/core/tests/test_frame_preview.py +++ b/uvdat/core/tests/test_frame_preview.py @@ -1,7 +1,9 @@ from __future__ import annotations +from django.core.files.base import ContentFile import pytest +from uvdat.core.models import RasterFramePreview from uvdat.core.raster_style import ( apply_source_filters_to_style_query, build_thumbnail_style_query, @@ -61,3 +63,169 @@ def test_resolve_preview_max_dimension( assert ( resolve_preview_max_dimension(resolution_fraction, raster_max_dimension) == expected_max_px ) + + +@pytest.mark.django_db +def test_multiframe_previews_for_style_returns_none_for_single_frame( + layer_style_factory, + layer_frame_factory, +): + layer_style = layer_style_factory() + layer_frame_factory(layer=layer_style.layer, index=0) + + assert layer_style.multiframe_previews() is None + + +@pytest.mark.django_db +def test_multiframe_previews_for_style_ordered_by_frame_index( + layer_style_factory, + layer_frame_factory, +): + layer_style = layer_style_factory() + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + layer_frame_factory(layer=layer_style.layer, index=1) + frame_2 = layer_frame_factory(layer=layer_style.layer, index=2) + + preview_0 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_0, + width=100, + height=80, + bounds={"srs": "EPSG:4326", "xmin": -1, "xmax": 1, "ymin": -2, "ymax": 2}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_2 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_2, + width=200, + height=150, + bounds={"srs": "EPSG:4326", "xmin": -3, "xmax": 3, "ymin": -4, "ymax": 4}, + ) + preview_2.image.save("frame-2.png", ContentFile(b"png2"), save=True) + + previews = layer_style.multiframe_previews() + assert previews == [ + { + "url": preview_0.image.url, + "width": 100, + "height": 80, + "bounds": preview_0.bounds, + }, + None, + { + "url": preview_2.image.url, + "width": 200, + "height": 150, + "bounds": preview_2.bounds, + }, + ] + + +@pytest.mark.django_db +def test_preview_bounds_includes_corners( + layer_style_factory, + layer_frame_factory, +): + layer_style = layer_style_factory() + frame = layer_frame_factory(layer=layer_style.layer, index=0) + layer_frame_factory(layer=layer_style.layer, index=1) + + preview = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame, + width=100, + height=80, + bounds={ + "srs": "EPSG:4326", + "xmin": -1, + "xmax": 1, + "ymin": -2, + "ymax": 2, + "ul": {"x": -1, "y": 2}, + "ur": {"x": 1, "y": 2}, + "lr": {"x": 1, "y": -2}, + "ll": {"x": -1, "y": -2}, + }, + ) + preview.image.save("frame-0.png", ContentFile(b"png0"), save=True) + + previews = layer_style.multiframe_previews() + assert previews[0]["bounds"]["ul"] == {"x": -1, "y": 2} + + +@pytest.mark.django_db +def test_layer_style_api_includes_multiframe_previews( + authenticated_api_client, + layer_style_factory, + layer_frame_factory, + project, + user, +): + layer_style = layer_style_factory() + project.set_collaborators([user]) + project.datasets.set([layer_style.layer.dataset]) + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + layer_frame_factory(layer=layer_style.layer, index=1) + + preview = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_0, + width=100, + height=100, + bounds={}, + ) + preview.image.save("frame-0.png", ContentFile(b"png0"), save=True) + + resp = authenticated_api_client.get(f"/api/v1/layer-styles/{layer_style.id}/") + assert resp.status_code == 200 + data = resp.json() + assert data["multiframe_previews"] == [ + { + "url": preview.image.url, + "width": 100, + "height": 100, + "bounds": {}, + }, + None, + ] + + +@pytest.mark.django_db +def test_layer_api_includes_multiframe_previews( + authenticated_api_client, + layer_style_factory, + layer_frame_factory, + project, + user, +): + layer_style = layer_style_factory() + layer = layer_style.layer + layer.default_style = layer_style + layer.save(update_fields=["default_style"]) + project.set_collaborators([user]) + project.datasets.set([layer.dataset]) + frame_0 = layer_frame_factory(layer=layer, index=0) + layer_frame_factory(layer=layer, index=1) + + preview = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_0, + width=100, + height=100, + bounds={}, + ) + preview.image.save("frame-0.png", ContentFile(b"png0"), save=True) + + resp = authenticated_api_client.get(f"/api/v1/layers/{layer.id}/") + assert resp.status_code == 200 + data = resp.json() + assert data["multiframe_previews"] == [ + { + "url": preview.image.url, + "width": 100, + "height": 100, + "bounds": {}, + }, + None, + ] + assert "multiframe_previews" not in data.get("default_style", {}) From 25d27e991bc03ec81da00de1f18bf5e7e018fb2f Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 19 Jun 2026 11:06:14 -0400 Subject: [PATCH 16/36] client logic for frame previews --- web/src/components/sidebars/LayerStyle.vue | 9 + web/src/store/framePreview.ts | 360 +++++++++++++++++++++ web/src/store/index.ts | 2 + web/src/store/layer.ts | 9 + web/src/store/map.ts | 32 +- web/src/store/project.ts | 5 + web/src/store/style.ts | 37 +++ web/src/types.ts | 26 ++ web/src/utils/framePreviewCache.ts | 21 ++ web/src/utils/framePreviewLayer.ts | 300 +++++++++++++++++ 10 files changed, 800 insertions(+), 1 deletion(-) create mode 100644 web/src/store/framePreview.ts create mode 100644 web/src/utils/framePreviewCache.ts create mode 100644 web/src/utils/framePreviewLayer.ts diff --git a/web/src/components/sidebars/LayerStyle.vue b/web/src/components/sidebars/LayerStyle.vue index 179a5e331..a613aa7d6 100644 --- a/web/src/components/sidebars/LayerStyle.vue +++ b/web/src/components/sidebars/LayerStyle.vue @@ -585,6 +585,15 @@ const debouncedStyleSpecUpdated = debounce(() => { watch(currentStyleSpec, debouncedStyleSpecUpdated, { deep: true }); watch(() => props.activeLayer, init); + +watch( + () => props.activeLayer === props.layer, + (isEditing) => { + styleStore.setLayerStyleEditing(props.layer, isEditing); + }, + { immediate: true }, +); + onMounted(resetCurrentStyle); diff --git a/web/src/store/framePreview.ts b/web/src/store/framePreview.ts new file mode 100644 index 000000000..b854dbca8 --- /dev/null +++ b/web/src/store/framePreview.ts @@ -0,0 +1,360 @@ +import { defineStore } from "pinia"; +import type { FramePreview, Layer, LayerFrame, LayerStyle } from "@/types"; +import type { Map as MaplibreMap } from "maplibre-gl"; +import { + fadeRasterOpacities, + hidePreviewLayer, + PREVIEW_FADE_DURATION_MS, + PREVIEW_TO_TILES_DELAY_MS, + previewLayerId, + removeAllPreviewLayersForLayerKey, + removePreviewLayer, + removePreviewLayersExcept, + upsertPreviewLayer, + waitForRasterSourceLoaded, +} from "@/utils/framePreviewLayer"; +import { prefetchFramePreviewUrls } from "@/utils/framePreviewCache"; +import { useLayerStore, useMapStore, useStyleStore } from "."; + +interface PendingTransition { + timerId: number; + generation: number; +} + +function layerKey(layer: Layer) { + return `${layer.id}.${layer.copy_id}`; +} + +function orderedRasterFrames(frames: LayerFrame[]) { + return frames + .filter((frame) => frame.raster) + .toSorted((a, b) => a.index - b.index); +} + +function previewsForLayer(layer: Layer, style: LayerStyle | undefined) { + return style?.multiframe_previews ?? layer.multiframe_previews; +} + +function previewAtFrameIndex( + previews: (FramePreview | null)[] | undefined, + rasterFrames: LayerFrame[], + frameIndex: number, +): FramePreview | undefined { + const position = rasterFrames.findIndex( + (frame) => frame.index === frameIndex, + ); + if (position < 0) { + return undefined; + } + return previews?.[position] ?? undefined; +} + +function adjacentRasterFrames( + rasterFrames: LayerFrame[], + currentFrameIndex: number, +) { + const position = rasterFrames.findIndex( + (frame) => frame.index === currentFrameIndex, + ); + if (position < 0) { + return []; + } + return [rasterFrames[position - 1], rasterFrames[position + 1]].filter( + (frame): frame is LayerFrame => frame !== undefined, + ); +} + +export const useFramePreviewStore = defineStore("framePreview", () => { + const pendingTransitions = new Map(); + const activePreviewByLayerKey = new Map(); + const transitionGenerationByLayerKey = new Map(); + + function cancelTransition(layerKeyValue: string) { + const pending = pendingTransitions.get(layerKeyValue); + if (pending) { + window.clearTimeout(pending.timerId); + pendingTransitions.delete(layerKeyValue); + } + } + + function bumpGeneration(layerKeyValue: string) { + const next = (transitionGenerationByLayerKey.get(layerKeyValue) ?? 0) + 1; + transitionGenerationByLayerKey.set(layerKeyValue, next); + return next; + } + + function prefetchLayerPreviews(layer: Layer, style?: LayerStyle) { + const previews = previewsForLayer(layer, style); + if (!previews?.length) { + return; + } + prefetchFramePreviewUrls(previews.map((preview) => preview?.url)); + } + + async function preloadAdjacentPreviewLayers( + map: MaplibreMap, + layerKeyValue: string, + previews: (FramePreview | null)[] | undefined, + rasterFrames: LayerFrame[], + currentFrameIndex: number, + targetOpacity: number, + ) { + const adjacentFrames = adjacentRasterFrames( + rasterFrames, + currentFrameIndex, + ); + const keepFrameIndices = [ + currentFrameIndex, + ...adjacentFrames.map((frame) => frame.index), + ]; + + removePreviewLayersExcept(map, layerKeyValue, keepFrameIndices); + + const urlsToPrefetch: (string | null | undefined)[] = []; + await Promise.all( + adjacentFrames.map(async (frame) => { + const preview = previewAtFrameIndex( + previews, + rasterFrames, + frame.index, + ); + if (!preview || !frame.raster) { + return; + } + urlsToPrefetch.push(preview.url); + await upsertPreviewLayer( + map, + layerKeyValue, + frame.index, + preview, + frame.raster.metadata, + targetOpacity, + false, + ); + }), + ); + + const currentPreview = previewAtFrameIndex( + previews, + rasterFrames, + currentFrameIndex, + ); + urlsToPrefetch.push(currentPreview?.url); + prefetchFramePreviewUrls(urlsToPrefetch); + } + + function hidePreviousPreview(map: MaplibreMap, layerKeyValue: string) { + const previousFrameIndex = activePreviewByLayerKey.get(layerKeyValue); + if (previousFrameIndex !== undefined) { + hidePreviewLayer(map, layerKeyValue, previousFrameIndex); + } + } + + async function transitionToTiles( + layer: Layer, + frameIndex: number, + layerKeyValue: string, + generation: number, + targetOpacity: number, + tileLayerId: string, + tileSourceId: string, + ) { + const mapStore = useMapStore(); + const map = mapStore.getMap(); + const previewMapLayerId = previewLayerId(layerKeyValue, frameIndex); + + if (transitionGenerationByLayerKey.get(layerKeyValue) !== generation) { + return; + } + + await waitForRasterSourceLoaded(map, tileSourceId); + + if (transitionGenerationByLayerKey.get(layerKeyValue) !== generation) { + return; + } + + if (!map.getLayer(tileLayerId) || !map.getLayer(previewMapLayerId)) { + if (map.getLayer(tileLayerId)) { + map.setPaintProperty(tileLayerId, "raster-opacity", targetOpacity); + } + removePreviewLayer(map, layerKeyValue, frameIndex); + activePreviewByLayerKey.delete(layerKeyValue); + return; + } + + const tileOpacity = + (map.getPaintProperty(tileLayerId, "raster-opacity") as number) ?? 0; + const previewOpacity = + (map.getPaintProperty(previewMapLayerId, "raster-opacity") as number) ?? + targetOpacity; + await fadeRasterOpacities( + map, + [ + { id: previewMapLayerId, from: previewOpacity, to: 0 }, + { id: tileLayerId, from: tileOpacity, to: targetOpacity }, + ], + PREVIEW_FADE_DURATION_MS, + ); + + if (transitionGenerationByLayerKey.get(layerKeyValue) !== generation) { + return; + } + + removePreviewLayer(map, layerKeyValue, frameIndex); + activePreviewByLayerKey.delete(layerKeyValue); + } + + async function showPreviewThenTiles(layer: Layer) { + const layerStore = useLayerStore(); + const mapStore = useMapStore(); + const styleStore = useStyleStore(); + if (styleStore.isLayerStyleEditing(layer)) { + return; + } + const map = mapStore.getMap(); + const layerKeyValue = layerKey(layer); + const style = styleStore.selectedLayerStyles[layerKeyValue]; + const previews = previewsForLayer(layer, style); + + const frames = layerStore.layerFrames(layer); + const rasterFrames = orderedRasterFrames(frames); + if (rasterFrames.length <= 1) { + return; + } + + const currentFrame = rasterFrames.find( + (frame) => frame.index === layer.current_frame_index, + ); + if (!currentFrame?.raster || !layer.visible) { + return; + } + + const preview = previewAtFrameIndex( + previews, + rasterFrames, + layer.current_frame_index, + ); + + const tileSourceId = mapStore.sourceIdFromLayerFrame(layer, currentFrame); + const tileLayerId = `${tileSourceId}.raster`; + const targetOpacity = style?.style_spec?.opacity ?? 1; + + cancelTransition(layerKeyValue); + const generation = bumpGeneration(layerKeyValue); + hidePreviousPreview(map, layerKeyValue); + + if (!preview) { + if (map.getLayer(tileLayerId)) { + map.setPaintProperty(tileLayerId, "raster-opacity", targetOpacity); + } + void preloadAdjacentPreviewLayers( + map, + layerKeyValue, + previews, + rasterFrames, + layer.current_frame_index, + targetOpacity, + ); + return; + } + + const previewMapLayerId = await upsertPreviewLayer( + map, + layerKeyValue, + layer.current_frame_index, + preview, + currentFrame.raster.metadata, + targetOpacity, + ); + if (!previewMapLayerId) { + if (map.getLayer(tileLayerId)) { + map.setPaintProperty(tileLayerId, "raster-opacity", targetOpacity); + } + return; + } + + activePreviewByLayerKey.set(layerKeyValue, layer.current_frame_index); + + if (map.getLayer(tileLayerId)) { + map.setPaintProperty(tileLayerId, "raster-opacity", 0); + } + + void preloadAdjacentPreviewLayers( + map, + layerKeyValue, + previews, + rasterFrames, + layer.current_frame_index, + targetOpacity, + ); + + const timerId = window.setTimeout(() => { + pendingTransitions.delete(layerKeyValue); + void transitionToTiles( + layer, + layer.current_frame_index, + layerKeyValue, + generation, + targetOpacity, + tileLayerId, + tileSourceId, + ); + }, PREVIEW_TO_TILES_DELAY_MS); + + pendingTransitions.set(layerKeyValue, { timerId, generation }); + } + + function dismissPreviewForLayer(layer: Layer) { + const mapStore = useMapStore(); + const layerStore = useLayerStore(); + const styleStore = useStyleStore(); + const map = mapStore.getMap(); + const layerKeyValue = layerKey(layer); + + cancelTransition(layerKeyValue); + bumpGeneration(layerKeyValue); + removeAllPreviewLayersForLayerKey(map, layerKeyValue); + activePreviewByLayerKey.delete(layerKeyValue); + + const frames = layerStore.layerFrames(layer); + const currentFrame = frames.find( + (frame) => frame.index === layer.current_frame_index, + ); + if (!currentFrame?.raster) { + return; + } + + const tileLayerId = `${mapStore.sourceIdFromLayerFrame(layer, currentFrame)}.raster`; + const style = styleStore.selectedLayerStyles[layerKeyValue]; + const targetOpacity = style?.style_spec?.opacity ?? 1; + if (map.getLayer(tileLayerId)) { + map.setPaintProperty(tileLayerId, "raster-opacity", targetOpacity); + } + } + + function cleanupLayer(layer: Layer) { + const mapStore = useMapStore(); + const layerKeyValue = layerKey(layer); + cancelTransition(layerKeyValue); + transitionGenerationByLayerKey.delete(layerKeyValue); + activePreviewByLayerKey.delete(layerKeyValue); + removeAllPreviewLayersForLayerKey(mapStore.getMap(), layerKeyValue); + } + + function clearAll() { + pendingTransitions.forEach((pending) => + window.clearTimeout(pending.timerId), + ); + pendingTransitions.clear(); + transitionGenerationByLayerKey.clear(); + activePreviewByLayerKey.clear(); + } + + return { + prefetchLayerPreviews, + showPreviewThenTiles, + dismissPreviewForLayer, + cleanupLayer, + clearAll, + }; +}); diff --git a/web/src/store/index.ts b/web/src/store/index.ts index f78d38d5c..ea720cd28 100644 --- a/web/src/store/index.ts +++ b/web/src/store/index.ts @@ -7,6 +7,7 @@ import { useProjectStore } from "./project"; import { useStyleStore } from "./style"; import { usePanelStore } from "./panel"; import { useConversionStore } from "./conversion"; +import { useFramePreviewStore } from "./framePreview"; export { useAppStore, @@ -18,4 +19,5 @@ export { useStyleStore, usePanelStore, useConversionStore, + useFramePreviewStore, }; diff --git a/web/src/store/layer.ts b/web/src/store/layer.ts index 276d82769..11fab0087 100644 --- a/web/src/store/layer.ts +++ b/web/src/store/layer.ts @@ -22,6 +22,7 @@ import { useStyleStore, useNetworkStore, useProjectStore, + useFramePreviewStore, } from "."; interface SourceDBObjects { @@ -43,6 +44,7 @@ export const useLayerStore = defineStore("layer", () => { const networkStore = useNetworkStore(); const styleStore = useStyleStore(); const projectStore = useProjectStore(); + const framePreviewStore = useFramePreviewStore(); /** * Return the maplibre layers associated with a Layer DB object @@ -197,6 +199,10 @@ export const useLayerStore = defineStore("layer", () => { } selectedLayers.value = [newLayer, ...selectedLayers.value]; + framePreviewStore.prefetchLayerPreviews( + newLayer, + newLayer.default_style ?? undefined, + ); } watch(selectedLayers, updateLayersShown); @@ -220,6 +226,9 @@ export const useLayerStore = defineStore("layer", () => { ) { styleStore.selectedLayerStyles[styleId] = { ...layer.default_style, + ...(layer.multiframe_previews + ? { multiframe_previews: layer.multiframe_previews } + : {}), }; if ( styleStore.selectedLayerStyles[styleId]?.style_spec diff --git a/web/src/store/map.ts b/web/src/store/map.ts index a59464dd5..21cf32926 100644 --- a/web/src/store/map.ts +++ b/web/src/store/map.ts @@ -22,7 +22,17 @@ import { Map, Popup } from "maplibre-gl"; import { getBasemaps, getRasterDataValues } from "@/api/rest"; import { baseURL } from "@/api/auth"; import proj4 from "proj4"; -import { useStyleStore, useLayerStore, useAppStore, useProjectStore } from "."; +import { + useStyleStore, + useLayerStore, + useAppStore, + useProjectStore, + useFramePreviewStore, +} from "."; +import { + isPreviewMapLayerId, + removeAllPreviewLayersForLayerKey, +} from "@/utils/framePreviewLayer"; function getLayerIsVisible(layer: MapLibreLayerWithMetadata) { // Since visibility must be 'visible' for a feature click to even be registered, @@ -324,12 +334,21 @@ export const useMapStore = defineStore("map", () => { function removeLayers(layerIds: string[]) { const map = getMap(); + const framePreviewStore = useFramePreviewStore(); + const cleanedLayerKeys = new Set(); // Must collect all source Ids so they can be removed after all layers // have been removed, since multple layers may use the same source const sourceIdsToRemove = new Set(); const updatedLayerIds: string[] = []; layerIds.forEach((id) => { + if (isPreviewMapLayerId(id)) { + return; + } + const layerKey = id.split(".").slice(0, 2).join("."); + if (layerKey.includes(".")) { + cleanedLayerKeys.add(layerKey); + } // Rasters have implicit bounds layers that also need to be removed if (id.includes(".raster.")) { updatedLayerIds.push(id.replace(".raster.", ".bounds.")); @@ -348,6 +367,17 @@ export const useMapStore = defineStore("map", () => { sourceIdsToRemove.forEach((id) => { map.removeSource(id); }); + + cleanedLayerKeys.forEach((layerKey) => { + removeAllPreviewLayersForLayerKey(map, layerKey); + const [layerId, copyId] = layerKey.split(".").map(Number); + const layer = layerStore.selectedLayers.find( + (candidate) => candidate.id === layerId && candidate.copy_id === copyId, + ); + if (layer) { + framePreviewStore.cleanupLayer(layer); + } + }); } /** diff --git a/web/src/store/project.ts b/web/src/store/project.ts index d68c13df8..c835c3088 100644 --- a/web/src/store/project.ts +++ b/web/src/store/project.ts @@ -20,7 +20,9 @@ import { usePanelStore, useAppStore, useStyleStore, + useFramePreviewStore, } from "."; +import { clearFramePreviewCache } from "@/utils/framePreviewCache"; export const useProjectStore = defineStore("project", () => { const networkStore = useNetworkStore(); @@ -278,6 +280,9 @@ export const useProjectStore = defineStore("project", () => { layerStore.selectedLayers = []; styleStore.selectedLayerStyles = {}; + styleStore.clearStyleEditing(); + useFramePreviewStore().clearAll(); + clearFramePreviewCache(); mapStore.clickedFeature = undefined; diff --git a/web/src/store/style.ts b/web/src/store/style.ts index bee2dd72a..aad70d15b 100644 --- a/web/src/store/style.ts +++ b/web/src/store/style.ts @@ -22,6 +22,7 @@ import { useLayerStore, useProjectStore, useNetworkStore, + useFramePreviewStore, } from "."; export interface MapLayerStyleRaw { @@ -412,11 +413,37 @@ function getVectorVisibilityPaintProperty( export const useStyleStore = defineStore("style", () => { const selectedLayerStyles = ref>({}); const colormaps = ref([]); + const editingStyleLayerKeys = ref>(new Set()); const mapStore = useMapStore(); const projectStore = useProjectStore(); const layerStore = useLayerStore(); const networkStore = useNetworkStore(); + const framePreviewStore = useFramePreviewStore(); + + function layerStyleKey(layer: Layer) { + return `${layer.id}.${layer.copy_id}`; + } + + function isLayerStyleEditing(layer: Layer) { + return editingStyleLayerKeys.value.has(layerStyleKey(layer)); + } + + function setLayerStyleEditing(layer: Layer, editing: boolean) { + const key = layerStyleKey(layer); + const next = new Set(editingStyleLayerKeys.value); + if (editing) { + next.add(key); + framePreviewStore.dismissPreviewForLayer(layer); + } else { + next.delete(key); + } + editingStyleLayerKeys.value = next; + } + + function clearStyleEditing() { + editingStyleLayerKeys.value = new Set(); + } function getDefaultColor(layerId: number) { const color = chroma.hsl( @@ -492,6 +519,12 @@ export const useStyleStore = defineStore("style", () => { } }); networkStore.styleVisibleNetworks(); + + const hasMultiframeRaster = + frames.length > 1 && frames.some((f) => f.raster); + if (hasMultiframeRaster && !isLayerStyleEditing(layer)) { + void framePreviewStore.showPreviewThenTiles(layer); + } } type GeneratedLayerStyle = { @@ -668,6 +701,7 @@ export const useStyleStore = defineStore("style", () => { return { colormaps, selectedLayerStyles, + editingStyleLayerKeys, fetchColormaps, getRasterTilesQuery, getRasterSourceFilterParams, @@ -677,6 +711,9 @@ export const useStyleStore = defineStore("style", () => { getDefaultColor, getDefaultStyleSpec, getVectorColorPaintProperty, + isLayerStyleEditing, + setLayerStyleEditing, + clearStyleEditing, updateLayerStyles, setMapLayerStyle, returnMapLayerStyle, diff --git a/web/src/types.ts b/web/src/types.ts index 9cbeee9be..dc3a68045 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -28,6 +28,30 @@ export interface Dataset { n_layers?: number; } +export interface FramePreviewCorner { + x: number; + y: number; +} + +export interface FramePreviewBounds { + srs: string; + xmin: number; + xmax: number; + ymin: number; + ymax: number; + ul?: FramePreviewCorner; + ur?: FramePreviewCorner; + lr?: FramePreviewCorner; + ll?: FramePreviewCorner; +} + +export interface FramePreview { + url: string; + width: number; + height: number; + bounds: FramePreviewBounds; +} + export interface Layer { id: number; copy_id: number; @@ -38,6 +62,7 @@ export interface Layer { visible: boolean; current_frame_index: number; default_style: LayerStyle | null; + multiframe_previews?: (FramePreview | null)[]; } export interface LayerFrame { @@ -128,6 +153,7 @@ export interface LayerStyle { project?: number; is_default: boolean; style_spec?: StyleSpec; + multiframe_previews?: (FramePreview | null)[]; } export interface VectorData { diff --git a/web/src/utils/framePreviewCache.ts b/web/src/utils/framePreviewCache.ts new file mode 100644 index 000000000..5598c8cd3 --- /dev/null +++ b/web/src/utils/framePreviewCache.ts @@ -0,0 +1,21 @@ +const prefetchedUrls = new Set(); + +export async function getCachedPreviewObjectUrl( + url: string, +): Promise { + return url; +} + +export function prefetchFramePreviewUrls(urls: (string | null | undefined)[]) { + urls.forEach((url) => { + if (!url || prefetchedUrls.has(url)) { + return; + } + prefetchedUrls.add(url); + void fetch(url); + }); +} + +export function clearFramePreviewCache() { + prefetchedUrls.clear(); +} diff --git a/web/src/utils/framePreviewLayer.ts b/web/src/utils/framePreviewLayer.ts new file mode 100644 index 000000000..46850d8ee --- /dev/null +++ b/web/src/utils/framePreviewLayer.ts @@ -0,0 +1,300 @@ +import type { + FramePreview, + FramePreviewBounds, + FramePreviewCorner, + RasterMetadata, +} from "@/types"; +import type { Map, MapSourceDataEvent } from "maplibre-gl"; +import proj4 from "proj4"; +import { getCachedPreviewObjectUrl } from "./framePreviewCache"; + +export const PREVIEW_TO_TILES_DELAY_MS = 1500; +export const PREVIEW_FADE_DURATION_MS = 400; + +const CORNER_KEYS = ["ul", "ur", "lr", "ll"] as const; +type CornerKey = (typeof CORNER_KEYS)[number]; + +function clampRasterOpacity(value: number): number { + return Math.min(1, Math.max(0, value)); +} + +export function previewSourceId(layerKey: string, frameIndex: number) { + return `${layerKey}.preview.${frameIndex}`; +} + +export function previewLayerId(layerKey: string, frameIndex: number) { + return `${previewSourceId(layerKey, frameIndex)}.raster`; +} + +export function isPreviewMapLayerId(mapLayerId: string) { + return mapLayerId.includes(".preview."); +} + +function toLngLat(srs: string, x: number, y: number): [number, number] { + if (srs && srs !== "EPSG:4326") { + return proj4(srs, "EPSG:4326", [x, y]) as [number, number]; + } + return [x, y]; +} + +function hasCornerBounds( + bounds: FramePreviewBounds, +): bounds is FramePreviewBounds & Record { + return CORNER_KEYS.every((corner) => bounds[corner] !== undefined); +} + +function cornersFromRasterMetadata( + rasterBounds: RasterMetadata["bounds"], +): FramePreviewBounds | undefined { + if (!CORNER_KEYS.every((corner) => rasterBounds[corner])) { + return undefined; + } + + const corners = Object.fromEntries( + CORNER_KEYS.map((corner) => { + const point = rasterBounds[corner]!; + const [x, y] = toLngLat(rasterBounds.srs, point.x, point.y); + return [corner, { x, y }]; + }), + ) as Record; + + const lngs = CORNER_KEYS.map((corner) => corners[corner].x); + const lats = CORNER_KEYS.map((corner) => corners[corner].y); + + return { + srs: "EPSG:4326", + xmin: Math.min(...lngs), + xmax: Math.max(...lngs), + ymin: Math.min(...lats), + ymax: Math.max(...lats), + ...corners, + }; +} + +export function resolvePreviewBounds( + preview: FramePreview, + raster?: RasterMetadata, +): FramePreviewBounds { + if (hasCornerBounds(preview.bounds)) { + return preview.bounds; + } + + if (raster?.bounds) { + const rasterCorners = cornersFromRasterMetadata(raster.bounds); + if (rasterCorners) { + return rasterCorners; + } + } + + return preview.bounds; +} + +function boundsToCoordinates( + bounds: FramePreviewBounds, +): [[number, number], [number, number], [number, number], [number, number]] { + if (hasCornerBounds(bounds)) { + return [ + toLngLat(bounds.srs, bounds.ul.x, bounds.ul.y), + toLngLat(bounds.srs, bounds.ur.x, bounds.ur.y), + toLngLat(bounds.srs, bounds.lr.x, bounds.lr.y), + toLngLat(bounds.srs, bounds.ll.x, bounds.ll.y), + ]; + } + + const { xmin, xmax, ymin, ymax, srs } = bounds; + return [ + toLngLat(srs, xmin, ymax), + toLngLat(srs, xmax, ymax), + toLngLat(srs, xmax, ymin), + toLngLat(srs, xmin, ymin), + ]; +} + +export async function upsertPreviewLayer( + map: Map, + layerKey: string, + frameIndex: number, + preview: FramePreview, + raster?: RasterMetadata, + opacity = 1, + visible = true, +) { + const sourceId = previewSourceId(layerKey, frameIndex); + const mapLayerId = previewLayerId(layerKey, frameIndex); + const objectUrl = await getCachedPreviewObjectUrl(preview.url); + if (!objectUrl) { + return undefined; + } + + const coordinates = boundsToCoordinates( + resolvePreviewBounds(preview, raster), + ); + const existingSource = map.getSource(sourceId); + if (existingSource) { + if (map.getLayer(mapLayerId)) { + map.removeLayer(mapLayerId); + } + map.removeSource(sourceId); + } + map.addSource(sourceId, { + type: "image", + url: objectUrl, + coordinates, + }); + + const previewOpacity = clampRasterOpacity(opacity); + const visibility = visible ? "visible" : "none"; + + if (!map.getLayer(mapLayerId)) { + map.addLayer({ + id: mapLayerId, + type: "raster", + source: sourceId, + layout: { + visibility, + }, + paint: { + "raster-opacity": previewOpacity, + "raster-fade-duration": 0, + }, + }); + } else { + map.setPaintProperty(mapLayerId, "raster-opacity", previewOpacity); + map.setLayoutProperty(mapLayerId, "visibility", visibility); + } + + return mapLayerId; +} + +export function hidePreviewLayer( + map: Map, + layerKey: string, + frameIndex: number, +) { + const mapLayerId = previewLayerId(layerKey, frameIndex); + if (map.getLayer(mapLayerId)) { + map.setLayoutProperty(mapLayerId, "visibility", "none"); + } +} + +export function removePreviewLayer( + map: Map, + layerKey: string, + frameIndex: number, +) { + const sourceId = previewSourceId(layerKey, frameIndex); + const mapLayerId = previewLayerId(layerKey, frameIndex); + if (map.getLayer(mapLayerId)) { + map.removeLayer(mapLayerId); + } + if (map.getSource(sourceId)) { + map.removeSource(sourceId); + } +} + +function previewFrameIndexFromLayerId( + layerKey: string, + mapLayerId: string, +): number | undefined { + const prefix = `${layerKey}.preview.`; + if (!mapLayerId.startsWith(prefix)) { + return undefined; + } + const rest = mapLayerId.slice(prefix.length); + const frameIndex = Number.parseInt(rest.split(".")[0], 10); + return Number.isNaN(frameIndex) ? undefined : frameIndex; +} + +export function removePreviewLayersExcept( + map: Map, + layerKey: string, + keepFrameIndices: number[], +) { + const keep = new Set(keepFrameIndices); + const frameIndicesToRemove = new Set(); + + map.getStyle().layers?.forEach((layer) => { + const frameIndex = previewFrameIndexFromLayerId(layerKey, layer.id); + if (frameIndex !== undefined && !keep.has(frameIndex)) { + frameIndicesToRemove.add(frameIndex); + } + }); + + frameIndicesToRemove.forEach((frameIndex) => { + removePreviewLayer(map, layerKey, frameIndex); + }); +} + +export function removeAllPreviewLayersForLayerKey(map: Map, layerKey: string) { + map.getStyle().layers?.forEach((layer) => { + if (layer.id.startsWith(`${layerKey}.preview.`)) { + map.removeLayer(layer.id); + } + }); + Object.keys(map.getStyle().sources ?? {}).forEach((sourceId) => { + if (sourceId.startsWith(`${layerKey}.preview.`)) { + map.removeSource(sourceId); + } + }); +} + +export function waitForRasterSourceLoaded( + map: Map, + sourceId: string, + timeoutMs = 10000, +): Promise { + return new Promise((resolve) => { + if (!map.getSource(sourceId)) { + resolve(); + return; + } + if (map.isSourceLoaded(sourceId)) { + resolve(); + return; + } + + const timeout = window.setTimeout(() => { + map.off("sourcedata", onSourceData); + resolve(); + }, timeoutMs); + + function onSourceData(event: MapSourceDataEvent) { + if (event.sourceId === sourceId && event.isSourceLoaded) { + window.clearTimeout(timeout); + map.off("sourcedata", onSourceData); + resolve(); + } + } + + map.on("sourcedata", onSourceData); + }); +} + +export async function fadeRasterOpacities( + map: Map, + layers: { id: string; from: number; to: number }[], + durationMs: number, +) { + const start = performance.now(); + await new Promise((resolve) => { + function step(now: number) { + const progress = Math.min(1, (now - start) / durationMs); + layers.forEach(({ id, from, to }) => { + if (map.getLayer(id)) { + const opacity = progress >= 1 ? to : from + (to - from) * progress; + map.setPaintProperty( + id, + "raster-opacity", + clampRasterOpacity(opacity), + ); + } + }); + if (progress < 1) { + requestAnimationFrame(step); + } else { + resolve(); + } + } + requestAnimationFrame(step); + }); +} From ac40e490922fb679301e398e2adbd671150bad3f Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 19 Jun 2026 11:32:11 -0400 Subject: [PATCH 17/36] remove timing and use tile loaded as the transition --- web/src/store/framePreview.ts | 45 ++++++------------------------ web/src/utils/framePreviewLayer.ts | 1 - 2 files changed, 9 insertions(+), 37 deletions(-) diff --git a/web/src/store/framePreview.ts b/web/src/store/framePreview.ts index b854dbca8..48fddfeae 100644 --- a/web/src/store/framePreview.ts +++ b/web/src/store/framePreview.ts @@ -5,7 +5,6 @@ import { fadeRasterOpacities, hidePreviewLayer, PREVIEW_FADE_DURATION_MS, - PREVIEW_TO_TILES_DELAY_MS, previewLayerId, removeAllPreviewLayersForLayerKey, removePreviewLayer, @@ -16,11 +15,6 @@ import { import { prefetchFramePreviewUrls } from "@/utils/framePreviewCache"; import { useLayerStore, useMapStore, useStyleStore } from "."; -interface PendingTransition { - timerId: number; - generation: number; -} - function layerKey(layer: Layer) { return `${layer.id}.${layer.copy_id}`; } @@ -65,18 +59,9 @@ function adjacentRasterFrames( } export const useFramePreviewStore = defineStore("framePreview", () => { - const pendingTransitions = new Map(); const activePreviewByLayerKey = new Map(); const transitionGenerationByLayerKey = new Map(); - function cancelTransition(layerKeyValue: string) { - const pending = pendingTransitions.get(layerKeyValue); - if (pending) { - window.clearTimeout(pending.timerId); - pendingTransitions.delete(layerKeyValue); - } - } - function bumpGeneration(layerKeyValue: string) { const next = (transitionGenerationByLayerKey.get(layerKeyValue) ?? 0) + 1; transitionGenerationByLayerKey.set(layerKeyValue, next); @@ -239,7 +224,6 @@ export const useFramePreviewStore = defineStore("framePreview", () => { const tileLayerId = `${tileSourceId}.raster`; const targetOpacity = style?.style_spec?.opacity ?? 1; - cancelTransition(layerKeyValue); const generation = bumpGeneration(layerKeyValue); hidePreviousPreview(map, layerKeyValue); @@ -288,20 +272,15 @@ export const useFramePreviewStore = defineStore("framePreview", () => { targetOpacity, ); - const timerId = window.setTimeout(() => { - pendingTransitions.delete(layerKeyValue); - void transitionToTiles( - layer, - layer.current_frame_index, - layerKeyValue, - generation, - targetOpacity, - tileLayerId, - tileSourceId, - ); - }, PREVIEW_TO_TILES_DELAY_MS); - - pendingTransitions.set(layerKeyValue, { timerId, generation }); + void transitionToTiles( + layer, + layer.current_frame_index, + layerKeyValue, + generation, + targetOpacity, + tileLayerId, + tileSourceId, + ); } function dismissPreviewForLayer(layer: Layer) { @@ -311,7 +290,6 @@ export const useFramePreviewStore = defineStore("framePreview", () => { const map = mapStore.getMap(); const layerKeyValue = layerKey(layer); - cancelTransition(layerKeyValue); bumpGeneration(layerKeyValue); removeAllPreviewLayersForLayerKey(map, layerKeyValue); activePreviewByLayerKey.delete(layerKeyValue); @@ -335,17 +313,12 @@ export const useFramePreviewStore = defineStore("framePreview", () => { function cleanupLayer(layer: Layer) { const mapStore = useMapStore(); const layerKeyValue = layerKey(layer); - cancelTransition(layerKeyValue); transitionGenerationByLayerKey.delete(layerKeyValue); activePreviewByLayerKey.delete(layerKeyValue); removeAllPreviewLayersForLayerKey(mapStore.getMap(), layerKeyValue); } function clearAll() { - pendingTransitions.forEach((pending) => - window.clearTimeout(pending.timerId), - ); - pendingTransitions.clear(); transitionGenerationByLayerKey.clear(); activePreviewByLayerKey.clear(); } diff --git a/web/src/utils/framePreviewLayer.ts b/web/src/utils/framePreviewLayer.ts index 46850d8ee..386af43cd 100644 --- a/web/src/utils/framePreviewLayer.ts +++ b/web/src/utils/framePreviewLayer.ts @@ -8,7 +8,6 @@ import type { Map, MapSourceDataEvent } from "maplibre-gl"; import proj4 from "proj4"; import { getCachedPreviewObjectUrl } from "./framePreviewCache"; -export const PREVIEW_TO_TILES_DELAY_MS = 1500; export const PREVIEW_FADE_DURATION_MS = 400; const CORNER_KEYS = ["ul", "ur", "lr", "ll"] as const; From ff89a0bfccc9591d49d5d72b5966ea0b737590ab Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 19 Jun 2026 11:45:18 -0400 Subject: [PATCH 18/36] renable previews after exiting the style mode --- web/src/store/style.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/web/src/store/style.ts b/web/src/store/style.ts index aad70d15b..b3396d934 100644 --- a/web/src/store/style.ts +++ b/web/src/store/style.ts @@ -434,11 +434,13 @@ export const useStyleStore = defineStore("style", () => { const next = new Set(editingStyleLayerKeys.value); if (editing) { next.add(key); + editingStyleLayerKeys.value = next; framePreviewStore.dismissPreviewForLayer(layer); - } else { - next.delete(key); + return; } + next.delete(key); editingStyleLayerKeys.value = next; + updateLayerStyles(layer); } function clearStyleEditing() { From a325ad78a64caeb8e5d419c1723921ed0a62346d Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Thu, 25 Jun 2026 14:37:11 -0400 Subject: [PATCH 19/36] add status and style_fingerprint fields, make nullable fields for regenerating --- .../migrations/0024_raster_frame_preview.py | 27 +++++++++++++++---- uvdat/core/models/frame_preview.py | 21 ++++++++++++--- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/uvdat/core/migrations/0024_raster_frame_preview.py b/uvdat/core/migrations/0024_raster_frame_preview.py index ced146b13..fc2abc86b 100644 --- a/uvdat/core/migrations/0024_raster_frame_preview.py +++ b/uvdat/core/migrations/0024_raster_frame_preview.py @@ -1,4 +1,4 @@ -# Generated by Django 5.2.9 on 2026-06-12 +# Generated by Django 5.2.9 on 2026-06-25 from __future__ import annotations from django.db import migrations, models @@ -21,10 +21,27 @@ class Migration(migrations.Migration): auto_created=True, primary_key=True, serialize=False, verbose_name="ID" ), ), - ("image", s3_file_field.fields.S3FileField()), - ("width", models.PositiveIntegerField()), - ("height", models.PositiveIntegerField()), - ("bounds", models.JSONField()), + ( + "status", + models.CharField( + choices=[ + ("creating", "Creating"), + ("regenerating", "Regenerating"), + ("complete", "Complete"), + ("failed", "Failed"), + ], + default="creating", + max_length=16, + ), + ), + ( + "style_fingerprint", + models.CharField(blank=True, default="", max_length=64), + ), + ("image", s3_file_field.fields.S3FileField(blank=True, null=True)), + ("width", models.PositiveIntegerField(blank=True, null=True)), + ("height", models.PositiveIntegerField(blank=True, null=True)), + ("bounds", models.JSONField(blank=True, default=dict)), ( "layer_frame", models.ForeignKey( diff --git a/uvdat/core/models/frame_preview.py b/uvdat/core/models/frame_preview.py index f0051ab37..f78f46559 100644 --- a/uvdat/core/models/frame_preview.py +++ b/uvdat/core/models/frame_preview.py @@ -8,6 +8,13 @@ from .styles import LayerStyle +class PreviewStatus(models.TextChoices): + CREATING = "creating", "Creating" + REGENERATING = "regenerating", "Regenerating" + COMPLETE = "complete", "Complete" + FAILED = "failed", "Failed" + + class RasterFramePreview(models.Model): layer_style = models.ForeignKey( LayerStyle, @@ -19,10 +26,16 @@ class RasterFramePreview(models.Model): related_name="style_previews", on_delete=models.CASCADE, ) - image = S3FileField() - width = models.PositiveIntegerField() - height = models.PositiveIntegerField() - bounds = models.JSONField() + status = models.CharField( + max_length=16, + choices=PreviewStatus.choices, + default=PreviewStatus.CREATING, + ) + style_fingerprint = models.CharField(max_length=64, blank=True, default="") + image = S3FileField(blank=True, null=True) + width = models.PositiveIntegerField(null=True, blank=True) + height = models.PositiveIntegerField(null=True, blank=True) + bounds = models.JSONField(default=dict, blank=True) class Meta: constraints = [ From 581c224f78d2450b90103a0961bbba4c1d30cbb8 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Thu, 25 Jun 2026 14:50:25 -0400 Subject: [PATCH 20/36] restructure to frame_previews subfolder in core --- uvdat/core/frame_previews/__init__.py | 0 uvdat/core/{ => frame_previews}/raster_style.py | 0 .../types.py} | 0 uvdat/core/models/layer.py | 2 +- uvdat/core/models/styles.py | 2 +- uvdat/core/tasks/frame_preview.py | 15 +++++++++------ uvdat/core/tests/test_frame_preview.py | 4 ++-- 7 files changed, 13 insertions(+), 10 deletions(-) create mode 100644 uvdat/core/frame_previews/__init__.py rename uvdat/core/{ => frame_previews}/raster_style.py (100%) rename uvdat/core/{frame_preview_types.py => frame_previews/types.py} (100%) diff --git a/uvdat/core/frame_previews/__init__.py b/uvdat/core/frame_previews/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/uvdat/core/raster_style.py b/uvdat/core/frame_previews/raster_style.py similarity index 100% rename from uvdat/core/raster_style.py rename to uvdat/core/frame_previews/raster_style.py diff --git a/uvdat/core/frame_preview_types.py b/uvdat/core/frame_previews/types.py similarity index 100% rename from uvdat/core/frame_preview_types.py rename to uvdat/core/frame_previews/types.py diff --git a/uvdat/core/models/layer.py b/uvdat/core/models/layer.py index 1eeb6ddb1..c32bbe018 100644 --- a/uvdat/core/models/layer.py +++ b/uvdat/core/models/layer.py @@ -9,7 +9,7 @@ from .querysets import ProjectQuerySet if TYPE_CHECKING: - from uvdat.core.frame_preview_types import FramePreviewData + from uvdat.core.frame_previews.types import FramePreviewData def default_source_filters(): diff --git a/uvdat/core/models/styles.py b/uvdat/core/models/styles.py index e129a1929..c85e95b55 100644 --- a/uvdat/core/models/styles.py +++ b/uvdat/core/models/styles.py @@ -6,7 +6,7 @@ from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models -from uvdat.core.frame_preview_types import FramePreviewData +from uvdat.core.frame_previews.types import FramePreviewData from .colormap import Colormap from .layer import Layer diff --git a/uvdat/core/tasks/frame_preview.py b/uvdat/core/tasks/frame_preview.py index 64917f9c4..35dec5f03 100644 --- a/uvdat/core/tasks/frame_preview.py +++ b/uvdat/core/tasks/frame_preview.py @@ -11,6 +11,11 @@ from django_large_image import tilesource, utilities from PIL import Image +from uvdat.core.frame_previews.raster_style import ( + apply_source_filters_to_style_query, + build_raster_tiles_style_query, + raster_source_filter_kwargs, +) from uvdat.core.models import ( Colormap, Layer, @@ -19,14 +24,9 @@ RasterData, RasterFramePreview, ) -from uvdat.core.raster_style import ( - apply_source_filters_to_style_query, - build_raster_tiles_style_query, - raster_source_filter_kwargs, -) if TYPE_CHECKING: - from uvdat.core.frame_preview_types import FramePreviewBounds + from uvdat.core.frame_previews.types import FramePreviewBounds logger = logging.getLogger(__name__) @@ -148,6 +148,9 @@ def ensure_default_layer_style(layer: Layer, project: Project) -> LayerStyle: def generate_layer_style_previews( layer_style_id: int, resolution_fraction: float | None = None, + *, + fingerprint: str | None = None, + result_id: int | None = None, ): started = time.perf_counter() layer_style = LayerStyle.objects.select_related("layer").get(id=layer_style_id) diff --git a/uvdat/core/tests/test_frame_preview.py b/uvdat/core/tests/test_frame_preview.py index b84b4b078..8a89aa426 100644 --- a/uvdat/core/tests/test_frame_preview.py +++ b/uvdat/core/tests/test_frame_preview.py @@ -3,12 +3,12 @@ from django.core.files.base import ContentFile import pytest -from uvdat.core.models import RasterFramePreview -from uvdat.core.raster_style import ( +from uvdat.core.frame_previews.raster_style import ( apply_source_filters_to_style_query, build_thumbnail_style_query, raster_source_filter_kwargs, ) +from uvdat.core.models import RasterFramePreview from uvdat.core.tasks.frame_preview import ( FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION, FRAME_PREVIEW_MAX_PX, From 49022de39be18c43c001eb479a6a01b3ea6f49b4 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Thu, 25 Jun 2026 15:05:58 -0400 Subject: [PATCH 21/36] add preview regeneration utility functions --- .../frame_previews/preview_regeneration.py | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 uvdat/core/frame_previews/preview_regeneration.py diff --git a/uvdat/core/frame_previews/preview_regeneration.py b/uvdat/core/frame_previews/preview_regeneration.py new file mode 100644 index 000000000..51c52c585 --- /dev/null +++ b/uvdat/core/frame_previews/preview_regeneration.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import hashlib +import json + +from django.utils import timezone + +from uvdat.core.models import LayerStyle, RasterFramePreview, TaskResult +from uvdat.core.models.frame_preview import PreviewStatus +from uvdat.core.tasks.frame_preview import generate_layer_style_previews + + +def style_fingerprint(layer_style: LayerStyle) -> str: + payload = json.dumps(layer_style.repr_style_configs(), sort_keys=True, default=str) + return hashlib.sha256(payload.encode()).hexdigest() + + +def style_needs_previews(layer_style: LayerStyle) -> bool: + return layer_style.layer.is_multiframe_raster() + + +def supersede_pending_preview_tasks(layer_style_id: int) -> None: + TaskResult.objects.filter( + task_type="frame_preview", + completed__isnull=True, + inputs__layer_style_id=layer_style_id, + ).update( + completed=timezone.now(), + status="Superseded by newer style save.", + ) + + +def mark_previews_regenerating(layer_style: LayerStyle, fingerprint: str) -> list[int]: + """Upsert one preview row per multiframe frame and clear stale images.""" + frame_ids = [] + for frame in layer_style.layer.multiframe_raster_frames(): + preview, created = RasterFramePreview.objects.get_or_create( + layer_style=layer_style, + layer_frame=frame, + defaults={ + "status": PreviewStatus.CREATING, + "style_fingerprint": fingerprint, + }, + ) + preview.style_fingerprint = fingerprint + preview.status = PreviewStatus.CREATING if created else PreviewStatus.REGENERATING + + if preview.image: + preview.image.delete(save=False) + preview.image = None + preview.width = None + preview.height = None + preview.bounds = {} + preview.save() + frame_ids.append(frame.id) + return frame_ids + + +def invalidate_and_enqueue_previews(layer_style: LayerStyle) -> TaskResult | None: + if not style_needs_previews(layer_style): + return None + + fingerprint = style_fingerprint(layer_style) + mark_previews_regenerating(layer_style, fingerprint) + supersede_pending_preview_tasks(layer_style.id) + + result = TaskResult.objects.create( + name=f"Frame previews: {layer_style.name}", + task_type="frame_preview", + project=layer_style.project, + inputs={ + "layer_style_id": layer_style.id, + "layer_id": layer_style.layer_id, + "fingerprint": fingerprint, + }, + ) + + generate_layer_style_previews.delay( + layer_style.id, + fingerprint=fingerprint, + result_id=result.id, + ) + return result + + +def preview_status_for_style(layer_style: LayerStyle) -> str | None: + """Aggregate per-frame preview rows into a style-level status string.""" + if not style_needs_previews(layer_style): + return None + + frames = layer_style.layer.multiframe_raster_frames() + if not frames: + return None + + previews_by_frame = { + preview.layer_frame_id: preview for preview in layer_style.frame_previews.all() + } + + if all( + (preview := previews_by_frame.get(frame.id)) + and preview.status == PreviewStatus.COMPLETE + and preview.image + for frame in frames + ): + return "ready" + + statuses = [ + previews_by_frame[frame.id].status for frame in frames if frame.id in previews_by_frame + ] + + if any( + status in (PreviewStatus.CREATING, PreviewStatus.REGENERATING) for status in statuses + ) or any(status == PreviewStatus.COMPLETE for status in statuses): + result = "generating" + elif any(status == PreviewStatus.FAILED for status in statuses): + result = "failed" + else: + result = "none" + return result From b1ceda4889e726b197381f6788e8d92cb3adda2a Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Thu, 25 Jun 2026 15:06:37 -0400 Subject: [PATCH 22/36] on create/update style regenerate previews for raster multiframe data --- uvdat/core/rest/serializers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/uvdat/core/rest/serializers.py b/uvdat/core/rest/serializers.py index 8a6aa165e..64a646488 100644 --- a/uvdat/core/rest/serializers.py +++ b/uvdat/core/rest/serializers.py @@ -5,6 +5,7 @@ from django.contrib.gis.serializers import geojson from rest_framework import serializers +from uvdat.core.frame_previews.preview_regeneration import invalidate_and_enqueue_previews from uvdat.core.models import ( Basemap, Chart, @@ -171,12 +172,15 @@ def create(self, validated_data): style_spec = self.initial_data.pop("style_spec", None) instance = super().create(validated_data) instance.save_style_configs(style_spec) + invalidate_and_enqueue_previews(instance) return instance def update(self, instance, validated_data): style_spec = self.initial_data.pop("style_spec", None) instance.save_style_configs(style_spec) - return super().update(instance, validated_data) + instance = super().update(instance, validated_data) + invalidate_and_enqueue_previews(instance) + return instance class Meta: model = LayerStyle From c1b60562d3632fd27e26dc44826d4000ac271248 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Thu, 25 Jun 2026 15:10:41 -0400 Subject: [PATCH 23/36] only serialize preview images that are complete --- uvdat/core/models/styles.py | 6 +++++- uvdat/core/tests/test_frame_preview.py | 6 ++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/uvdat/core/models/styles.py b/uvdat/core/models/styles.py index c85e95b55..0e50373e6 100644 --- a/uvdat/core/models/styles.py +++ b/uvdat/core/models/styles.py @@ -9,6 +9,7 @@ from uvdat.core.frame_previews.types import FramePreviewData from .colormap import Colormap +from .frame_preview import PreviewStatus from .layer import Layer from .project import Project from .querysets import ProjectQuerySet @@ -247,7 +248,10 @@ def _multiframe_previews_by_frame(self, layer=None) -> list[RasterFramePreview | def serialize_frame_preview( preview: RasterFramePreview | None, ) -> FramePreviewData | None: - if preview is None or not preview.image: + + if preview is None: + return None + if preview.status != PreviewStatus.COMPLETE or not preview.image: return None return FramePreviewData( url=preview.image.url, diff --git a/uvdat/core/tests/test_frame_preview.py b/uvdat/core/tests/test_frame_preview.py index 8a89aa426..354a07cb3 100644 --- a/uvdat/core/tests/test_frame_preview.py +++ b/uvdat/core/tests/test_frame_preview.py @@ -9,6 +9,7 @@ raster_source_filter_kwargs, ) from uvdat.core.models import RasterFramePreview +from uvdat.core.models.frame_preview import PreviewStatus from uvdat.core.tasks.frame_preview import ( FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION, FRAME_PREVIEW_MAX_PX, @@ -89,6 +90,7 @@ def test_multiframe_previews_for_style_ordered_by_frame_index( preview_0 = RasterFramePreview.objects.create( layer_style=layer_style, layer_frame=frame_0, + status=PreviewStatus.COMPLETE, width=100, height=80, bounds={"srs": "EPSG:4326", "xmin": -1, "xmax": 1, "ymin": -2, "ymax": 2}, @@ -97,6 +99,7 @@ def test_multiframe_previews_for_style_ordered_by_frame_index( preview_2 = RasterFramePreview.objects.create( layer_style=layer_style, layer_frame=frame_2, + status=PreviewStatus.COMPLETE, width=200, height=150, bounds={"srs": "EPSG:4326", "xmin": -3, "xmax": 3, "ymin": -4, "ymax": 4}, @@ -133,6 +136,7 @@ def test_preview_bounds_includes_corners( preview = RasterFramePreview.objects.create( layer_style=layer_style, layer_frame=frame, + status=PreviewStatus.COMPLETE, width=100, height=80, bounds={ @@ -170,6 +174,7 @@ def test_layer_style_api_includes_multiframe_previews( preview = RasterFramePreview.objects.create( layer_style=layer_style, layer_frame=frame_0, + status=PreviewStatus.COMPLETE, width=100, height=100, bounds={}, @@ -210,6 +215,7 @@ def test_layer_api_includes_multiframe_previews( preview = RasterFramePreview.objects.create( layer_style=layer_style, layer_frame=frame_0, + status=PreviewStatus.COMPLETE, width=100, height=100, bounds={}, From 1dd20cd14282ac5e4c867e90c6557219baff976b Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Thu, 25 Jun 2026 15:19:32 -0400 Subject: [PATCH 24/36] preview_status return --- uvdat/core/rest/serializers.py | 24 +++++++- uvdat/core/tests/test_frame_preview.py | 81 ++++++++++++++++++++++---- 2 files changed, 94 insertions(+), 11 deletions(-) diff --git a/uvdat/core/rest/serializers.py b/uvdat/core/rest/serializers.py index 64a646488..41da2cee5 100644 --- a/uvdat/core/rest/serializers.py +++ b/uvdat/core/rest/serializers.py @@ -5,7 +5,10 @@ from django.contrib.gis.serializers import geojson from rest_framework import serializers -from uvdat.core.frame_previews.preview_regeneration import invalidate_and_enqueue_previews +from uvdat.core.frame_previews.preview_regeneration import ( + invalidate_and_enqueue_previews, + preview_status_for_style, +) from uvdat.core.models import ( Basemap, Chart, @@ -189,29 +192,47 @@ class Meta: class LayerStyleWithPreviewsSerializer(LayerStyleSerializer): multiframe_previews = serializers.SerializerMethodField() + preview_status = serializers.SerializerMethodField() def _preview_layer(self, obj): return self.context.get("preview_layer") or obj.layer + def get_preview_status(self, obj): + return preview_status_for_style(obj) + def get_multiframe_previews(self, obj): + if preview_status_for_style(obj) != "ready": + return None return obj.multiframe_previews(layer=self._preview_layer(obj)) def to_representation(self, instance): data = super().to_representation(instance) _omit_null_field(data, "multiframe_previews") + _omit_null_field(data, "preview_status") return data class LayerSerializer(serializers.ModelSerializer): default_style = LayerStyleSerializer() multiframe_previews = serializers.SerializerMethodField() + preview_status = serializers.SerializerMethodField() + + def get_preview_status(self, obj): + if obj.default_style_id is None: + return None + return preview_status_for_style(obj.default_style) def get_multiframe_previews(self, obj): + if obj.default_style_id is None: + return None + if preview_status_for_style(obj.default_style) != "ready": + return None return obj.default_multiframe_previews() def to_representation(self, instance): data = super().to_representation(instance) _omit_null_field(data, "multiframe_previews") + _omit_null_field(data, "preview_status") return data class Meta: @@ -223,6 +244,7 @@ class Meta: "dataset", "default_style", "multiframe_previews", + "preview_status", ] diff --git a/uvdat/core/tests/test_frame_preview.py b/uvdat/core/tests/test_frame_preview.py index 354a07cb3..fbbc6e260 100644 --- a/uvdat/core/tests/test_frame_preview.py +++ b/uvdat/core/tests/test_frame_preview.py @@ -169,9 +169,9 @@ def test_layer_style_api_includes_multiframe_previews( project.set_collaborators([user]) project.datasets.set([layer_style.layer.dataset]) frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) - layer_frame_factory(layer=layer_style.layer, index=1) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) - preview = RasterFramePreview.objects.create( + preview_0 = RasterFramePreview.objects.create( layer_style=layer_style, layer_frame=frame_0, status=PreviewStatus.COMPLETE, @@ -179,22 +179,68 @@ def test_layer_style_api_includes_multiframe_previews( height=100, bounds={}, ) - preview.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_1, + status=PreviewStatus.COMPLETE, + width=100, + height=100, + bounds={}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) resp = authenticated_api_client.get(f"/api/v1/layer-styles/{layer_style.id}/") assert resp.status_code == 200 data = resp.json() + assert data["preview_status"] == "ready" assert data["multiframe_previews"] == [ { - "url": preview.image.url, + "url": preview_0.image.url, + "width": 100, + "height": 100, + "bounds": {}, + }, + { + "url": preview_1.image.url, "width": 100, "height": 100, "bounds": {}, }, - None, ] +@pytest.mark.django_db +def test_api_omits_previews_while_generating( + authenticated_api_client, + layer_style_factory, + layer_frame_factory, + project, + user, +): + layer_style = layer_style_factory() + project.set_collaborators([user]) + project.datasets.set([layer_style.layer.dataset]) + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + layer_frame_factory(layer=layer_style.layer, index=1) + + preview = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_0, + status=PreviewStatus.COMPLETE, + width=100, + height=100, + bounds={}, + ) + preview.image.save("frame-0.png", ContentFile(b"png0"), save=True) + + resp = authenticated_api_client.get(f"/api/v1/layer-styles/{layer_style.id}/") + assert resp.status_code == 200 + data = resp.json() + assert data["preview_status"] == "generating" + assert "multiframe_previews" not in data + + @pytest.mark.django_db def test_layer_api_includes_multiframe_previews( authenticated_api_client, @@ -210,9 +256,9 @@ def test_layer_api_includes_multiframe_previews( project.set_collaborators([user]) project.datasets.set([layer.dataset]) frame_0 = layer_frame_factory(layer=layer, index=0) - layer_frame_factory(layer=layer, index=1) + frame_1 = layer_frame_factory(layer=layer, index=1) - preview = RasterFramePreview.objects.create( + preview_0 = RasterFramePreview.objects.create( layer_style=layer_style, layer_frame=frame_0, status=PreviewStatus.COMPLETE, @@ -220,18 +266,33 @@ def test_layer_api_includes_multiframe_previews( height=100, bounds={}, ) - preview.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_1, + status=PreviewStatus.COMPLETE, + width=100, + height=100, + bounds={}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) resp = authenticated_api_client.get(f"/api/v1/layers/{layer.id}/") assert resp.status_code == 200 data = resp.json() + assert data["preview_status"] == "ready" assert data["multiframe_previews"] == [ { - "url": preview.image.url, + "url": preview_0.image.url, + "width": 100, + "height": 100, + "bounds": {}, + }, + { + "url": preview_1.image.url, "width": 100, "height": 100, "bounds": {}, }, - None, ] assert "multiframe_previews" not in data.get("default_style", {}) From 4cad529198a71336472d379c513d5a9ffaa947d4 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Thu, 25 Jun 2026 15:25:31 -0400 Subject: [PATCH 25/36] fixing tests --- uvdat/core/models/frame_preview.py | 3 +-- uvdat/core/tests/factories.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/uvdat/core/models/frame_preview.py b/uvdat/core/models/frame_preview.py index f78f46559..26fc7b43c 100644 --- a/uvdat/core/models/frame_preview.py +++ b/uvdat/core/models/frame_preview.py @@ -5,7 +5,6 @@ from s3_file_field import S3FileField from .layer import LayerFrame -from .styles import LayerStyle class PreviewStatus(models.TextChoices): @@ -17,7 +16,7 @@ class PreviewStatus(models.TextChoices): class RasterFramePreview(models.Model): layer_style = models.ForeignKey( - LayerStyle, + "LayerStyle", related_name="frame_previews", on_delete=models.CASCADE, ) diff --git a/uvdat/core/tests/factories.py b/uvdat/core/tests/factories.py index d392a6293..f5de15e35 100644 --- a/uvdat/core/tests/factories.py +++ b/uvdat/core/tests/factories.py @@ -132,7 +132,7 @@ class Meta: name = factory.Faker("name") layer = factory.SubFactory(LayerFactory) - vector = factory.SubFactory(VectorDataFactory) + vector = None raster = factory.SubFactory(RasterDataFactory) From 2a220ac7409e1f5cf28dcdd97e37ee3ed898ef66 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Thu, 25 Jun 2026 16:15:12 -0400 Subject: [PATCH 26/36] celery task for regenerating styles --- uvdat/core/frame_previews/fingerprint.py | 13 + .../frame_previews/preview_regeneration.py | 18 +- uvdat/core/management/commands/ingest.py | 8 +- uvdat/core/tasks/frame_preview.py | 270 ++++++++++++++++-- 4 files changed, 259 insertions(+), 50 deletions(-) create mode 100644 uvdat/core/frame_previews/fingerprint.py diff --git a/uvdat/core/frame_previews/fingerprint.py b/uvdat/core/frame_previews/fingerprint.py new file mode 100644 index 000000000..07cb9e49a --- /dev/null +++ b/uvdat/core/frame_previews/fingerprint.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +import hashlib +import json +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from uvdat.core.models import LayerStyle + + +def style_fingerprint(layer_style: LayerStyle) -> str: + payload = json.dumps(layer_style.repr_style_configs(), sort_keys=True, default=str) + return hashlib.sha256(payload.encode()).hexdigest() diff --git a/uvdat/core/frame_previews/preview_regeneration.py b/uvdat/core/frame_previews/preview_regeneration.py index 51c52c585..39896a3c4 100644 --- a/uvdat/core/frame_previews/preview_regeneration.py +++ b/uvdat/core/frame_previews/preview_regeneration.py @@ -1,18 +1,10 @@ from __future__ import annotations -import hashlib -import json - from django.utils import timezone +from uvdat.core.frame_previews.fingerprint import style_fingerprint from uvdat.core.models import LayerStyle, RasterFramePreview, TaskResult from uvdat.core.models.frame_preview import PreviewStatus -from uvdat.core.tasks.frame_preview import generate_layer_style_previews - - -def style_fingerprint(layer_style: LayerStyle) -> str: - payload = json.dumps(layer_style.repr_style_configs(), sort_keys=True, default=str) - return hashlib.sha256(payload.encode()).hexdigest() def style_needs_previews(layer_style: LayerStyle) -> bool: @@ -75,11 +67,9 @@ def invalidate_and_enqueue_previews(layer_style: LayerStyle) -> TaskResult | Non }, ) - generate_layer_style_previews.delay( - layer_style.id, - fingerprint=fingerprint, - result_id=result.id, - ) + from uvdat.core.tasks.frame_preview import generate_layer_style_previews # noqa: PLC0415 + + generate_layer_style_previews.delay(layer_style.id, fingerprint, result.id) return result diff --git a/uvdat/core/management/commands/ingest.py b/uvdat/core/management/commands/ingest.py index f3fe30f68..d0d542dd4 100644 --- a/uvdat/core/management/commands/ingest.py +++ b/uvdat/core/management/commands/ingest.py @@ -17,11 +17,9 @@ import djclick as click import pooch +from uvdat.core.frame_previews.preview_regeneration import invalidate_and_enqueue_previews from uvdat.core.models import Chart, Dataset, FileItem, Layer, Project -from uvdat.core.tasks.frame_preview import ( - ensure_default_layer_style, - generate_layer_style_previews, -) +from uvdat.core.tasks.frame_preview import ensure_default_layer_style DATA_FOLDER = Path(os.environ.get("INGEST_BIND_MOUNT_POINT", "sample_data")) DOWNLOADS_FOLDER = DATA_FOLDER / "downloads" @@ -169,7 +167,7 @@ def generate_ingest_multiframe_previews(converted_dataset_names: set[str]) -> in if not layer.is_multiframe_raster(): continue style = ensure_default_layer_style(layer, project) - generate_layer_style_previews(style.id) + invalidate_and_enqueue_previews(style) style_count += 1 return style_count diff --git a/uvdat/core/tasks/frame_preview.py b/uvdat/core/tasks/frame_preview.py index 35dec5f03..cca01750b 100644 --- a/uvdat/core/tasks/frame_preview.py +++ b/uvdat/core/tasks/frame_preview.py @@ -1,5 +1,6 @@ from __future__ import annotations +from dataclasses import dataclass import io import json import logging @@ -11,6 +12,7 @@ from django_large_image import tilesource, utilities from PIL import Image +from uvdat.core.frame_previews.fingerprint import style_fingerprint from uvdat.core.frame_previews.raster_style import ( apply_source_filters_to_style_query, build_raster_tiles_style_query, @@ -23,17 +25,30 @@ Project, RasterData, RasterFramePreview, + TaskResult, ) +from uvdat.core.models.frame_preview import PreviewStatus if TYPE_CHECKING: from uvdat.core.frame_previews.types import FramePreviewBounds +"""Celery tasks and helpers for multiframe raster frame preview images. + +Previews are styled PNG thumbnails stored on ``RasterFramePreview`` rows. They +let the frontend show a fast full-frame image before tile loading during frame +scrubbing. Generation is keyed by a style fingerprint so rapid successive style +saves do not publish stale images from an older task run. +""" + logger = logging.getLogger(__name__) +# Thumbnail sizing: default to 1/8 of FRAME_PREVIEW_MAX_PX (512px), but never +# below FRAME_PREVIEW_MIN_PX when the source raster is large enough to allow it. FRAME_PREVIEW_MAX_PX = 4096 FRAME_PREVIEW_MIN_PX = 1024 FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION = 1 / 8 +# Applied when ingest creates a default style for a new multiframe raster layer. DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC = { "default_frame": 0, "opacity": 1, @@ -43,6 +58,38 @@ } +@dataclass(frozen=True) +class _PreviewGenerationContext: + """Immutable inputs shared across all frames in one task invocation.""" + + layer_style: LayerStyle + fingerprint: str + base_style_query: dict[str, Any] + resolution_fraction: float | None = None + + @property + def layer_style_id(self) -> int: + return self.layer_style.id + + +@dataclass(frozen=True) +class _FramePreviewImage: + """PNG payload and metadata produced by ``generate_frame_preview_png``.""" + + png_bytes: bytes + width: int + height: int + bounds: FramePreviewBounds | None + + +@dataclass(frozen=True) +class _PreviewGenerationStats: + """Per-task frame counts written to ``TaskResult.outputs`` on completion.""" + + ready_count: int + failed_count: int + + def resolve_resolution_fraction(resolution_fraction: float | None = None) -> float: if resolution_fraction is None: return FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION @@ -59,10 +106,12 @@ def resolve_preview_max_dimension( resolution_fraction: float | None = None, raster_max_dimension: int | None = None, ) -> int: + """Pick a thumbnail edge length, clamped to the source raster's size.""" fraction = resolve_resolution_fraction(resolution_fraction) fractional = round(FRAME_PREVIEW_MAX_PX * fraction) if not raster_max_dimension or raster_max_dimension < 2: return max(2, fractional) + # Small rasters use their native size; larger ones get at least MIN_PX. floor = min(raster_max_dimension, FRAME_PREVIEW_MIN_PX) return max(2, min(max(fractional, floor), raster_max_dimension)) @@ -109,6 +158,12 @@ def generate_frame_preview_png( base_style_query: dict[str, Any], resolution_fraction: float | None = None, ) -> tuple[bytes, int, int, FramePreviewBounds | None]: + """Render one frame as a styled PNG via large-image. + + Frame selection is passed through ``source_filters`` (e.g. ``{"frame": 3}``), + not embedded in the style query, so one style query can be reused for every + frame in a multiframe layer. + """ style_query = apply_source_filters_to_style_query(base_style_query, source_filters) style = json.dumps(style_query) if style_query else None source_kwargs = raster_source_filter_kwargs(source_filters) @@ -134,6 +189,7 @@ def generate_frame_preview_png( def ensure_default_layer_style(layer: Layer, project: Project) -> LayerStyle: + """Return the project's Default style for a layer, creating it on ingest if needed.""" style = LayerStyle.objects.filter(layer=layer, project=project, name="Default").first() if style is None: style = LayerStyle.objects.create(name="Default", layer=layer, project=project) @@ -144,20 +200,178 @@ def ensure_default_layer_style(layer: Layer, project: Project) -> LayerStyle: return style +def _fingerprint_matches(layer_style: LayerStyle, fingerprint: str) -> bool: + """Return whether the style's current configs still match the task fingerprint. + + ``repr_style_configs`` always reads from the database, so this detects a + style save that happened after this task was enqueued. + """ + return style_fingerprint(layer_style) == fingerprint + + +def _open_task_result(result_id: int | None, layer_style_id: int) -> TaskResult | None: + """Load the TaskResult for this run, or None if it was already superseded.""" + if result_id is None: + return None + + result = TaskResult.objects.filter(id=result_id).first() + if result is None or result.completed is not None: + logger.info( + "Skipping preview generation for style=%s; task result %s already closed", + layer_style_id, + result_id, + ) + return None + return result + + +def _save_frame_preview( + preview: RasterFramePreview, + layer_style_id: int, + frame_index: int, + image: _FramePreviewImage, +) -> None: + """Persist a generated preview and mark the row complete (API-servable).""" + preview.width = image.width + preview.height = image.height + preview.bounds = image.bounds or {} + preview.status = PreviewStatus.COMPLETE + preview.image.save( + f"frame-previews/{layer_style_id}/{frame_index}.png", + ContentFile(image.png_bytes), + save=False, + ) + preview.save() + + +def _mark_frame_preview_failed( + preview: RasterFramePreview, + layer_style: LayerStyle, + fingerprint: str, +) -> None: + """Mark a row failed only when both the row and style still match this task.""" + if preview.style_fingerprint == fingerprint and _fingerprint_matches(layer_style, fingerprint): + preview.status = PreviewStatus.FAILED + preview.save(update_fields=["status"]) + + +def _process_frame_preview(ctx: _PreviewGenerationContext, frame) -> str: + """Generate one frame preview. + + Returns ``ready``, ``failed``, ``skipped``, or ``superseded``. A superseded + outcome means a newer style save arrived and this task must stop writing. + """ + if not _fingerprint_matches(ctx.layer_style, ctx.fingerprint): + return "superseded" + + try: + preview = RasterFramePreview.objects.get( + layer_style=ctx.layer_style, + layer_frame=frame, + ) + except RasterFramePreview.DoesNotExist: + logger.warning( + "Missing preview row for style=%s frame=%s; skipping", + ctx.layer_style_id, + frame.id, + ) + return "skipped" + + # Row fingerprint is set on enqueue; skip frames already claimed by a newer save. + if preview.style_fingerprint != ctx.fingerprint: + return "skipped" + + try: + png_bytes, width, height, bounds = generate_frame_preview_png( + frame.raster, + frame.source_filters, + ctx.base_style_query, + ctx.resolution_fraction, + ) + except Exception: + logger.exception( + "Failed to generate frame preview for style=%s frame=%s", + ctx.layer_style_id, + frame.id, + ) + _mark_frame_preview_failed(preview, ctx.layer_style, ctx.fingerprint) + return "failed" + + # PNG generation is expensive; re-check before writing to storage. + if not _fingerprint_matches(ctx.layer_style, ctx.fingerprint): + return "superseded" + + _save_frame_preview( + preview, + ctx.layer_style_id, + frame.index, + _FramePreviewImage(png_bytes, width, height, bounds), + ) + return "ready" + + +def _complete_preview_task( + result: TaskResult | None, + ctx: _PreviewGenerationContext, + stats: _PreviewGenerationStats, +) -> None: + """Finalize the TaskResult, which triggers a WebSocket notification.""" + if not _fingerprint_matches(ctx.layer_style, ctx.fingerprint): + logger.info( + "Skipping task completion for style=%s; style superseded after loop", + ctx.layer_style_id, + ) + return + + if result is None: + return + + result.write_outputs( + { + "layer_style_id": ctx.layer_style_id, + "layer_id": ctx.layer_style.layer_id, + "fingerprint": ctx.fingerprint, + "ready_count": stats.ready_count, + "failed_count": stats.failed_count, + } + ) + result.complete() + + @shared_task def generate_layer_style_previews( layer_style_id: int, - resolution_fraction: float | None = None, - *, - fingerprint: str | None = None, + fingerprint: str, result_id: int | None = None, + resolution_fraction: float | None = None, ): + """Generate styled PNG previews for every frame in a multiframe raster style. + + Enqueued by ``invalidate_and_enqueue_previews`` after a style save or ingest. + Preview rows are created upstream with ``creating``/``regenerating`` status and + cleared images; this task fills them in and sets ``complete`` or ``failed``. + + ``fingerprint`` is a sha256 of ``repr_style_configs()`` at enqueue time. The + task aborts whenever the live style no longer matches, so rapid double-saves + only publish previews for the latest style version. + """ started = time.perf_counter() - layer_style = LayerStyle.objects.select_related("layer").get(id=layer_style_id) + layer_style = LayerStyle.objects.select_related("layer", "project").get(id=layer_style_id) frames = layer_style.layer.multiframe_raster_frames() if len(frames) <= 1: return + if not _fingerprint_matches(layer_style, fingerprint): + logger.info( + "Skipping superseded preview generation for style=%s", + layer_style_id, + ) + return + + result = _open_task_result(result_id, layer_style_id) + if result_id is not None and result is None: + return + logger.info( "Generating %d multiframe raster previews for style=%s layer=%r", len(frames), @@ -167,40 +381,34 @@ def generate_layer_style_previews( style_spec = layer_style.repr_style_configs() colormaps_by_id = _colormaps_for_style(layer_style) - base_style_query = build_raster_tiles_style_query(style_spec, colormaps_by_id) + ctx = _PreviewGenerationContext( + layer_style=layer_style, + fingerprint=fingerprint, + base_style_query=build_raster_tiles_style_query(style_spec, colormaps_by_id), + resolution_fraction=resolution_fraction, + ) ready_count = 0 failed_count = 0 for frame in frames: - try: - png_bytes, width, height, bounds = generate_frame_preview_png( - frame.raster, - frame.source_filters, - base_style_query, - resolution_fraction, - ) - preview, _created = RasterFramePreview.objects.update_or_create( - layer_style=layer_style, - layer_frame=frame, - defaults={ - "width": width, - "height": height, - "bounds": bounds or {}, - }, - ) - preview.image.save( - f"frame-previews/{layer_style_id}/{frame.index}.png", - ContentFile(png_bytes), - save=True, + outcome = _process_frame_preview(ctx, frame) + if outcome == "superseded": + logger.info( + "Aborting preview generation for style=%s; style superseded", + layer_style_id, ) + return + if outcome == "ready": ready_count += 1 - except Exception: + elif outcome == "failed": failed_count += 1 - logger.exception( - "Failed to generate frame preview for style=%s frame=%s", - layer_style_id, - frame.id, - ) + + _complete_preview_task( + result, + ctx, + _PreviewGenerationStats(ready_count, failed_count), + ) + logger.info( "Multiframe raster previews for style=%s layer=%r: %d ready, %d failed in %.2fs", layer_style_id, From 36d25d375dbe2c46231f3d4470fbd804cee7e143 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 26 Jun 2026 08:04:53 -0400 Subject: [PATCH 27/36] handle ingestion, update fingerprinting --- uvdat/core/admin.py | 2 +- uvdat/core/frame_previews/fingerprint.py | 9 ++- .../frame_previews/preview_regeneration.py | 12 +++- uvdat/core/management/commands/ingest.py | 6 +- uvdat/core/models/styles.py | 1 + uvdat/core/tests/test_frame_preview.py | 55 ++++++++++++++++++- 6 files changed, 79 insertions(+), 6 deletions(-) diff --git a/uvdat/core/admin.py b/uvdat/core/admin.py index 7a049c0d4..43ecfa81e 100644 --- a/uvdat/core/admin.py +++ b/uvdat/core/admin.py @@ -72,7 +72,7 @@ class LayerStyleAdmin(admin.ModelAdmin): @admin.register(RasterFramePreview) class RasterFramePreviewAdmin(admin.ModelAdmin): - list_display = ["id", "layer_style", "layer_frame", "width", "height"] + list_display = ["id", "layer_style", "layer_frame", "width", "height", "status"] list_select_related = ["layer_style", "layer_frame"] diff --git a/uvdat/core/frame_previews/fingerprint.py b/uvdat/core/frame_previews/fingerprint.py index 07cb9e49a..7b519a035 100644 --- a/uvdat/core/frame_previews/fingerprint.py +++ b/uvdat/core/frame_previews/fingerprint.py @@ -8,6 +8,13 @@ from uvdat.core.models import LayerStyle +def _fingerprint_payload(configs: dict) -> str: + normalized = dict(configs) + normalized["default_frame"] = int(normalized["default_frame"]) + normalized["opacity"] = float(normalized["opacity"]) + return json.dumps(normalized, sort_keys=True, default=str) + + def style_fingerprint(layer_style: LayerStyle) -> str: - payload = json.dumps(layer_style.repr_style_configs(), sort_keys=True, default=str) + payload = _fingerprint_payload(layer_style.repr_style_configs()) return hashlib.sha256(payload.encode()).hexdigest() diff --git a/uvdat/core/frame_previews/preview_regeneration.py b/uvdat/core/frame_previews/preview_regeneration.py index 39896a3c4..2b8593c51 100644 --- a/uvdat/core/frame_previews/preview_regeneration.py +++ b/uvdat/core/frame_previews/preview_regeneration.py @@ -48,10 +48,15 @@ def mark_previews_regenerating(layer_style: LayerStyle, fingerprint: str) -> lis return frame_ids -def invalidate_and_enqueue_previews(layer_style: LayerStyle) -> TaskResult | None: +def invalidate_and_enqueue_previews( + layer_style: LayerStyle, + *, + asynchronous: bool = True, +) -> TaskResult | None: if not style_needs_previews(layer_style): return None + layer_style.refresh_from_db() fingerprint = style_fingerprint(layer_style) mark_previews_regenerating(layer_style, fingerprint) supersede_pending_preview_tasks(layer_style.id) @@ -69,7 +74,10 @@ def invalidate_and_enqueue_previews(layer_style: LayerStyle) -> TaskResult | Non from uvdat.core.tasks.frame_preview import generate_layer_style_previews # noqa: PLC0415 - generate_layer_style_previews.delay(layer_style.id, fingerprint, result.id) + if asynchronous: + generate_layer_style_previews.delay(layer_style.id, fingerprint, result.id) + else: + generate_layer_style_previews.apply(args=(layer_style.id, fingerprint, result.id)) return result diff --git a/uvdat/core/management/commands/ingest.py b/uvdat/core/management/commands/ingest.py index d0d542dd4..47718de66 100644 --- a/uvdat/core/management/commands/ingest.py +++ b/uvdat/core/management/commands/ingest.py @@ -167,7 +167,11 @@ def generate_ingest_multiframe_previews(converted_dataset_names: set[str]) -> in if not layer.is_multiframe_raster(): continue style = ensure_default_layer_style(layer, project) - invalidate_and_enqueue_previews(style) + click.echo( + f"\t\t Generating multiframe raster previews for style {style.name!r} " + f"(layer {layer.name!r})..." + ) + invalidate_and_enqueue_previews(style, asynchronous=False) style_count += 1 return style_count diff --git a/uvdat/core/models/styles.py b/uvdat/core/models/styles.py index 0e50373e6..45aaa1846 100644 --- a/uvdat/core/models/styles.py +++ b/uvdat/core/models/styles.py @@ -166,6 +166,7 @@ def save_style_configs(self, style_spec): # noqa: C901, PLR0912, PLR0915 filter_config.save() filter_config_ids.append(filter_config.id) FilterConfig.objects.filter(style=self).exclude(id__in=filter_config_ids).delete() + self.save(update_fields=["default_frame", "opacity"]) def repr_style_configs(self): colors = [] diff --git a/uvdat/core/tests/test_frame_preview.py b/uvdat/core/tests/test_frame_preview.py index fbbc6e260..cd86eda7c 100644 --- a/uvdat/core/tests/test_frame_preview.py +++ b/uvdat/core/tests/test_frame_preview.py @@ -3,14 +3,17 @@ from django.core.files.base import ContentFile import pytest +from uvdat.core.frame_previews.fingerprint import style_fingerprint +from uvdat.core.frame_previews.preview_regeneration import invalidate_and_enqueue_previews from uvdat.core.frame_previews.raster_style import ( apply_source_filters_to_style_query, build_thumbnail_style_query, raster_source_filter_kwargs, ) -from uvdat.core.models import RasterFramePreview +from uvdat.core.models import LayerStyle, RasterFramePreview from uvdat.core.models.frame_preview import PreviewStatus from uvdat.core.tasks.frame_preview import ( + DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC, FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION, FRAME_PREVIEW_MAX_PX, FRAME_PREVIEW_MIN_PX, @@ -66,6 +69,56 @@ def test_resolve_preview_max_dimension( ) +@pytest.mark.django_db +def test_style_fingerprint_matches_db_after_ingest_style_setup(layer_style_factory): + """Enqueue fingerprint must match what the Celery task reads from the database.""" + style = layer_style_factory() + style.save_style_configs(DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC) + + assert style_fingerprint(style) == style_fingerprint(LayerStyle.objects.get(pk=style.pk)) + + +@pytest.mark.django_db +def test_invalidate_and_enqueue_previews_uses_db_fingerprint( + layer_style_factory, + layer_frame_factory, + mocker, +): + layer_style = layer_style_factory() + layer_frame_factory(layer=layer_style.layer, index=0) + layer_frame_factory(layer=layer_style.layer, index=1) + layer_style.save_style_configs(DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC) + + delay = mocker.patch("uvdat.core.tasks.frame_preview.generate_layer_style_previews.delay") + + invalidate_and_enqueue_previews(layer_style) + + delay.assert_called_once() + _, fingerprint, _ = delay.call_args.args + assert fingerprint == style_fingerprint(LayerStyle.objects.get(pk=layer_style.pk)) + + +@pytest.mark.django_db +def test_invalidate_and_enqueue_previews_runs_synchronously( + layer_style_factory, + layer_frame_factory, + mocker, +): + layer_style = layer_style_factory() + layer_frame_factory(layer=layer_style.layer, index=0) + layer_frame_factory(layer=layer_style.layer, index=1) + layer_style.save_style_configs(DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC) + + delay = mocker.patch("uvdat.core.tasks.frame_preview.generate_layer_style_previews.delay") + apply = mocker.patch("uvdat.core.tasks.frame_preview.generate_layer_style_previews.apply") + + invalidate_and_enqueue_previews(layer_style, asynchronous=False) + + apply.assert_called_once() + delay.assert_not_called() + assert apply.call_args.kwargs["args"][0] == layer_style.id + + @pytest.mark.django_db def test_multiframe_previews_for_style_returns_none_for_single_frame( layer_style_factory, From b1ce08820736ab1a4320dc8ccc481da9e877feac Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 26 Jun 2026 08:46:25 -0400 Subject: [PATCH 28/36] swap to using annotate to reduce python aggregation --- .../frame_previews/preview_regeneration.py | 28 +++--- uvdat/core/models/layer.py | 2 +- uvdat/core/models/styles.py | 42 ++++----- uvdat/core/rest/querysets.py | 93 +++++++++++++++++-- uvdat/core/rest/serializers.py | 13 ++- uvdat/core/tests/test_frame_preview.py | 73 +++++++++++++-- 6 files changed, 185 insertions(+), 66 deletions(-) diff --git a/uvdat/core/frame_previews/preview_regeneration.py b/uvdat/core/frame_previews/preview_regeneration.py index 2b8593c51..c71a11604 100644 --- a/uvdat/core/frame_previews/preview_regeneration.py +++ b/uvdat/core/frame_previews/preview_regeneration.py @@ -3,7 +3,7 @@ from django.utils import timezone from uvdat.core.frame_previews.fingerprint import style_fingerprint -from uvdat.core.models import LayerStyle, RasterFramePreview, TaskResult +from uvdat.core.models import Layer, LayerStyle, RasterFramePreview, TaskResult from uvdat.core.models.frame_preview import PreviewStatus @@ -102,16 +102,18 @@ def preview_status_for_style(layer_style: LayerStyle) -> str | None: ): return "ready" - statuses = [ - previews_by_frame[frame.id].status for frame in frames if frame.id in previews_by_frame - ] + return "notready" - if any( - status in (PreviewStatus.CREATING, PreviewStatus.REGENERATING) for status in statuses - ) or any(status == PreviewStatus.COMPLETE for status in statuses): - result = "generating" - elif any(status == PreviewStatus.FAILED for status in statuses): - result = "failed" - else: - result = "none" - return result + +def get_layer_style_preview_status(layer_style: LayerStyle) -> str | None: + if "preview_status" in layer_style.__dict__: + return layer_style.preview_status + return preview_status_for_style(layer_style) + + +def get_layer_preview_status(layer: Layer) -> str | None: + if "preview_status" in layer.__dict__: + return layer.preview_status + if layer.default_style_id is None: + return None + return get_layer_style_preview_status(layer.default_style) diff --git a/uvdat/core/models/layer.py b/uvdat/core/models/layer.py index c32bbe018..147310ad2 100644 --- a/uvdat/core/models/layer.py +++ b/uvdat/core/models/layer.py @@ -44,7 +44,7 @@ def is_multiframe_raster(self) -> bool: return len(prefetched) > 1 return self.frames.filter(raster__isnull=False).count() > 1 - def default_multiframe_previews(self) -> list[FramePreviewData | None] | None: + def default_multiframe_previews(self) -> list[FramePreviewData] | None: if self.default_style_id is None: return None return self.default_style.multiframe_previews(layer=self) diff --git a/uvdat/core/models/styles.py b/uvdat/core/models/styles.py index 45aaa1846..75f420b8f 100644 --- a/uvdat/core/models/styles.py +++ b/uvdat/core/models/styles.py @@ -231,29 +231,8 @@ def repr_style_configs(self): "filters": filters, } - def _multiframe_previews_by_frame(self, layer=None) -> list[RasterFramePreview | None] | None: - layer = layer or self.layer - frames = layer.multiframe_raster_frames() - if len(frames) <= 1: - return None - - previews_by_frame_id = { - preview.layer_frame_id: preview for preview in self.frame_previews.all() - } - return [ - previews_by_frame_id.get(frame.id) if frame.id in previews_by_frame_id else None - for frame in frames - ] - @staticmethod - def serialize_frame_preview( - preview: RasterFramePreview | None, - ) -> FramePreviewData | None: - - if preview is None: - return None - if preview.status != PreviewStatus.COMPLETE or not preview.image: - return None + def serialize_frame_preview(preview: RasterFramePreview) -> FramePreviewData: return FramePreviewData( url=preview.image.url, width=preview.width, @@ -261,11 +240,22 @@ def serialize_frame_preview( bounds=preview.bounds, ) - def multiframe_previews(self, layer=None) -> list[FramePreviewData | None] | None: - previews = self._multiframe_previews_by_frame(layer=layer) - if previews is None: + def multiframe_previews(self, layer=None) -> list[FramePreviewData] | None: + layer = layer or self.layer + frames = layer.multiframe_raster_frames() + if len(frames) <= 1: + return None + + previews_by_frame_id = { + preview.layer_frame_id: preview for preview in self.frame_previews.all() + } + ordered = [previews_by_frame_id.get(frame.id) for frame in frames] + if not all( + preview is not None and preview.status == PreviewStatus.COMPLETE and preview.image + for preview in ordered + ): return None - return [self.serialize_frame_preview(preview) for preview in previews] + return [self.serialize_frame_preview(preview) for preview in ordered] def get_default_colormap(): diff --git a/uvdat/core/rest/querysets.py b/uvdat/core/rest/querysets.py index bf567cd48..f8337c386 100644 --- a/uvdat/core/rest/querysets.py +++ b/uvdat/core/rest/querysets.py @@ -1,8 +1,9 @@ from __future__ import annotations -from django.db.models import Prefetch, QuerySet +from django.db.models import Case, CharField, Count, F, Prefetch, Q, QuerySet, Value, When from uvdat.core.models import Layer, LayerFrame, LayerStyle, RasterFramePreview +from uvdat.core.models.frame_preview import PreviewStatus RASTER_FRAMES_QUERYSET = ( LayerFrame.objects.filter(raster__isnull=False).select_related("raster").order_by("index") @@ -26,6 +27,74 @@ ) +def _preview_status_case() -> Case: + return Case( + When(_raster_frame_count__lte=1, then=Value(None, output_field=CharField(null=True))), + When( + _complete_with_image_count=F("_raster_frame_count"), + then=Value("ready"), + ), + default=Value("notready"), + output_field=CharField(null=True), + ) + + +def _annotate_layer_style_preview_counts(queryset: QuerySet) -> QuerySet: + return queryset.annotate( + _raster_frame_count=Count( + "layer__frames", + filter=Q(layer__frames__raster__isnull=False), + distinct=True, + ), + _complete_with_image_count=Count( + "frame_previews", + filter=Q( + frame_previews__status=PreviewStatus.COMPLETE, + frame_previews__image__isnull=False, + ) + & ~Q(frame_previews__image=""), + distinct=True, + ), + ) + + +def _annotate_layer_preview_counts(queryset: QuerySet) -> QuerySet: + return queryset.annotate( + _raster_frame_count=Count( + "frames", + filter=Q(frames__raster__isnull=False), + distinct=True, + ), + _complete_with_image_count=Count( + "default_style__frame_previews", + filter=Q( + default_style__frame_previews__status=PreviewStatus.COMPLETE, + default_style__frame_previews__image__isnull=False, + ) + & ~Q(default_style__frame_previews__image=""), + distinct=True, + ), + ) + + +def annotate_layer_style_preview_status(queryset: QuerySet) -> QuerySet: + return _annotate_layer_style_preview_counts(queryset).annotate( + preview_status=_preview_status_case(), + ) + + +def annotate_layer_preview_status(queryset: QuerySet) -> QuerySet: + return _annotate_layer_preview_counts(queryset).annotate( + preview_status=Case( + When( + default_style_id__isnull=True, then=Value(None, output_field=CharField(null=True)) + ), + default=_preview_status_case(), + output_field=CharField(null=True), + ), + ) + + def layer_queryset_with_previews( queryset: QuerySet | None = None, *, @@ -34,17 +103,21 @@ def layer_queryset_with_previews( # This is for /layer-styles/?layer={layer_id} so we need to prefetch the frames for all styles if for_layer_style: qs = queryset if queryset is not None else LayerStyle.objects.all() - return qs.select_related("layer", "layer__default_style").prefetch_related( - FRAME_PREVIEWS_PREFETCH, - STYLE_LAYER_RASTER_FRAMES_PREFETCH, + return annotate_layer_style_preview_status( + qs.select_related("layer", "layer__default_style").prefetch_related( + FRAME_PREVIEWS_PREFETCH, + STYLE_LAYER_RASTER_FRAMES_PREFETCH, + ) ) qs = queryset if queryset is not None else Layer.objects.all() # This is for /layers/{layer_id} so we need to prefetch the frames and the default style - return qs.select_related("dataset", "default_style").prefetch_related( - LAYER_RASTER_FRAMES_PREFETCH, - Prefetch( - "default_style__frame_previews", - queryset=RasterFramePreview.objects.select_related("layer_frame"), - ), + return annotate_layer_preview_status( + qs.select_related("dataset", "default_style").prefetch_related( + LAYER_RASTER_FRAMES_PREFETCH, + Prefetch( + "default_style__frame_previews", + queryset=RasterFramePreview.objects.select_related("layer_frame"), + ), + ) ) diff --git a/uvdat/core/rest/serializers.py b/uvdat/core/rest/serializers.py index 41da2cee5..5c53b8306 100644 --- a/uvdat/core/rest/serializers.py +++ b/uvdat/core/rest/serializers.py @@ -6,8 +6,9 @@ from rest_framework import serializers from uvdat.core.frame_previews.preview_regeneration import ( + get_layer_preview_status, + get_layer_style_preview_status, invalidate_and_enqueue_previews, - preview_status_for_style, ) from uvdat.core.models import ( Basemap, @@ -198,10 +199,10 @@ def _preview_layer(self, obj): return self.context.get("preview_layer") or obj.layer def get_preview_status(self, obj): - return preview_status_for_style(obj) + return get_layer_style_preview_status(obj) def get_multiframe_previews(self, obj): - if preview_status_for_style(obj) != "ready": + if get_layer_style_preview_status(obj) != "ready": return None return obj.multiframe_previews(layer=self._preview_layer(obj)) @@ -218,14 +219,12 @@ class LayerSerializer(serializers.ModelSerializer): preview_status = serializers.SerializerMethodField() def get_preview_status(self, obj): - if obj.default_style_id is None: - return None - return preview_status_for_style(obj.default_style) + return get_layer_preview_status(obj) def get_multiframe_previews(self, obj): if obj.default_style_id is None: return None - if preview_status_for_style(obj.default_style) != "ready": + if get_layer_preview_status(obj) != "ready": return None return obj.default_multiframe_previews() diff --git a/uvdat/core/tests/test_frame_preview.py b/uvdat/core/tests/test_frame_preview.py index cd86eda7c..71c23aed8 100644 --- a/uvdat/core/tests/test_frame_preview.py +++ b/uvdat/core/tests/test_frame_preview.py @@ -131,7 +131,7 @@ def test_multiframe_previews_for_style_returns_none_for_single_frame( @pytest.mark.django_db -def test_multiframe_previews_for_style_ordered_by_frame_index( +def test_multiframe_previews_for_style_returns_none_when_partial( layer_style_factory, layer_frame_factory, ): @@ -159,6 +159,47 @@ def test_multiframe_previews_for_style_ordered_by_frame_index( ) preview_2.image.save("frame-2.png", ContentFile(b"png2"), save=True) + assert layer_style.multiframe_previews() is None + + +@pytest.mark.django_db +def test_multiframe_previews_for_style_ordered_by_frame_index( + layer_style_factory, + layer_frame_factory, +): + layer_style = layer_style_factory() + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) + frame_2 = layer_frame_factory(layer=layer_style.layer, index=2) + + preview_0 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_0, + status=PreviewStatus.COMPLETE, + width=100, + height=80, + bounds={"srs": "EPSG:4326", "xmin": -1, "xmax": 1, "ymin": -2, "ymax": 2}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_1, + status=PreviewStatus.COMPLETE, + width=120, + height=90, + bounds={"srs": "EPSG:4326", "xmin": -2, "xmax": 2, "ymin": -3, "ymax": 3}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + preview_2 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_2, + status=PreviewStatus.COMPLETE, + width=200, + height=150, + bounds={"srs": "EPSG:4326", "xmin": -3, "xmax": 3, "ymin": -4, "ymax": 4}, + ) + preview_2.image.save("frame-2.png", ContentFile(b"png2"), save=True) + previews = layer_style.multiframe_previews() assert previews == [ { @@ -167,7 +208,12 @@ def test_multiframe_previews_for_style_ordered_by_frame_index( "height": 80, "bounds": preview_0.bounds, }, - None, + { + "url": preview_1.image.url, + "width": 120, + "height": 90, + "bounds": preview_1.bounds, + }, { "url": preview_2.image.url, "width": 200, @@ -183,12 +229,12 @@ def test_preview_bounds_includes_corners( layer_frame_factory, ): layer_style = layer_style_factory() - frame = layer_frame_factory(layer=layer_style.layer, index=0) - layer_frame_factory(layer=layer_style.layer, index=1) + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) - preview = RasterFramePreview.objects.create( + preview_0 = RasterFramePreview.objects.create( layer_style=layer_style, - layer_frame=frame, + layer_frame=frame_0, status=PreviewStatus.COMPLETE, width=100, height=80, @@ -204,7 +250,16 @@ def test_preview_bounds_includes_corners( "ll": {"x": -1, "y": -2}, }, ) - preview.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_1, + status=PreviewStatus.COMPLETE, + width=100, + height=80, + bounds={}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) previews = layer_style.multiframe_previews() assert previews[0]["bounds"]["ul"] == {"x": -1, "y": 2} @@ -264,7 +319,7 @@ def test_layer_style_api_includes_multiframe_previews( @pytest.mark.django_db -def test_api_omits_previews_while_generating( +def test_api_omits_previews_while_not_ready( authenticated_api_client, layer_style_factory, layer_frame_factory, @@ -290,7 +345,7 @@ def test_api_omits_previews_while_generating( resp = authenticated_api_client.get(f"/api/v1/layer-styles/{layer_style.id}/") assert resp.status_code == 200 data = resp.json() - assert data["preview_status"] == "generating" + assert data["preview_status"] == "notready" assert "multiframe_previews" not in data From d87eea5a5e1c8c1ac0a931b83c2a0585bedf8e46 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 3 Jul 2026 11:37:08 -0400 Subject: [PATCH 29/36] suppress/prevent websocket notifications for updates when run sync in ingest --- uvdat/core/management/commands/ingest.py | 8 +++- uvdat/core/models/task_result.py | 50 ++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/uvdat/core/management/commands/ingest.py b/uvdat/core/management/commands/ingest.py index 47718de66..09cfba543 100644 --- a/uvdat/core/management/commands/ingest.py +++ b/uvdat/core/management/commands/ingest.py @@ -19,6 +19,7 @@ from uvdat.core.frame_previews.preview_regeneration import invalidate_and_enqueue_previews from uvdat.core.models import Chart, Dataset, FileItem, Layer, Project +from uvdat.core.models.task_result import suppress_task_notifications from uvdat.core.tasks.frame_preview import ensure_default_layer_style DATA_FOLDER = Path(os.environ.get("INGEST_BIND_MOUNT_POINT", "sample_data")) @@ -171,7 +172,12 @@ def generate_ingest_multiframe_previews(converted_dataset_names: set[str]) -> in f"\t\t Generating multiframe raster previews for style {style.name!r} " f"(layer {layer.name!r})..." ) - invalidate_and_enqueue_previews(style, asynchronous=False) + # Run synchronously: ingest is a one-shot CLI process, so previews + # must be generated inline rather than handed to a Celery worker. + # Suppress the TaskResult WebSocket notification: there is no client + # session listening and Redis is typically unreachable here. + with suppress_task_notifications(): + invalidate_and_enqueue_previews(style, asynchronous=False) style_count += 1 return style_count diff --git a/uvdat/core/models/task_result.py b/uvdat/core/models/task_result.py index 38cede07d..3544b9e01 100644 --- a/uvdat/core/models/task_result.py +++ b/uvdat/core/models/task_result.py @@ -1,6 +1,9 @@ from __future__ import annotations +import contextlib +from contextvars import ContextVar import json +import logging from asgiref.sync import async_to_sync from channels.layers import get_channel_layer @@ -16,6 +19,26 @@ from .project import Project from .querysets import ProjectQuerySet +logger = logging.getLogger(__name__) + +# Set while running in a context (e.g. `manage.py ingest`) where TaskResult +# WebSocket notifications are meaningless: tasks run synchronously with no +# client session listening. When set, result_post_save skips the push entirely +# rather than attempting it and logging a warning. +_suppress_notifications: ContextVar[bool] = ContextVar( + "suppress_task_notifications", default=False +) + + +@contextlib.contextmanager +def suppress_task_notifications(): + """Silence TaskResult WebSocket notifications for the current context.""" + token = _suppress_notifications.set(True) + try: + yield + finally: + _suppress_notifications.reset(token) + class TaskResult(models.Model): name = models.CharField(max_length=255) @@ -74,6 +97,12 @@ def complete(self): @receiver(post_save, sender=TaskResult) def result_post_save(sender, instance, **kwargs): + # In contexts such as `manage.py ingest`, tasks run synchronously with no + # client session listening, so the push is meaningless -- skip it silently + # (no attempt, no warning). See suppress_task_notifications(). + if _suppress_notifications.get(): + return + # Prevent circular import from uvdat.core.rest.serializers import TaskResultSerializer # noqa: PLC0415 @@ -82,6 +111,21 @@ def result_post_save(sender, instance, **kwargs): if instance.project: group_name = f"analytics_{instance.project.id}" channel_layer = get_channel_layer() - async_to_sync(channel_layer.group_send)( - group_name, {"type": "send_notification", "message": json.dumps(payload)} - ) + if channel_layer is None: + return + + # This WebSocket push streams live TaskResult updates to browser clients + # subscribed to a channel group. It is best-effort: a failed notification + # must never abort the surrounding TaskResult.save(). Failures are logged at + # WARNING so a genuinely broken channel layer in production is still visible. + try: + async_to_sync(channel_layer.group_send)( + group_name, {"type": "send_notification", "message": json.dumps(payload)} + ) + except Exception: # noqa: BLE001 - notification failures must never propagate + logger.warning( + "Failed to send TaskResult notification for %s to group %r", + instance, + group_name, + exc_info=True, + ) From 77119f69397de084ef07e36c1e7dfc96cddc6ba0 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 3 Jul 2026 11:38:19 -0400 Subject: [PATCH 30/36] frame preview indicator in the UI for layers and legend --- web/src/components/sidebars/LayersPanel.vue | 36 ++++++++++++++++++++- web/src/components/sidebars/LegendPanel.vue | 29 ++++++++++++++++- web/src/store/framePreview.ts | 36 +++++++++++++++++++++ 3 files changed, 99 insertions(+), 2 deletions(-) diff --git a/web/src/components/sidebars/LayersPanel.vue b/web/src/components/sidebars/LayersPanel.vue index 981b2191f..7f4aea58a 100644 --- a/web/src/components/sidebars/LayersPanel.vue +++ b/web/src/components/sidebars/LayersPanel.vue @@ -8,10 +8,11 @@ import CompareLayerStyle from "./CompareLayerStyle.vue"; import DetailView from "../DetailView.vue"; import SliderNumericInput from "../SliderNumericInput.vue"; -import { useLayerStore, useMapStore } from "@/store"; +import { useLayerStore, useMapStore, useFramePreviewStore } from "@/store"; import { useMapCompareStore } from "@/store/compare"; const layerStore = useLayerStore(); const mapStore = useMapStore(); +const framePreviewStore = useFramePreviewStore(); const compareStore = useMapCompareStore(); const isComparing = computed(() => compareStore.isComparing); const orientation = computed(() => compareStore.orientation); @@ -189,6 +190,21 @@ function setLayerActive(layer: Layer, active: boolean) { {{ element.name }} + + diff --git a/web/src/store/framePreview.ts b/web/src/store/framePreview.ts index 48fddfeae..09c74456b 100644 --- a/web/src/store/framePreview.ts +++ b/web/src/store/framePreview.ts @@ -1,4 +1,5 @@ import { defineStore } from "pinia"; +import { ref } from "vue"; import type { FramePreview, Layer, LayerFrame, LayerStyle } from "@/types"; import type { Map as MaplibreMap } from "maplibre-gl"; import { @@ -62,6 +63,31 @@ export const useFramePreviewStore = defineStore("framePreview", () => { const activePreviewByLayerKey = new Map(); const transitionGenerationByLayerKey = new Map(); + // Reactive set of layer keys whose preview overlay is currently visible on + // the map (i.e. the user is looking at a preview image, not the real tiles). + // Used to drive UI indicators in the layers and legend panels. + const displayingPreviewLayerKeys = ref>(new Set()); + + function markPreviewDisplayed(layerKeyValue: string) { + if (!displayingPreviewLayerKeys.value.has(layerKeyValue)) { + const next = new Set(displayingPreviewLayerKeys.value); + next.add(layerKeyValue); + displayingPreviewLayerKeys.value = next; + } + } + + function clearPreviewDisplayed(layerKeyValue: string) { + if (displayingPreviewLayerKeys.value.has(layerKeyValue)) { + const next = new Set(displayingPreviewLayerKeys.value); + next.delete(layerKeyValue); + displayingPreviewLayerKeys.value = next; + } + } + + function isDisplayingPreview(layer: Layer) { + return displayingPreviewLayerKeys.value.has(layerKey(layer)); + } + function bumpGeneration(layerKeyValue: string) { const next = (transitionGenerationByLayerKey.get(layerKeyValue) ?? 0) + 1; transitionGenerationByLayerKey.set(layerKeyValue, next); @@ -164,6 +190,7 @@ export const useFramePreviewStore = defineStore("framePreview", () => { } removePreviewLayer(map, layerKeyValue, frameIndex); activePreviewByLayerKey.delete(layerKeyValue); + clearPreviewDisplayed(layerKeyValue); return; } @@ -187,6 +214,7 @@ export const useFramePreviewStore = defineStore("framePreview", () => { removePreviewLayer(map, layerKeyValue, frameIndex); activePreviewByLayerKey.delete(layerKeyValue); + clearPreviewDisplayed(layerKeyValue); } async function showPreviewThenTiles(layer: Layer) { @@ -228,6 +256,7 @@ export const useFramePreviewStore = defineStore("framePreview", () => { hidePreviousPreview(map, layerKeyValue); if (!preview) { + clearPreviewDisplayed(layerKeyValue); if (map.getLayer(tileLayerId)) { map.setPaintProperty(tileLayerId, "raster-opacity", targetOpacity); } @@ -251,6 +280,7 @@ export const useFramePreviewStore = defineStore("framePreview", () => { targetOpacity, ); if (!previewMapLayerId) { + clearPreviewDisplayed(layerKeyValue); if (map.getLayer(tileLayerId)) { map.setPaintProperty(tileLayerId, "raster-opacity", targetOpacity); } @@ -258,6 +288,7 @@ export const useFramePreviewStore = defineStore("framePreview", () => { } activePreviewByLayerKey.set(layerKeyValue, layer.current_frame_index); + markPreviewDisplayed(layerKeyValue); if (map.getLayer(tileLayerId)) { map.setPaintProperty(tileLayerId, "raster-opacity", 0); @@ -293,6 +324,7 @@ export const useFramePreviewStore = defineStore("framePreview", () => { bumpGeneration(layerKeyValue); removeAllPreviewLayersForLayerKey(map, layerKeyValue); activePreviewByLayerKey.delete(layerKeyValue); + clearPreviewDisplayed(layerKeyValue); const frames = layerStore.layerFrames(layer); const currentFrame = frames.find( @@ -315,15 +347,19 @@ export const useFramePreviewStore = defineStore("framePreview", () => { const layerKeyValue = layerKey(layer); transitionGenerationByLayerKey.delete(layerKeyValue); activePreviewByLayerKey.delete(layerKeyValue); + clearPreviewDisplayed(layerKeyValue); removeAllPreviewLayersForLayerKey(mapStore.getMap(), layerKeyValue); } function clearAll() { transitionGenerationByLayerKey.clear(); activePreviewByLayerKey.clear(); + displayingPreviewLayerKeys.value = new Set(); } return { + displayingPreviewLayerKeys, + isDisplayingPreview, prefetchLayerPreviews, showPreviewThenTiles, dismissPreviewForLayer, From 2ca3b61c79968c42d42364fbd75714daf0bc4cd0 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 3 Jul 2026 11:58:23 -0400 Subject: [PATCH 31/36] frame preview dirty and reload state --- web/src/api/rest.ts | 4 + web/src/components/sidebars/LayerStyle.vue | 19 ++++- web/src/store/analysis.ts | 5 ++ web/src/store/framePreview.ts | 91 +++++++++++++++++++++- web/src/store/layer.ts | 3 +- web/src/types.ts | 7 ++ 6 files changed, 125 insertions(+), 4 deletions(-) diff --git a/web/src/api/rest.ts b/web/src/api/rest.ts index 1c8019f31..e89ba8075 100644 --- a/web/src/api/rest.ts +++ b/web/src/api/rest.ts @@ -270,6 +270,10 @@ export async function getLayerStyles(layerId: number): Promise { return (await apiClient.get(`layer-styles/?layer=${layerId}`)).data.results; } +export async function getLayerStyle(styleId: number): Promise { + return (await apiClient.get(`layer-styles/${styleId}/`)).data; +} + export async function createLayerStyle(data: LayerStyle): Promise { return (await apiClient.post("layer-styles/", data)).data; } diff --git a/web/src/components/sidebars/LayerStyle.vue b/web/src/components/sidebars/LayerStyle.vue index 0e4f3f716..90ee4d362 100644 --- a/web/src/components/sidebars/LayerStyle.vue +++ b/web/src/components/sidebars/LayerStyle.vue @@ -27,12 +27,14 @@ import { usePanelStore, useLayerStore, useAppStore, + useFramePreviewStore, } from "@/store"; const styleStore = useStyleStore(); const projectStore = useProjectStore(); const panelStore = usePanelStore(); const layerStore = useLayerStore(); const appStore = useAppStore(); +const framePreviewStore = useFramePreviewStore(); const emit = defineEmits(["setLayerActive"]); const props = defineProps<{ @@ -90,6 +92,16 @@ const setCurrentLayerStyle = (style: LayerStyle) => { styleStore.selectedLayerStyles[styleKey.value] = style; }; +// After a style save, the backend invalidates multiframe raster previews and +// regenerates them asynchronously. Apply the saved style (which the API returns +// with preview_status "notready" and no previews), drop any stale previews, and +// remove the on-map preview overlay so tiles are shown until regeneration +// completes and the WebSocket handler reattaches fresh previews. +const markStyleSavedAndInvalidatePreviews = (style: LayerStyle) => { + setCurrentLayerStyle({ ...style, multiframe_previews: undefined }); + framePreviewStore.dismissPreviewForLayer(props.layer); +}; + const appliedStyleName = computed(() => { if (currentLayerStyle.value.id) return currentLayerStyle.value.name; else return undefined; @@ -187,6 +199,9 @@ function selectStyle(style: LayerStyle) { setCurrentLayerStyle(style); currentStyleSpec.value = style.style_spec; currentGroups.value = { color: undefined, size: undefined }; + // Remove any preview overlay tied to the previously selected style so the map + // only shows previews that belong to the style now in effect. + framePreviewStore.dismissPreviewForLayer(props.layer); } function fetchRasterBands() { @@ -506,7 +521,7 @@ function save() { style_spec: currentStyleSpec.value, }).then((style) => { if (style) { - setCurrentLayerStyle(style); + markStyleSavedAndInvalidatePreviews(style); newName.value = undefined; newNameMode.value = undefined; // update other styles in case default overriden @@ -529,7 +544,7 @@ function saveAsNew() { style_spec: currentStyleSpec.value, }).then((style: LayerStyle) => { if (style) { - setCurrentLayerStyle(style); + markStyleSavedAndInvalidatePreviews(style); newName.value = undefined; newNameMode.value = undefined; // update other styles in case default overriden diff --git a/web/src/store/analysis.ts b/web/src/store/analysis.ts index 0807ab834..d3b8f0548 100644 --- a/web/src/store/analysis.ts +++ b/web/src/store/analysis.ts @@ -7,6 +7,7 @@ import type { Chart, AnalysisType, TaskResult } from "@/types"; import { defineStore } from "pinia"; import { ref, watch } from "vue"; import { useProjectStore } from "./project"; +import { useFramePreviewStore } from "./framePreview"; export const useAnalysisStore = defineStore("analysis", () => { const projectStore = useProjectStore(); @@ -50,6 +51,10 @@ export const useAnalysisStore = defineStore("analysis", () => { ws.value = new WebSocket(url); ws.value.onmessage = (event: any) => { const data = JSON.parse(JSON.parse(event.data)); + if (data.task_type === "frame_preview" && data.completed) { + // Regenerated previews are ready; reload and reattach them to layers. + useFramePreviewStore().onPreviewTaskComplete(data); + } if (currentResult.value && data.id === currentResult.value.id) { // only overwrite attributes expecting updates // overwriting the whole currentResult object will cause diff --git a/web/src/store/framePreview.ts b/web/src/store/framePreview.ts index 09c74456b..0619fc8ab 100644 --- a/web/src/store/framePreview.ts +++ b/web/src/store/framePreview.ts @@ -1,7 +1,14 @@ import { defineStore } from "pinia"; import { ref } from "vue"; -import type { FramePreview, Layer, LayerFrame, LayerStyle } from "@/types"; +import type { + FramePreview, + Layer, + LayerFrame, + LayerStyle, + TaskResult, +} from "@/types"; import type { Map as MaplibreMap } from "maplibre-gl"; +import { getLayerStyle } from "@/api/rest"; import { fadeRasterOpacities, hidePreviewLayer, @@ -26,7 +33,32 @@ function orderedRasterFrames(frames: LayerFrame[]) { .toSorted((a, b) => a.index - b.index); } +// Previews are only safe to display once the backend reports the style as +// "ready" (every frame has a complete image). After a style save the API +// returns "notready" and omits multiframe_previews until regeneration finishes. +function previewsAreReady( + layer: Layer, + style: LayerStyle | undefined, +): boolean { + const status = + style?.preview_status ?? + (style?.id !== undefined && style.id === layer.default_style?.id + ? layer.preview_status + : undefined); + if (status !== undefined) { + return status === "ready"; + } + // Backward compatibility for payloads that omit preview_status entirely. + return true; +} + function previewsForLayer(layer: Layer, style: LayerStyle | undefined) { + // Guard against falling back to the layer's default-style previews when the + // active style's previews are stale/regenerating; that would render another + // style's images against the current style. + if (!previewsAreReady(layer, style)) { + return undefined; + } return style?.multiframe_previews ?? layer.multiframe_previews; } @@ -357,12 +389,69 @@ export const useFramePreviewStore = defineStore("framePreview", () => { displayingPreviewLayerKeys.value = new Set(); } + // Called when a "frame_preview" TaskResult completes over the analytics + // WebSocket. Reloads the freshly generated previews and reattaches them to + // every selected layer copy that still has the regenerated style applied. + async function onPreviewTaskComplete(task: TaskResult) { + const layerStyleId = task.inputs?.layer_style_id; + const layerId = task.inputs?.layer_id; + if (layerStyleId === undefined || layerId === undefined) { + return; + } + + const layerStore = useLayerStore(); + const styleStore = useStyleStore(); + + let updatedStyle: LayerStyle; + try { + updatedStyle = await getLayerStyle(layerStyleId); + } catch { + // If the style was deleted or the fetch fails, there is nothing to attach. + return; + } + + // Keep availableLayers current so re-adding this layer picks up the new + // default-style previews. Fire-and-forget; selected copies are updated below. + void layerStore.fetchAvailableLayer(layerId).catch(() => undefined); + + layerStore.selectedLayers.forEach((layer) => { + if (layer.id !== layerId) { + return; + } + const key = layerKey(layer); + const selectedStyle = styleStore.selectedLayerStyles[key]; + // Only reattach when this copy still has the regenerated style selected; + // the user may have swapped to a different style while generation ran. + if (!selectedStyle || selectedStyle.id !== layerStyleId) { + return; + } + + styleStore.selectedLayerStyles[key] = { + ...selectedStyle, + preview_status: updatedStyle.preview_status, + multiframe_previews: updatedStyle.multiframe_previews, + }; + + // Mirror onto the layer object so the default-style fallback stays valid. + if (updatedStyle.is_default) { + layer.multiframe_previews = updatedStyle.multiframe_previews; + layer.preview_status = updatedStyle.preview_status; + } + + prefetchLayerPreviews(layer, styleStore.selectedLayerStyles[key]); + if (!styleStore.isLayerStyleEditing(layer)) { + void showPreviewThenTiles(layer); + } + }); + } + return { displayingPreviewLayerKeys, isDisplayingPreview, prefetchLayerPreviews, showPreviewThenTiles, dismissPreviewForLayer, + onPreviewTaskComplete, cleanupLayer, clearAll, }; diff --git a/web/src/store/layer.ts b/web/src/store/layer.ts index 11fab0087..e61d764ce 100644 --- a/web/src/store/layer.ts +++ b/web/src/store/layer.ts @@ -226,7 +226,8 @@ export const useLayerStore = defineStore("layer", () => { ) { styleStore.selectedLayerStyles[styleId] = { ...layer.default_style, - ...(layer.multiframe_previews + preview_status: layer.preview_status, + ...(layer.preview_status === "ready" && layer.multiframe_previews ? { multiframe_previews: layer.multiframe_previews } : {}), }; diff --git a/web/src/types.ts b/web/src/types.ts index dc3a68045..fef05327a 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -52,6 +52,11 @@ export interface FramePreview { bounds: FramePreviewBounds; } +// Style-level preview lifecycle, derived by the API from per-frame rows. +// "ready" means every raster frame has a complete preview image; anything +// else (missing, generating, regenerating, failed) is reported as "notready". +export type PreviewStatus = "ready" | "notready"; + export interface Layer { id: number; copy_id: number; @@ -63,6 +68,7 @@ export interface Layer { current_frame_index: number; default_style: LayerStyle | null; multiframe_previews?: (FramePreview | null)[]; + preview_status?: PreviewStatus; } export interface LayerFrame { @@ -154,6 +160,7 @@ export interface LayerStyle { is_default: boolean; style_spec?: StyleSpec; multiframe_previews?: (FramePreview | null)[]; + preview_status?: PreviewStatus; } export interface VectorData { From 6cb2ecace4ac320ed36b87b91d273dade42bc74b Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 3 Jul 2026 12:56:59 -0400 Subject: [PATCH 32/36] ensure a default style when deleting others --- uvdat/core/models/layer.py | 16 ++++++ uvdat/core/rest/dataset.py | 4 +- uvdat/core/rest/layer.py | 12 +++- uvdat/core/tests/test_frame_preview.py | 76 ++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 2 deletions(-) diff --git a/uvdat/core/models/layer.py b/uvdat/core/models/layer.py index 147310ad2..4c14f1066 100644 --- a/uvdat/core/models/layer.py +++ b/uvdat/core/models/layer.py @@ -44,6 +44,22 @@ def is_multiframe_raster(self) -> bool: return len(prefetched) > 1 return self.frames.filter(raster__isnull=False).count() > 1 + def ensure_default_style(self): + """Guarantee the layer has a default style whenever any style exists. + + ``default_style`` uses ``on_delete=SET_NULL``, so removing the style a + layer pointed at (outside the viewset's reassign path) can leave the + layer with styles but no default, which hides its frame previews. Prefer + the conventional "Default" style, otherwise fall back to the oldest one. + """ + if self.default_style_id is not None: + return self.default_style + style = self.styles.filter(name="Default").first() or self.styles.order_by("id").first() + if style is not None: + self.default_style = style + self.save(update_fields=["default_style"]) + return style + def default_multiframe_previews(self) -> list[FramePreviewData] | None: if self.default_style_id is None: return None diff --git a/uvdat/core/rest/dataset.py b/uvdat/core/rest/dataset.py index f0f2ea2d5..a86b57135 100644 --- a/uvdat/core/rest/dataset.py +++ b/uvdat/core/rest/dataset.py @@ -5,7 +5,7 @@ from rest_framework.viewsets import ModelViewSet from uvdat.core.access_control import DatasetGuardianPermission -from uvdat.core.models import Dataset, DatasetTag, Network +from uvdat.core.models import Dataset, DatasetTag, Layer, Network from uvdat.core.rest.querysets import layer_queryset_with_previews from uvdat.core.rest.serializers import ( DatasetSerializer, @@ -44,6 +44,8 @@ def tags(self, request, **kwargs): @action(detail=True, methods=["get"]) def layers(self, request, **kwargs): dataset: Dataset = self.get_object() + for layer in Layer.objects.filter(dataset=dataset, default_style__isnull=True): + layer.ensure_default_style() layers = list(layer_queryset_with_previews().filter(dataset=dataset)) serializer = LayerSerializer(layers, many=True) return Response(serializer.data, status=200) diff --git a/uvdat/core/rest/layer.py b/uvdat/core/rest/layer.py index 33ff4f076..57f850e82 100644 --- a/uvdat/core/rest/layer.py +++ b/uvdat/core/rest/layer.py @@ -21,6 +21,12 @@ class LayerViewSet(ReadOnlyModelViewSet): def get_queryset(self): return layer_queryset_with_previews() + def retrieve(self, request, *args, **kwargs): + if Layer.objects.filter(pk=kwargs.get("pk"), default_style__isnull=True).exists(): + layer = Layer.objects.get(pk=kwargs["pk"]) + layer.ensure_default_style() + return super().retrieve(request, *args, **kwargs) + @action(detail=True, methods=["get"]) def frames(self, request, **kwargs): layer: Layer = self.get_object() @@ -57,7 +63,11 @@ def create(self, request, **kwargs): instance = serializer.save() except jsonschema.exceptions.ValidationError as e: return Response(e.message, status=400) - if is_default and instance.layer.default_style != instance: + # Always keep a default style: adopt the new one when the layer has none, + # so a layer with styles is never left without a default (and no previews). + if ( + is_default or instance.layer.default_style_id is None + ) and instance.layer.default_style != instance: instance.layer.default_style = instance instance.layer.save() return Response(serializer.data, status=200) diff --git a/uvdat/core/tests/test_frame_preview.py b/uvdat/core/tests/test_frame_preview.py index 71c23aed8..80b8085a9 100644 --- a/uvdat/core/tests/test_frame_preview.py +++ b/uvdat/core/tests/test_frame_preview.py @@ -404,3 +404,79 @@ def test_layer_api_includes_multiframe_previews( }, ] assert "multiframe_previews" not in data.get("default_style", {}) + + +@pytest.mark.django_db +def test_dataset_layers_self_heal_missing_default_style( + authenticated_api_client, + layer_style_factory, + layer_frame_factory, + project, + user, +): + """A layer with styles but no default_style should still surface previews. + + ``default_style`` can be nulled (``on_delete=SET_NULL``) when the style it + pointed at is removed. The layers endpoint should adopt an existing style as + the default so its frame previews are not lost. + """ + layer_style = layer_style_factory(name="Default") + layer = layer_style.layer + layer.default_style = None + layer.save(update_fields=["default_style"]) + project.set_collaborators([user]) + project.datasets.set([layer.dataset]) + frame_0 = layer_frame_factory(layer=layer, index=0) + frame_1 = layer_frame_factory(layer=layer, index=1) + + for frame in (frame_0, frame_1): + preview = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame, + status=PreviewStatus.COMPLETE, + width=100, + height=100, + bounds={}, + ) + preview.image.save(f"frame-{frame.index}.png", ContentFile(b"png"), save=True) + + resp = authenticated_api_client.get(f"/api/v1/datasets/{layer.dataset_id}/layers/") + assert resp.status_code == 200 + layers = resp.json() + assert len(layers) == 1 + layer_data = layers[0] + assert layer_data["default_style"]["id"] == layer_style.id + assert layer_data["preview_status"] == "ready" + assert len(layer_data["multiframe_previews"]) == 2 + + layer.refresh_from_db() + assert layer.default_style_id == layer_style.id + + +@pytest.mark.django_db +def test_create_style_sets_default_when_layer_has_none( + authenticated_api_client, + layer_factory, + project, + user, +): + """Creating the first style for a layer without a default adopts it.""" + layer = layer_factory() + project.set_collaborators([user]) + project.datasets.set([layer.dataset]) + assert layer.default_style_id is None + + resp = authenticated_api_client.post( + "/api/v1/layer-styles/", + { + "name": "terrain", + "layer": layer.id, + "project": project.id, + "style_spec": DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC, + }, + format="json", + ) + assert resp.status_code == 200 + + layer.refresh_from_db() + assert layer.default_style_id == resp.json()["id"] From 82c81365dac532a044b1ef5b2ae7db180e03706f Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 3 Jul 2026 13:17:17 -0400 Subject: [PATCH 33/36] linting --- uvdat/core/models/task_result.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/uvdat/core/models/task_result.py b/uvdat/core/models/task_result.py index 3544b9e01..aa412f541 100644 --- a/uvdat/core/models/task_result.py +++ b/uvdat/core/models/task_result.py @@ -25,9 +25,7 @@ # WebSocket notifications are meaningless: tasks run synchronously with no # client session listening. When set, result_post_save skips the push entirely # rather than attempting it and logging a warning. -_suppress_notifications: ContextVar[bool] = ContextVar( - "suppress_task_notifications", default=False -) +_suppress_notifications: ContextVar[bool] = ContextVar("suppress_task_notifications", default=False) @contextlib.contextmanager From 3ef0ac6f1baaccfc9355f70ce6f78667246f50d9 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Thu, 9 Jul 2026 13:53:27 -0400 Subject: [PATCH 34/36] shortcircuit to prevent preview task if the fingerprint is the same --- .../frame_previews/preview_regeneration.py | 22 ++++ uvdat/core/tests/test_frame_preview.py | 103 +++++++++++++++++- 2 files changed, 124 insertions(+), 1 deletion(-) diff --git a/uvdat/core/frame_previews/preview_regeneration.py b/uvdat/core/frame_previews/preview_regeneration.py index c71a11604..00a02dde0 100644 --- a/uvdat/core/frame_previews/preview_regeneration.py +++ b/uvdat/core/frame_previews/preview_regeneration.py @@ -22,6 +22,25 @@ def supersede_pending_preview_tasks(layer_style_id: int) -> None: ) +def previews_current_for_fingerprint(layer_style: LayerStyle, fingerprint: str) -> bool: + """Return whether every frame already has a complete preview for this fingerprint.""" + frames = layer_style.layer.multiframe_raster_frames() + if not frames: + return False + + previews_by_frame = { + preview.layer_frame_id: preview for preview in layer_style.frame_previews.all() + } + + return all( + (preview := previews_by_frame.get(frame.id)) + and preview.status == PreviewStatus.COMPLETE + and preview.image + and preview.style_fingerprint == fingerprint + for frame in frames + ) + + def mark_previews_regenerating(layer_style: LayerStyle, fingerprint: str) -> list[int]: """Upsert one preview row per multiframe frame and clear stale images.""" frame_ids = [] @@ -58,6 +77,9 @@ def invalidate_and_enqueue_previews( layer_style.refresh_from_db() fingerprint = style_fingerprint(layer_style) + if previews_current_for_fingerprint(layer_style, fingerprint): + return None + mark_previews_regenerating(layer_style, fingerprint) supersede_pending_preview_tasks(layer_style.id) diff --git a/uvdat/core/tests/test_frame_preview.py b/uvdat/core/tests/test_frame_preview.py index 80b8085a9..f46718590 100644 --- a/uvdat/core/tests/test_frame_preview.py +++ b/uvdat/core/tests/test_frame_preview.py @@ -4,7 +4,10 @@ import pytest from uvdat.core.frame_previews.fingerprint import style_fingerprint -from uvdat.core.frame_previews.preview_regeneration import invalidate_and_enqueue_previews +from uvdat.core.frame_previews.preview_regeneration import ( + get_layer_style_preview_status, + invalidate_and_enqueue_previews, +) from uvdat.core.frame_previews.raster_style import ( apply_source_filters_to_style_query, build_thumbnail_style_query, @@ -98,6 +101,104 @@ def test_invalidate_and_enqueue_previews_uses_db_fingerprint( assert fingerprint == style_fingerprint(LayerStyle.objects.get(pk=layer_style.pk)) +@pytest.mark.django_db +def test_invalidate_and_enqueue_previews_skips_when_previews_current( + layer_style_factory, + layer_frame_factory, + mocker, +): + layer_style = layer_style_factory() + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) + layer_style.save_style_configs(DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC) + fingerprint = style_fingerprint(layer_style) + + preview_0 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_0, + status=PreviewStatus.COMPLETE, + style_fingerprint=fingerprint, + width=100, + height=80, + bounds={"srs": "EPSG:4326", "xmin": -1, "xmax": 1, "ymin": -2, "ymax": 2}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_1, + status=PreviewStatus.COMPLETE, + style_fingerprint=fingerprint, + width=120, + height=90, + bounds={"srs": "EPSG:4326", "xmin": -2, "xmax": 2, "ymin": -3, "ymax": 3}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + + delay = mocker.patch("uvdat.core.tasks.frame_preview.generate_layer_style_previews.delay") + + result = invalidate_and_enqueue_previews(layer_style) + + delay.assert_not_called() + assert result is None + assert get_layer_style_preview_status(layer_style) == "ready" + + preview_0.refresh_from_db() + preview_1.refresh_from_db() + assert preview_0.status == PreviewStatus.COMPLETE + assert preview_1.status == PreviewStatus.COMPLETE + assert preview_0.image.name + assert preview_1.image.name + assert preview_0.width == 100 + assert preview_1.width == 120 + + +@pytest.mark.django_db +def test_invalidate_and_enqueue_previews_runs_when_fingerprint_changed( + layer_style_factory, + layer_frame_factory, + mocker, +): + layer_style = layer_style_factory() + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) + layer_style.save_style_configs(DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC) + + preview_0 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_0, + status=PreviewStatus.COMPLETE, + style_fingerprint="stale-fingerprint", + width=100, + height=80, + bounds={}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_1, + status=PreviewStatus.COMPLETE, + style_fingerprint="stale-fingerprint", + width=120, + height=90, + bounds={}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + + delay = mocker.patch("uvdat.core.tasks.frame_preview.generate_layer_style_previews.delay") + + result = invalidate_and_enqueue_previews(layer_style) + + delay.assert_called_once() + assert result is not None + + preview_0.refresh_from_db() + preview_1.refresh_from_db() + assert preview_0.status == PreviewStatus.REGENERATING + assert preview_1.status == PreviewStatus.REGENERATING + assert not preview_0.image + assert not preview_1.image + + @pytest.mark.django_db def test_invalidate_and_enqueue_previews_runs_synchronously( layer_style_factory, From b560e97c3219200fd4cefd375b0822e23af481dc Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Thu, 9 Jul 2026 14:03:42 -0400 Subject: [PATCH 35/36] invalidate frame previews when editing/creating new styles --- .../frame_previews/preview_regeneration.py | 11 +++ uvdat/core/tests/test_frame_preview.py | 67 +++++++++++++++++++ web/src/components/sidebars/LayerStyle.vue | 23 ++++++- web/src/store/framePreview.ts | 33 ++++++--- 4 files changed, 122 insertions(+), 12 deletions(-) diff --git a/uvdat/core/frame_previews/preview_regeneration.py b/uvdat/core/frame_previews/preview_regeneration.py index 00a02dde0..0320aedd0 100644 --- a/uvdat/core/frame_previews/preview_regeneration.py +++ b/uvdat/core/frame_previews/preview_regeneration.py @@ -41,6 +41,16 @@ def previews_current_for_fingerprint(layer_style: LayerStyle, fingerprint: str) ) +def clear_style_preview_instance_cache(layer_style: LayerStyle) -> None: + """Drop queryset annotations and prefetches that go stale after invalidation.""" + layer_style.refresh_from_db() + layer_style.__dict__.pop("preview_status", None) + layer_style.__dict__.pop("_raster_frame_count", None) + layer_style.__dict__.pop("_complete_with_image_count", None) + if cache := getattr(layer_style, "_prefetched_objects_cache", None): + cache.pop("frame_previews", None) + + def mark_previews_regenerating(layer_style: LayerStyle, fingerprint: str) -> list[int]: """Upsert one preview row per multiframe frame and clear stale images.""" frame_ids = [] @@ -81,6 +91,7 @@ def invalidate_and_enqueue_previews( return None mark_previews_regenerating(layer_style, fingerprint) + clear_style_preview_instance_cache(layer_style) supersede_pending_preview_tasks(layer_style.id) result = TaskResult.objects.create( diff --git a/uvdat/core/tests/test_frame_preview.py b/uvdat/core/tests/test_frame_preview.py index f46718590..b0649f5aa 100644 --- a/uvdat/core/tests/test_frame_preview.py +++ b/uvdat/core/tests/test_frame_preview.py @@ -419,6 +419,73 @@ def test_layer_style_api_includes_multiframe_previews( ] +@pytest.mark.django_db +def test_layer_style_patch_reports_notready_after_preview_invalidation( + authenticated_api_client, + layer_style_factory, + layer_frame_factory, + project, + user, + mocker, +): + layer_style = layer_style_factory() + project.set_collaborators([user]) + project.datasets.set([layer_style.layer.dataset]) + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) + layer_style.save_style_configs(DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC) + fingerprint = style_fingerprint(layer_style) + + preview_0 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_0, + status=PreviewStatus.COMPLETE, + style_fingerprint=fingerprint, + width=100, + height=100, + bounds={}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_1, + status=PreviewStatus.COMPLETE, + style_fingerprint=fingerprint, + width=100, + height=100, + bounds={}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + + mocker.patch("uvdat.core.tasks.frame_preview.generate_layer_style_previews.delay") + + updated_spec = { + **DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC, + "colors": [ + { + "name": "all", + "visible": True, + "use_feature_props": True, + "single_color": "#ff0000", + } + ], + } + resp = authenticated_api_client.patch( + f"/api/v1/layer-styles/{layer_style.id}/", + { + "name": layer_style.name, + "layer": layer_style.layer_id, + "project": layer_style.project_id, + "style_spec": updated_spec, + }, + format="json", + ) + assert resp.status_code == 200 + data = resp.json() + assert data["preview_status"] == "notready" + assert "multiframe_previews" not in data + + @pytest.mark.django_db def test_api_omits_previews_while_not_ready( authenticated_api_client, diff --git a/web/src/components/sidebars/LayerStyle.vue b/web/src/components/sidebars/LayerStyle.vue index 90ee4d362..d2805907d 100644 --- a/web/src/components/sidebars/LayerStyle.vue +++ b/web/src/components/sidebars/LayerStyle.vue @@ -98,7 +98,28 @@ const setCurrentLayerStyle = (style: LayerStyle) => { // remove the on-map preview overlay so tiles are shown until regeneration // completes and the WebSocket handler reattaches fresh previews. const markStyleSavedAndInvalidatePreviews = (style: LayerStyle) => { - setCurrentLayerStyle({ ...style, multiframe_previews: undefined }); + const previewStatus = + style.preview_status === "ready" && !style.multiframe_previews + ? "notready" + : (style.preview_status ?? "notready"); + const invalidatedStyle: LayerStyle = { + ...style, + preview_status: previewStatus, + multiframe_previews: undefined, + }; + setCurrentLayerStyle(invalidatedStyle); + if (invalidatedStyle.is_default) { + layerStore.selectedLayers = layerStore.selectedLayers.map((layer) => { + if (layer.id !== props.layer.id || layer.copy_id !== props.layer.copy_id) { + return layer; + } + return { + ...layer, + preview_status: invalidatedStyle.preview_status, + multiframe_previews: undefined, + }; + }); + } framePreviewStore.dismissPreviewForLayer(props.layer); }; diff --git a/web/src/store/framePreview.ts b/web/src/store/framePreview.ts index 0619fc8ab..5482eb8af 100644 --- a/web/src/store/framePreview.ts +++ b/web/src/store/framePreview.ts @@ -40,26 +40,37 @@ function previewsAreReady( layer: Layer, style: LayerStyle | undefined, ): boolean { - const status = - style?.preview_status ?? - (style?.id !== undefined && style.id === layer.default_style?.id - ? layer.preview_status - : undefined); - if (status !== undefined) { - return status === "ready"; + if (style?.preview_status !== undefined) { + return style.preview_status === "ready"; + } + if ( + style?.id !== undefined && + style.id === layer.default_style?.id && + layer.preview_status !== undefined + ) { + return layer.preview_status === "ready"; } // Backward compatibility for payloads that omit preview_status entirely. return true; } function previewsForLayer(layer: Layer, style: LayerStyle | undefined) { - // Guard against falling back to the layer's default-style previews when the - // active style's previews are stale/regenerating; that would render another - // style's images against the current style. if (!previewsAreReady(layer, style)) { return undefined; } - return style?.multiframe_previews ?? layer.multiframe_previews; + if (style?.multiframe_previews?.length) { + return style.multiframe_previews; + } + // Layer payloads attach previews for the default style only. Use them when the + // selected style is that default and the style object has no preview list. + if ( + style?.id !== undefined && + style.id === layer.default_style?.id && + layer.multiframe_previews?.length + ) { + return layer.multiframe_previews; + } + return undefined; } function previewAtFrameIndex( From 3cbb2a019741f2ea645377da743be704ced53bc7 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Thu, 9 Jul 2026 14:25:51 -0400 Subject: [PATCH 36/36] client linting --- web/src/components/sidebars/LayerStyle.vue | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web/src/components/sidebars/LayerStyle.vue b/web/src/components/sidebars/LayerStyle.vue index d2805907d..cea032e11 100644 --- a/web/src/components/sidebars/LayerStyle.vue +++ b/web/src/components/sidebars/LayerStyle.vue @@ -110,7 +110,10 @@ const markStyleSavedAndInvalidatePreviews = (style: LayerStyle) => { setCurrentLayerStyle(invalidatedStyle); if (invalidatedStyle.is_default) { layerStore.selectedLayers = layerStore.selectedLayers.map((layer) => { - if (layer.id !== props.layer.id || layer.copy_id !== props.layer.copy_id) { + if ( + layer.id !== props.layer.id || + layer.copy_id !== props.layer.copy_id + ) { return layer; } return {