diff --git a/Cargo.toml b/Cargo.toml index b7d66737..d0b32063 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -130,6 +130,7 @@ sqlx = { version = "0.8.6", default-features = false, features = [ "postgres", "runtime-tokio-rustls", ] } +sqlx-tracing = { version = "0.2.1", default-features = false, features = ["postgres"] } redis = { version = "0.27", default-features = false, features = [ "tokio-comp", "connection-manager", @@ -165,6 +166,16 @@ tempfile = "3.0" chrono = { version = "0.4.44", default-features = false, features = ["clock", "serde"] } chrono-english = "0.1.8" rust_decimal = { version = "1.37", default-features = false, features = ["std", "serde", "serde-with-str"] } +polars = { version = "0.46", default-features = false, features = [ + "lazy", + "dtype-struct", + "dtype-decimal", + "strings", + "temporal", + "is_in", + "abs", + "regex", +] } toon = "0.1" # LLM / REPL diff --git a/apis/architect-exchange/domain.yaml b/apis/architect-exchange/domain.yaml index 433bb561..648769e7 100644 --- a/apis/architect-exchange/domain.yaml +++ b/apis/architect-exchange/domain.yaml @@ -277,7 +277,7 @@ entities: id_from: - account_id - symbol - description: Signed derivative exposure for one instrument in an account. + description: Signed derivative exposure for one instrument in an account. Ad-hoc ranking and money sums of fetched positions use program postfix, not extra catalog entities. discovery: names: - position @@ -940,7 +940,7 @@ entities: Fill: id_field: trade_id - description: Private execution against an account, including fees and side. + description: Private execution against an account, including fees and side. Ad-hoc rollups (fees by symbol, notional) use program postfix on fetched rows, not extra catalog entities. discovery: names: - fill diff --git a/crates/plasm-agent-core/Cargo.toml b/crates/plasm-agent-core/Cargo.toml index 1f8fbcf5..4c954fd2 100644 --- a/crates/plasm-agent-core/Cargo.toml +++ b/crates/plasm-agent-core/Cargo.toml @@ -67,6 +67,7 @@ subtle = { workspace = true } # even when we only use `InMemoryStorage` at runtime (see auth_framework::storage::memory). auth-framework = { workspace = true } sqlx = { workspace = true, features = ["migrate", "chrono", "uuid"] } +sqlx-tracing = { workspace = true } opentelemetry = { workspace = true } minijinja = { version = "2.19.0", default-features = false, features = ["builtins", "serde"] } futures = { workspace = true } @@ -74,6 +75,7 @@ redis = { workspace = true } dashmap = { workspace = true } [dev-dependencies] +plasm-otel = { path = "../plasm-otel", features = ["testing"] } dhat = "0.3" tempfile = { workspace = true } insta = { workspace = true, features = ["json"] } diff --git a/crates/plasm-agent-core/src/blocking_compute.rs b/crates/plasm-agent-core/src/blocking_compute.rs index e7c34e67..0e6f5004 100644 --- a/crates/plasm-agent-core/src/blocking_compute.rs +++ b/crates/plasm-agent-core/src/blocking_compute.rs @@ -52,7 +52,7 @@ impl BlockingComputePool { .acquire() .await .map_err(|_| ComputePoolError::Closed)?; - let span = tracing::debug_span!("plasm.blocking_compute", label); + let span = crate::spans::blocking_compute(label); let out = tokio::task::spawn_blocking(move || { let _guard = span.enter(); f() diff --git a/crates/plasm-agent-core/src/flow_policy_repository.rs b/crates/plasm-agent-core/src/flow_policy_repository.rs index 6d5bf25b..3de6e82a 100644 --- a/crates/plasm-agent-core/src/flow_policy_repository.rs +++ b/crates/plasm-agent-core/src/flow_policy_repository.rs @@ -1,9 +1,10 @@ //! sqlx-backed persistence for project-scoped plan flow policies. +use crate::traced_pg::PgPool; use chrono::{DateTime, Utc}; use serde_json::Value; use sqlx::postgres::PgPoolOptions; -use sqlx::{PgPool, Row}; +use sqlx::Row; use thiserror::Error; use crate::plan_flow_policy::{FlowPolicy, FlowPolicySnapshot, PolicyRevision}; @@ -65,6 +66,7 @@ impl FlowPolicyRepository { .connect(database_url) .await?; sqlx::migrate!("./migrations").run(&pool).await?; + let pool = crate::traced_pg::wrap(pool); Ok(Self { pool }) } diff --git a/crates/plasm-agent-core/src/graph_rehydrate/rehydrator.rs b/crates/plasm-agent-core/src/graph_rehydrate/rehydrator.rs index 5b5f9f8b..967b1986 100644 --- a/crates/plasm-agent-core/src/graph_rehydrate/rehydrator.rs +++ b/crates/plasm-agent-core/src/graph_rehydrate/rehydrator.rs @@ -13,7 +13,9 @@ use crate::execute_session::ExecuteSession; use crate::server_state::PlasmHostState; use super::ctx::GraphSurfaceWalkCtx; -use super::walk::{collect_entities, collect_row_json, snapshot_hot_entities, stream_rows}; +#[cfg(test)] +use super::walk::stream_rows; +use super::walk::{collect_entities, collect_row_json, snapshot_hot_entities}; /// Hot-cache snapshot + target count for spill rehydrate after the graph lock is released. pub(crate) struct GraphSpillSyncPlan { @@ -233,6 +235,7 @@ impl<'a> GraphSurfaceRehydrator<'a> { self.rehydrate_rows(hot, entity_type, logical_count).await } + #[cfg(test)] pub(crate) async fn stream_entity_rows( &self, hot_snapshot: Arc<[CachedEntity]>, diff --git a/crates/plasm-agent-core/src/graph_rehydrate/walk.rs b/crates/plasm-agent-core/src/graph_rehydrate/walk.rs index de927625..004f8e29 100644 --- a/crates/plasm-agent-core/src/graph_rehydrate/walk.rs +++ b/crates/plasm-agent-core/src/graph_rehydrate/walk.rs @@ -125,6 +125,7 @@ where }) } +#[cfg(test)] pub(crate) async fn stream_rows( ctx: &GraphSurfaceWalkCtx<'_>, hot_snapshot: Arc<[CachedEntity]>, @@ -180,7 +181,7 @@ pub(crate) async fn collect_entities( out.truncate(logical_count); crate::graph_cache_metrics::record_graph_rehydrate( "full", - out.len(), + stats.rows_yielded, stats.pages_read, started.elapsed(), ); diff --git a/crates/plasm-agent-core/src/http.rs b/crates/plasm-agent-core/src/http.rs index f6f2afc7..22e82d7f 100644 --- a/crates/plasm-agent-core/src/http.rs +++ b/crates/plasm-agent-core/src/http.rs @@ -224,7 +224,7 @@ pub fn discovery_execute_router(state: PlasmHostState) -> Router { let traced = pre_internal .merge(oss_traced_routes()) - .layer(TraceLayer::new_for_http().make_span_with(tower_http_trace_parent_span)); + .route_layer(TraceLayer::new_for_http().make_span_with(tower_http_trace_parent_span)); Router::new() .merge(health_public) @@ -285,7 +285,7 @@ pub async fn serve_discovery_execute_on_listener_opts( ) -> Result<(), Box> { let addr = listener.local_addr()?; let port = addr.port(); - tracing::info!("plasm HTTP listening on http://{addr}"); + tracing::info!(%addr, "plasm HTTP listening"); if opts.emit_stderr_route_help { eprint_http_command_help(port); } @@ -317,7 +317,7 @@ pub async fn serve_discovery_execute_and_mcp_unified( ) -> Result<(), Box> { let addr = listener.local_addr()?; let port = addr.port(); - tracing::info!("plasm HTTP+MCP unified listening on http://{addr}"); + tracing::info!(%addr, "plasm HTTP+MCP unified listening"); if opts.emit_stderr_route_help { eprint_http_command_help(port); } diff --git a/crates/plasm-agent-core/src/http_discovery.rs b/crates/plasm-agent-core/src/http_discovery.rs index 7ad7de00..b80e98a4 100644 --- a/crates/plasm-agent-core/src/http_discovery.rs +++ b/crates/plasm-agent-core/src/http_discovery.rs @@ -25,6 +25,7 @@ use crate::server_state::{PlasmHostState, ToolModelHostError}; use crate::tool_model::ToolModelQuery; use crate::tool_model_service::ToolModelServiceError; use crate::typed_discovery_host::run_typed_catalog_discovery; +use tracing::Instrument; #[derive(Debug, Deserialize)] pub struct IncludeCgsQuery { @@ -229,6 +230,15 @@ fn log_discovery_response(out: &DiscoveryResult) { async fn post_discover_typed( Extension(st): Extension, Json(query): Json, +) -> Response { + post_discover_typed_inner(st, query) + .instrument(crate::spans::discover_query()) + .await +} + +pub(crate) async fn post_discover_typed_inner( + st: PlasmHostState, + query: DiscoveryQuery, ) -> Response { tracing::debug!( utterance_len = query.utterance.len(), @@ -303,6 +313,12 @@ async fn post_discover( Extension(st): Extension, Json(query): Json, ) -> Response { + post_discover_inner(st, query) + .instrument(crate::spans::discover_query()) + .await +} + +pub(crate) async fn post_discover_inner(st: PlasmHostState, query: CapabilityQuery) -> Response { tracing::debug!( tokens = query.tokens.len(), phrases = query.phrases.len(), diff --git a/crates/plasm-agent-core/src/http_execute/routes/handlers/mod.rs b/crates/plasm-agent-core/src/http_execute/routes/handlers/mod.rs index 02717300..0f088de7 100644 --- a/crates/plasm-agent-core/src/http_execute/routes/handlers/mod.rs +++ b/crates/plasm-agent-core/src/http_execute/routes/handlers/mod.rs @@ -12,5 +12,7 @@ pub(crate) use artifacts::{ }; pub(crate) use create::post_create_execute_session; pub(crate) use run_post::post_run_execute_session; +#[cfg(test)] +pub(crate) use run_post::post_run_execute_session_inner; pub(crate) use session_get::handle_execute_session_get; pub(crate) use stream::get_operation_progress_stream; diff --git a/crates/plasm-agent-core/src/http_execute/routes/handlers/run_post.rs b/crates/plasm-agent-core/src/http_execute/routes/handlers/run_post.rs index 2fed2e4e..2e6d4497 100644 --- a/crates/plasm-agent-core/src/http_execute/routes/handlers/run_post.rs +++ b/crates/plasm-agent-core/src/http_execute/routes/handlers/run_post.rs @@ -1,5 +1,7 @@ //! POST run execute session. +use tracing::Instrument; + use super::super::super::*; use super::plan_run_response::respond_plan_run_live_result; @@ -14,6 +16,20 @@ pub(crate) async fn post_run_execute_session( Query(run_q): Query, headers: HeaderMap, body: Bytes, +) -> Response { + post_run_execute_session_inner(st, principal, prompt_hash, session_id, run_q, headers, body) + .instrument(crate::spans::execute_run_post()) + .await +} + +pub(crate) async fn post_run_execute_session_inner( + st: PlasmHostState, + principal: Option, + prompt_hash: PromptHashHex, + session_id: ExecuteSessionId, + run_q: ExecuteRunQuery, + headers: HeaderMap, + body: Bytes, ) -> Response { let Some(sess) = st .get_execute_session(prompt_hash.as_str(), session_id.as_str()) @@ -49,15 +65,15 @@ pub(crate) async fn post_run_execute_session( Ok(k) => k, Err(AcceptNegotiationError::NoSupportedMediaType) => { return problem_response( - Problem::custom( - ProblemStatus::NOT_ACCEPTABLE, - Uri::from_static(problem_types::EXECUTE_UNSUPPORTED_ACCEPT), - ) - .with_title("Not Acceptable") - .with_detail( - "supported Accept values include application/json, application/x-ndjson, text/plain, text/toon (default when Accept is omitted: text/toon)", - ), - ); + Problem::custom( + ProblemStatus::NOT_ACCEPTABLE, + Uri::from_static(problem_types::EXECUTE_UNSUPPORTED_ACCEPT), + ) + .with_title("Not Acceptable") + .with_detail( + "supported Accept values include application/json, application/x-ndjson, text/plain, text/toon (default when Accept is omitted: text/toon)", + ), + ); } }; @@ -320,15 +336,15 @@ pub(crate) async fn post_run_execute_session( crate::PlanGateDecision::Proceed(_) => {} crate::PlanGateDecision::NeedsReview => { return problem_response( - Problem::custom( - ProblemStatus::BAD_REQUEST, - Uri::from_static(problem_types::EXECUTE_INVALID_EXPRESSION), - ) - .with_title("Bad Request") - .with_detail( - "plan_requires_review: call plan dry-run first, then pass plan_commit_ref or force=true", - ), - ); + Problem::custom( + ProblemStatus::BAD_REQUEST, + Uri::from_static(problem_types::EXECUTE_INVALID_EXPRESSION), + ) + .with_title("Bad Request") + .with_detail( + "plan_requires_review: call plan dry-run first, then pass plan_commit_ref or force=true", + ), + ); } crate::PlanGateDecision::Denied(denial) => { return problem_response( diff --git a/crates/plasm-agent-core/src/http_execute/routes/tests.rs b/crates/plasm-agent-core/src/http_execute/routes/tests.rs index 217f0345..2f7f8749 100644 --- a/crates/plasm-agent-core/src/http_execute/routes/tests.rs +++ b/crates/plasm-agent-core/src/http_execute/routes/tests.rs @@ -866,3 +866,65 @@ fn negotiate_accept_variants() { ); assert!(negotiate_accept(Some("application/soap+xml")).is_err()); } + +#[test] +fn http_request_parents_execute_run_post_on_handler() { + use crate::execute_path_ids::{ExecuteSessionId, PromptHashHex}; + use crate::http_execute::response::ExecuteRunQuery; + use axum::body::Bytes; + use axum::http::HeaderMap; + use plasm_otel::span_capture::{find_span, is_child_of, with_captured_spans}; + use plasm_otel::tower_http_trace_parent_span; + use tracing::Instrument; + + let req = Request::builder() + .method("POST") + .uri("/execute/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") + .body(()) + .unwrap(); + + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let ((), spans) = with_captured_spans(|| { + let http = tower_http_trace_parent_span(&req); + // Create `execute_run_post` only after `http.request` is entered — tracing records + // parent at span construction time (mirrors TraceLayer then handler `.instrument`). + rt.block_on( + async { + async { + let st = test_state_with_registry(); + let prompt_hash: PromptHashHex = + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + .parse() + .expect("prompt hash"); + let session_id: ExecuteSessionId = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + .parse() + .expect("session id"); + let _ = super::handlers::post_run_execute_session_inner( + st, + None, + prompt_hash, + session_id, + ExecuteRunQuery::default(), + HeaderMap::new(), + Bytes::new(), + ) + .await; + } + .instrument(crate::spans::execute_run_post()) + .await; + } + .instrument(http), + ); + }); + + let parent = find_span(&spans, "plasm_agent.http.request").expect("http.request"); + let child = find_span(&spans, "plasm_agent.execute.run_post").expect("execute.run_post"); + assert!( + is_child_of(child, parent), + "execute.run_post must be child of http.request; spans={:?}", + spans.iter().map(|s| s.name.as_ref()).collect::>() + ); +} diff --git a/crates/plasm-agent-core/src/http_oauth_link.rs b/crates/plasm-agent-core/src/http_oauth_link.rs index c2953c44..27727157 100644 --- a/crates/plasm-agent-core/src/http_oauth_link.rs +++ b/crates/plasm-agent-core/src/http_oauth_link.rs @@ -21,7 +21,7 @@ use serde::Deserialize; use serde_json::json; use sha2::{Digest, Sha256}; use std::time::Duration; -use tracing::instrument; +use tracing::Instrument; use crate::control_plane_http::internal_or_outbound_setup_authorized; use crate::oauth_binding_kv::write_oauth_binding_pointer; @@ -225,15 +225,20 @@ struct StartBody { auth_config_id: Option, } -#[instrument( - skip(st, headers, body), - target = "plasm_agent::oauth_link", - fields(oauth.phase = "start") -)] async fn start_handler( Extension(st): Extension, headers: axum::http::HeaderMap, Json(body): Json, +) -> Result, OauthStartJsonError> { + start_handler_inner(st, headers, body) + .instrument(crate::spans::oauth_link_start()) + .await +} + +async fn start_handler_inner( + st: PlasmHostState, + headers: axum::http::HeaderMap, + body: StartBody, ) -> Result, OauthStartJsonError> { if !internal_or_outbound_setup_authorized(&headers, "oauth-link start") { return Err(oauth_start_json_err( @@ -475,15 +480,20 @@ struct DevicePollBody { device_code: String, } -#[instrument( - skip(st, headers, body), - target = "plasm_agent::oauth_link", - fields(oauth.phase = "device_start") -)] async fn device_start_handler( Extension(st): Extension, headers: axum::http::HeaderMap, Json(body): Json, +) -> Result, OauthStartJsonError> { + device_start_handler_inner(st, headers, body) + .instrument(crate::spans::oauth_link_device_start()) + .await +} + +async fn device_start_handler_inner( + st: PlasmHostState, + headers: axum::http::HeaderMap, + body: DeviceStartBody, ) -> Result, OauthStartJsonError> { if !internal_or_outbound_setup_authorized(&headers, "oauth-link device start") { return Err(oauth_start_json_err( @@ -614,15 +624,20 @@ async fn device_start_handler( }))) } -#[instrument( - skip(st, headers, body), - target = "plasm_agent::oauth_link", - fields(oauth.phase = "device_poll") -)] async fn device_poll_handler( Extension(st): Extension, headers: axum::http::HeaderMap, Json(body): Json, +) -> Result, OauthStartJsonError> { + device_poll_handler_inner(st, headers, body) + .instrument(crate::spans::oauth_link_device_poll()) + .await +} + +async fn device_poll_handler_inner( + st: PlasmHostState, + headers: axum::http::HeaderMap, + body: DevicePollBody, ) -> Result, OauthStartJsonError> { if !internal_or_outbound_setup_authorized(&headers, "oauth-link device poll") { return Err(oauth_start_json_err( diff --git a/crates/plasm-agent-core/src/incoming_auth.rs b/crates/plasm-agent-core/src/incoming_auth.rs index eaa06e0f..86ccbd1e 100644 --- a/crates/plasm-agent-core/src/incoming_auth.rs +++ b/crates/plasm-agent-core/src/incoming_auth.rs @@ -394,7 +394,7 @@ pub async fn incoming_auth_http_middleware( .unwrap_or(""); let span = crate::spans::security_incoming_http(principal.0.is_some(), tenant_id); req.extensions_mut().insert(principal); - Ok(async move { next.run(req).await }.instrument(span).await) + Ok(next.run(req).instrument(span).await) } #[cfg(test)] diff --git a/crates/plasm-agent-core/src/lib.rs b/crates/plasm-agent-core/src/lib.rs index 7456f580..e2b52239 100644 --- a/crates/plasm-agent-core/src/lib.rs +++ b/crates/plasm-agent-core/src/lib.rs @@ -110,6 +110,7 @@ mod plan_dry_compact; mod plan_dry_display; pub mod release_version; pub(crate) mod tool_model_service; +pub mod traced_pg; pub use plan_dry_display::PlanDryVerdict; mod approval_gate; mod flow_catalog; @@ -195,6 +196,8 @@ pub mod terminal_plan_run; pub use graph_page_spill_host::graph_page_spill_for_execute; pub mod mcp_logical_ref; pub mod session_identity; +#[cfg(test)] +mod span_graph_tests; pub mod spans; mod stream_consume; pub mod subcommand_util; diff --git a/crates/plasm-agent-core/src/mcp_config_repository.rs b/crates/plasm-agent-core/src/mcp_config_repository.rs index 734f30f5..0de184e0 100644 --- a/crates/plasm-agent-core/src/mcp_config_repository.rs +++ b/crates/plasm-agent-core/src/mcp_config_repository.rs @@ -2,10 +2,11 @@ use std::collections::{HashMap, HashSet}; +use crate::traced_pg::PgPool; use chrono::Utc; use serde_json::{json, Value}; use sqlx::postgres::PgPoolOptions; -use sqlx::{PgPool, Postgres, QueryBuilder, Row}; +use sqlx::{Postgres, QueryBuilder, Row}; use thiserror::Error; use uuid::Uuid; @@ -76,7 +77,7 @@ pub struct McpConfigRepository { pool: PgPool, } -async fn project_mcp_configs_table_exists(pool: &PgPool) -> Result { +async fn project_mcp_configs_table_exists(pool: &sqlx::PgPool) -> Result { sqlx::query_scalar( r#"SELECT EXISTS ( SELECT 1 @@ -91,7 +92,7 @@ async fn project_mcp_configs_table_exists(pool: &PgPool) -> Result Result { +async fn sqlx_migrations_table_exists(pool: &sqlx::PgPool) -> Result { sqlx::query_scalar( r#"SELECT EXISTS ( SELECT 1 @@ -114,7 +115,10 @@ const LEGACY_SQUASHED_MIGRATION_VERSIONS: &[i64] = &[ 20260512130000, ]; -async fn delete_migration_ledger_version(pool: &PgPool, version: i64) -> Result<(), sqlx::Error> { +async fn delete_migration_ledger_version( + pool: &sqlx::PgPool, + version: i64, +) -> Result<(), sqlx::Error> { sqlx::query("DELETE FROM _sqlx_migrations WHERE version = $1") .bind(version) .execute(pool) @@ -123,7 +127,7 @@ async fn delete_migration_ledger_version(pool: &PgPool, version: i64) -> Result< } /// Remove ledger rows for migrations squashed out of the embedded set so `sqlx::migrate` can run. -async fn prune_squashed_migration_ledger(pool: &PgPool) -> Result<(), sqlx::Error> { +async fn prune_squashed_migration_ledger(pool: &sqlx::PgPool) -> Result<(), sqlx::Error> { if !sqlx_migrations_table_exists(pool).await? { return Ok(()); } @@ -133,7 +137,7 @@ async fn prune_squashed_migration_ledger(pool: &PgPool) -> Result<(), sqlx::Erro Ok(()) } -async fn run_mcp_config_migrations(pool: &PgPool) -> Result<(), McpConfigRepositoryError> { +async fn run_mcp_config_migrations(pool: &sqlx::PgPool) -> Result<(), McpConfigRepositoryError> { prune_squashed_migration_ledger(pool).await?; let migrator = sqlx::migrate!("./migrations"); match migrator.run(pool).await { @@ -160,6 +164,7 @@ impl McpConfigRepository { if !project_mcp_configs_table_exists(&pool).await? { return Err(McpConfigRepositoryError::PostMigrateSchemaMissing); } + let pool = crate::traced_pg::wrap(pool); Ok(Self { pool }) } @@ -673,12 +678,12 @@ impl McpConfigRepository { .bind(auth_optional_entry_ids) .bind(now) .bind(now) - .execute(&mut *tx) + .execute(&mut tx.executor()) .await?; sqlx::query("DELETE FROM project_mcp_allowed_graphs WHERE config_id = $1") .bind(runtime.id) - .execute(&mut *tx) + .execute(&mut tx.executor()) .await?; for eid in &runtime.allowed_entry_ids { @@ -692,13 +697,13 @@ impl McpConfigRepository { .bind(Uuid::new_v4()) .bind(runtime.id) .bind(eid) - .execute(&mut *tx) + .execute(&mut tx.executor()) .await?; } sqlx::query("DELETE FROM project_mcp_allowed_capabilities WHERE config_id = $1") .bind(runtime.id) - .execute(&mut *tx) + .execute(&mut tx.executor()) .await?; for (entry_id, names) in &runtime.capabilities_by_entry { @@ -718,14 +723,14 @@ impl McpConfigRepository { .bind(runtime.id) .bind(entry_id) .bind(cap) - .execute(&mut *tx) + .execute(&mut tx.executor()) .await?; } } sqlx::query("DELETE FROM project_mcp_auth_bindings WHERE config_id = $1") .bind(runtime.id) - .execute(&mut *tx) + .execute(&mut tx.executor()) .await?; for (entry_id, auth_id) in &runtime.auth_config_by_entry { @@ -742,13 +747,13 @@ impl McpConfigRepository { .bind(auth_id) .bind(now) .bind(now) - .execute(&mut *tx) + .execute(&mut tx.executor()) .await?; } sqlx::query("DELETE FROM project_mcp_credentials WHERE config_id = $1") .bind(runtime.id) - .execute(&mut *tx) + .execute(&mut tx.executor()) .await?; for hash in &runtime.credential_secret_hashes { @@ -762,7 +767,7 @@ impl McpConfigRepository { .bind(&hash[..]) .bind(now) .bind(now) - .execute(&mut *tx) + .execute(&mut tx.executor()) .await?; } diff --git a/crates/plasm-agent-core/src/mcp_server/call_tool_dispatch.rs b/crates/plasm-agent-core/src/mcp_server/call_tool_dispatch.rs index 6ca4cfd8..c2188834 100644 --- a/crates/plasm-agent-core/src/mcp_server/call_tool_dispatch.rs +++ b/crates/plasm-agent-core/src/mcp_server/call_tool_dispatch.rs @@ -10,11 +10,23 @@ use rust_mcp_sdk::McpServer; use super::discover::mcp_call_tool_error_class; use super::schema::args_value; use super::{mcp_key, PlasmMcpHandler}; +use tracing::Instrument; pub(crate) async fn dispatch_plasm_mcp_call_tool_request( handler: &PlasmMcpHandler, params: CallToolRequestParams, runtime: Arc, +) -> Result { + let tool_name = params.name.clone(); + dispatch_plasm_mcp_call_tool_request_inner(handler, params, runtime) + .instrument(crate::spans::mcp_call_tool(tool_name.as_str())) + .await +} + +async fn dispatch_plasm_mcp_call_tool_request_inner( + handler: &PlasmMcpHandler, + params: CallToolRequestParams, + runtime: Arc, ) -> Result { fn record_workflow_tool( tname: &'static str, @@ -205,7 +217,7 @@ pub(crate) async fn dispatch_plasm_mcp_call_tool_request( "unknown_tool", Duration::from_secs(0), ); - Err(CallToolError::unknown_tool(params.name)) + Err(CallToolError::unknown_tool(params.name.clone())) } } } diff --git a/crates/plasm-agent-core/src/mcp_server/committed_plasm_run.rs b/crates/plasm-agent-core/src/mcp_server/committed_plasm_run.rs index efb1028c..0f737780 100644 --- a/crates/plasm-agent-core/src/mcp_server/committed_plasm_run.rs +++ b/crates/plasm-agent-core/src/mcp_server/committed_plasm_run.rs @@ -1,6 +1,7 @@ //! Live MCP `plasm_run` — reviewed commits (`pcN`) and paging continuations via unified `run_ref`. use std::sync::Arc; +use tracing::Instrument; use plasm_core::{PagingHandle, PlanCommitRef, PromptPipelineConfig, SymbolMapCrossRequestCache}; @@ -215,6 +216,12 @@ fn prepare_live_dry( } pub async fn execute_mcp_live_run(run: ExecuteMcpLiveRun) -> Result { + execute_mcp_live_run_inner(run) + .instrument(crate::spans::plan_live_run()) + .await +} + +async fn execute_mcp_live_run_inner(run: ExecuteMcpLiveRun) -> Result { if !run.wait_live { return Err("plasm_run requires live execute".to_string()); } diff --git a/crates/plasm-agent-core/src/mcp_server/initialize.rs b/crates/plasm-agent-core/src/mcp_server/initialize.rs index 9fe5a88c..be1cff35 100644 --- a/crates/plasm-agent-core/src/mcp_server/initialize.rs +++ b/crates/plasm-agent-core/src/mcp_server/initialize.rs @@ -173,7 +173,7 @@ pub async fn run_mcp_server(host: &str, port: u16, plasm: Arc) - let listener = tokio::net::TcpListener::bind(&addr) .await .map_err(|e| SdkError::internal_error().with_message(&format!("bind {addr}: {e}")))?; - tracing::info!("MCP stateless HTTP listening on http://{addr}"); + tracing::info!(%addr, "MCP stateless HTTP listening"); axum::serve(listener, router) .await .map_err(|e| SdkError::internal_error().with_message(&e.to_string()))?; diff --git a/crates/plasm-agent-core/src/mcp_server/plasm_tool_dry_run.rs b/crates/plasm-agent-core/src/mcp_server/plasm_tool_dry_run.rs index f3a23afa..e0c8fb49 100644 --- a/crates/plasm-agent-core/src/mcp_server/plasm_tool_dry_run.rs +++ b/crates/plasm-agent-core/src/mcp_server/plasm_tool_dry_run.rs @@ -37,6 +37,16 @@ pub(crate) struct PlasmDryRunContext<'a> { pub(crate) async fn execute_plasm_tool_dry_run( ctx: PlasmDryRunContext<'_>, program: &str, +) -> Result { + use tracing::Instrument; + execute_plasm_tool_dry_run_inner(ctx, program) + .instrument(crate::spans::plan_dry_run(program.len())) + .await +} + +async fn execute_plasm_tool_dry_run_inner( + ctx: PlasmDryRunContext<'_>, + program: &str, ) -> Result { let total_started = Instant::now(); let plan_name = format!("plasm_dag_call_{}", ctx.call_index); diff --git a/crates/plasm-agent-core/src/oauth_link_session.rs b/crates/plasm-agent-core/src/oauth_link_session.rs index 9ad0c8c2..6199bd74 100644 --- a/crates/plasm-agent-core/src/oauth_link_session.rs +++ b/crates/plasm-agent-core/src/oauth_link_session.rs @@ -20,7 +20,7 @@ use plasm_runtime::{ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::time::Duration; -use tracing::instrument; +use tracing::Instrument; /// KV key prefix for in-flight OAuth link sessions (`{PENDING_PREFIX}{csrf_state}`). pub const PENDING_PREFIX: &str = "plasm:oauth_link:pending:"; @@ -308,18 +308,21 @@ impl OauthLinkSession { impl OauthLinkSession { /// Run the authorization-code + PKCE token exchange, store [`OutboundOAuthKvV1`] at /// `core.hosted_kv_key`, and return redirect parameters for the SaaS. - #[instrument( - skip(self, redirect_uri), - target = "plasm_agent::oauth_link", - fields( - oauth.phase = "token_exchange", - entry_id = %self.core.entry_id, - ) - )] pub async fn exchange_and_store( self, redirect_uri: String, http_timeout: Duration, + ) -> Result { + let span = crate::spans::oauth_link_token_exchange(&self.core.entry_id); + self.exchange_and_store_inner(redirect_uri, http_timeout) + .instrument(span) + .await + } + + async fn exchange_and_store_inner( + self, + redirect_uri: String, + http_timeout: Duration, ) -> Result { let http = build_oauth_token_http_client(http_timeout) .map_err(|_| OauthExchangeError::HttpClient)?; diff --git a/crates/plasm-agent-core/src/oauth_provider_pull.rs b/crates/plasm-agent-core/src/oauth_provider_pull.rs index efc89335..b0cc9fab 100644 --- a/crates/plasm-agent-core/src/oauth_provider_pull.rs +++ b/crates/plasm-agent-core/src/oauth_provider_pull.rs @@ -80,7 +80,7 @@ pub async fn init_oauth_provider_pull_from_postgres( .connect(&settings.database_url) .await { - Ok(p) => Arc::new(p), + Ok(p) => Arc::new(crate::traced_pg::wrap(p)), Err(e) => { return OauthProviderPullInitOutcome::ConnectFailed { error: e.to_string(), diff --git a/crates/plasm-agent-core/src/oauth_provider_repository.rs b/crates/plasm-agent-core/src/oauth_provider_repository.rs index 2f56ae3c..4f980e96 100644 --- a/crates/plasm-agent-core/src/oauth_provider_repository.rs +++ b/crates/plasm-agent-core/src/oauth_provider_repository.rs @@ -1,6 +1,7 @@ //! sqlx persistence for `oauth_provider_apps` (outbound OAuth provider registry). -use sqlx::{PgPool, Row}; +use crate::traced_pg::PgPool; +use sqlx::Row; #[derive(Debug, Clone, serde::Serialize)] pub struct OauthProviderAppRow { diff --git a/crates/plasm-agent-core/src/oauth_runtime_source.rs b/crates/plasm-agent-core/src/oauth_runtime_source.rs index a8367407..617b0793 100644 --- a/crates/plasm-agent-core/src/oauth_runtime_source.rs +++ b/crates/plasm-agent-core/src/oauth_runtime_source.rs @@ -5,7 +5,8 @@ use std::collections::HashMap; use std::future::Future; use std::pin::Pin; -use sqlx::{PgPool, Row}; +use crate::traced_pg::PgPool; +use sqlx::Row; use thiserror::Error; use crate::oauth_link_catalog::OauthLinkCatalog; diff --git a/crates/plasm-agent-core/src/plan_dry_display.rs b/crates/plasm-agent-core/src/plan_dry_display.rs index b4d51d1a..ae2d85b3 100644 --- a/crates/plasm-agent-core/src/plan_dry_display.rs +++ b/crates/plasm-agent-core/src/plan_dry_display.rs @@ -156,6 +156,9 @@ pub enum PlanDryOp { Dedupe { keys: Vec, }, + With { + columns: Vec, + }, Render { columns: Vec, template_chars: usize, @@ -336,6 +339,7 @@ pub(crate) fn human_ux_headline_for_op(op: &PlanDryOp) -> String { PlanDryOp::Limit { count } => format!("Take first {count}"), PlanDryOp::Dedupe { keys } if keys.is_empty() => "Distinct rows".into(), PlanDryOp::Dedupe { keys } => format!("Dedupe on {}", keys.join(", ")), + PlanDryOp::With { columns } => format!("Add columns {}", columns.join(", ")), PlanDryOp::Render { .. } => "Render text".into(), PlanDryOp::ForEach { .. } => "For each row".into(), PlanDryOp::Relation { .. } => "Follow relation".into(), @@ -374,6 +378,7 @@ pub(crate) fn human_ux_summary_for_op(op: &PlanDryOp) -> String { PlanDryOp::Aggregate { .. } => "Summarize".into(), PlanDryOp::Dedupe { keys } if keys.is_empty() => "Distinct rows".into(), PlanDryOp::Dedupe { keys } => format!("Dedupe on {}", keys.join(", ")), + PlanDryOp::With { columns } => format!("Add {}", columns.join(", ")), PlanDryOp::Render { columns, .. } => format!("Render {}", columns.join(", ")), PlanDryOp::Relation { relation, target, .. @@ -408,6 +413,7 @@ pub(crate) fn render_plan_dry_op(op: &PlanDryOp) -> String { format!("dedupe {}", keys.join(", ")) } } + PlanDryOp::With { columns } => format!("with {}", columns.join(", ")), PlanDryOp::Render { columns, template_chars, @@ -499,6 +505,12 @@ fn compact_op_from_compute( ComputeOp::DedupeBy { keys } => PlanDryOp::Dedupe { keys: keys.iter().map(|k| k.dotted()).collect(), }, + ComputeOp::With { columns } => PlanDryOp::With { + columns: columns + .iter() + .map(|c| c.name.as_str().to_string()) + .collect(), + }, ComputeOp::Render { columns, template, .. } => PlanDryOp::Render { diff --git a/crates/plasm-agent-core/src/plan_flow.rs b/crates/plasm-agent-core/src/plan_flow.rs index be9a45c4..ace116b9 100644 --- a/crates/plasm-agent-core/src/plan_flow.rs +++ b/crates/plasm-agent-core/src/plan_flow.rs @@ -624,7 +624,8 @@ impl<'a, P: FlowPolicyEvaluator + ?Sized> FlowPass<'a, P> { ComputeOp::Filter { .. } | ComputeOp::Sort { .. } | ComputeOp::Limit { .. } - | ComputeOp::DedupeBy { .. } => { + | ComputeOp::DedupeBy { .. } + | ComputeOp::With { .. } => { out = source_facts.clone(); } ComputeOp::GroupBy { aggregates, .. } | ComputeOp::Aggregate { aggregates, .. } => { diff --git a/crates/plasm-agent-core/src/plasm_dag/postfix/postfix_op.rs b/crates/plasm-agent-core/src/plasm_dag/postfix/postfix_op.rs index 2781854d..16cc206c 100644 --- a/crates/plasm-agent-core/src/plasm_dag/postfix/postfix_op.rs +++ b/crates/plasm-agent-core/src/plasm_dag/postfix/postfix_op.rs @@ -70,7 +70,26 @@ pub(in crate::plasm_dag) fn postfix_op_to_compute( cgs: cgs.as_ref(), symbol_map: None, }; - plasm_core::type_check_row_predicate(&row_pred, &tc_ctx).map_err(|e| e.to_string())?; + let predicates = crate::row_predicate_lower::lower_row_predicate_to_plan( + &row_pred, + session, + &qe, + state.cross_cache, + )?; + let extra = resolve_immediate_compute_schema(state, staged, source); + let mut catalog_pred = row_pred.clone(); + if let Some(schema) = extra.as_ref() { + catalog_pred.0.retain(|c| { + !schema + .fields + .iter() + .any(|f| f.name.as_str() == c.field.as_str()) + }); + } + if !catalog_pred.0.is_empty() { + plasm_core::type_check_row_predicate(&catalog_pred, &tc_ctx) + .map_err(|e| e.to_string())?; + } let mut paths = Vec::new(); for clause in &row_pred.0 { paths.push(FieldPath::from_dotted(clause.field.as_str())?); @@ -85,13 +104,13 @@ pub(in crate::plasm_dag) fn postfix_op_to_compute( "filter(...)", )?; } - let predicates = crate::row_predicate_lower::lower_row_predicate_to_plan( - &row_pred, + let schema = compute_passthrough_or_fallback_schema( session, - &qe, - state.cross_cache, - )?; - let schema = synthetic_schema_passthrough_rows(session, state, staged, source)?; + state, + staged, + source, + "PlanFilter", + ); Ok(mk(ComputeOp::Filter { predicates }, schema, false)) } PlasmPostfixOp::Sort { args } => { @@ -229,10 +248,43 @@ pub(in crate::plasm_dag) fn postfix_op_to_compute( let schema = synthetic_schema_passthrough_rows(session, state, staged, source)?; Ok(mk(ComputeOp::DedupeBy { keys: vec![] }, schema, false)) } + PlasmPostfixOp::With { body } => { + let columns = plasm_core::parse_with_body(body).map_err(|e| e.to_string())?; + let schema = synthetic_schema_passthrough_rows(session, state, staged, source)?; + let mut schema = schema; + for col in &columns { + schema.fields.push(plasm_core::SyntheticFieldSchema { + name: col.name.clone(), + value_kind: SyntheticValueKind::Unknown, + source: None, + }); + } + Ok(mk(ComputeOp::With { columns }, schema, false)) + } PlasmPostfixOp::Projection { fields } => { let qe = resolve_qualified_entity_for_dag_source(state, staged, source.to_string()); + let source_schema = resolve_immediate_compute_schema(state, staged, source); let mut map = BTreeMap::new(); - for field in parse_field_list(session, state.cross_cache, qe.as_ref(), fields)? { + for field in + parse_field_list(session, state.cross_cache, qe.as_ref(), fields).or_else(|_| { + fields + .split(',') + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|raw| { + let path = FieldPath::from_dotted(raw)?; + let resolved = resolve_sort_field_path( + session, + state.cross_cache, + qe.as_ref(), + source_schema.as_ref(), + &path, + )?; + Ok(resolved.dotted()) + }) + .collect::, String>>() + })? + { map.insert( OutputName::new(field.clone())?, FieldPath::from_dotted(&field)?, diff --git a/crates/plasm-agent-core/src/plasm_dag/postfix/row_suffix.rs b/crates/plasm-agent-core/src/plasm_dag/postfix/row_suffix.rs index 2caae3be..f37216f9 100644 --- a/crates/plasm-agent-core/src/plasm_dag/postfix/row_suffix.rs +++ b/crates/plasm-agent-core/src/plasm_dag/postfix/row_suffix.rs @@ -92,6 +92,7 @@ pub(in crate::plasm_dag) fn row_suffix_to_postfix(suffix: &RowSuffix) -> Option< RowSuffix::GroupBy { args } => Some(PlasmPostfixOp::GroupBy { args: args.clone() }), RowSuffix::Dedupe { keys } => Some(PlasmPostfixOp::Dedupe { keys: keys.clone() }), RowSuffix::Distinct { keys } => Some(PlasmPostfixOp::Distinct { keys: keys.clone() }), + RowSuffix::With { body } => Some(PlasmPostfixOp::With { body: body.clone() }), RowSuffix::Singleton => Some(PlasmPostfixOp::Singleton), RowSuffix::PageSize { n } => Some(PlasmPostfixOp::PageSize(*n as usize)), RowSuffix::Relation { .. } => None, diff --git a/crates/plasm-agent-core/src/plasm_dag/schema_validate/compute_schema.rs b/crates/plasm-agent-core/src/plasm_dag/schema_validate/compute_schema.rs index 69f3930d..8073fac9 100644 --- a/crates/plasm-agent-core/src/plasm_dag/schema_validate/compute_schema.rs +++ b/crates/plasm-agent-core/src/plasm_dag/schema_validate/compute_schema.rs @@ -36,21 +36,16 @@ pub(in crate::plasm_dag) fn infer_render_columns_for_node( cols.extend(aggregates.iter().map(|a| a.name.clone())); Ok(cols) } - ComputeOp::Sort { .. } | ComputeOp::Limit { .. } | ComputeOp::DedupeBy { .. } => { + ComputeOp::Sort { .. } | ComputeOp::Limit { .. } | ComputeOp::DedupeBy { .. } | ComputeOp::Filter { .. } => { let parent = lookup_dag_node(state, staged, parent_id.as_str()).ok_or_else(|| { format!("template column inference: missing upstream node `{parent_id}`") })?; infer_render_columns_for_node(session, state, staged, parent) } + ComputeOp::With { .. } => Ok(schema.fields.iter().map(|f| f.name.clone()).collect()), ComputeOp::Render { .. } => Err( "cannot infer columns from a row-to-text template result; bind a row-producing query/relation/projection, or write explicit `[field,...] < { - let parent = lookup_dag_node(state, staged, parent_id.as_str()).ok_or_else(|| { - format!("template column inference: missing upstream node `{parent_id}`") - })?; - infer_render_columns_for_node(session, state, staged, parent) - } }, DagNodeSource::Surface { qualified_entity, .. diff --git a/crates/plasm-agent-core/src/plasm_plan.rs b/crates/plasm-agent-core/src/plasm_plan.rs index f3948ab7..d4d95d3c 100644 --- a/crates/plasm-agent-core/src/plasm_plan.rs +++ b/crates/plasm-agent-core/src/plasm_plan.rs @@ -1660,7 +1660,8 @@ fn analyze_static_cardinality( ComputeOp::Project { .. } | ComputeOp::Filter { .. } | ComputeOp::Sort { .. } - | ComputeOp::DedupeBy { .. } => inner(plan, by_id, &compute.source, memo), + | ComputeOp::DedupeBy { .. } + | ComputeOp::With { .. } => inner(plan, by_id, &compute.source, memo), ComputeOp::Limit { count } if *count <= 1 => { CardinalityAnalysis::StaticSingleton } @@ -1755,7 +1756,8 @@ fn validated_analyze_static_cardinality( ComputeOp::Project { .. } | ComputeOp::Filter { .. } | ComputeOp::Sort { .. } - | ComputeOp::DedupeBy { .. } => inner(plan, by_id, c.compute.source.as_str(), memo), + | ComputeOp::DedupeBy { .. } + | ComputeOp::With { .. } => inner(plan, by_id, c.compute.source.as_str(), memo), ComputeOp::Limit { count } if *count <= 1 => CardinalityAnalysis::StaticSingleton, ComputeOp::Limit { .. } | ComputeOp::GroupBy { .. } => { CardinalityAnalysis::PluralOrUnknown @@ -1825,6 +1827,11 @@ fn validate_compute_template( validate_predicate(p, node_index, j)?; } } + ComputeOp::With { columns } if columns.is_empty() => { + return Err(format!( + "plan.nodes[{node_index}].compute.with.columns must be non-empty" + )); + } ComputeOp::GroupBy { aggregates, .. } | ComputeOp::Aggregate { aggregates } => { if aggregates.is_empty() { return Err(format!( diff --git a/crates/plasm-agent-core/src/plasm_plan_run/compute_eval/compute_ops.rs b/crates/plasm-agent-core/src/plasm_plan_run/compute_eval/compute_ops.rs index 7a6d27f3..d332f11b 100644 --- a/crates/plasm-agent-core/src/plasm_plan_run/compute_eval/compute_ops.rs +++ b/crates/plasm-agent-core/src/plasm_plan_run/compute_eval/compute_ops.rs @@ -2,11 +2,11 @@ use std::collections::BTreeMap; use std::sync::Arc; use minijinja::value::{Enumerator, Object, ObjectRepr}; +use plasm_runtime::{eval_compute_ops, ComputeEvalOutcome}; use crate::plasm_plan::OutputName; use crate::plasm_render_compile::render_context_hint; -use super::super::value_at_field_path as value_at_path; use super::super::*; pub(crate) async fn eval_compute_with_row_source( @@ -18,94 +18,15 @@ pub(crate) async fn eval_compute_with_row_source( session_id: &str, cgs: &CGS, ) -> Result, String> { - match row_source { - MaterializedRowSource::Inline(rows) => { - eval_compute_from_rows(compute, rows, cross_binding_rows) - } - MaterializedRowSource::GraphBacked { - entity_type, - logical_count, - hot_snapshot, - } => { - if matches!(&compute.op, ComputeOp::Render { .. }) { - let rows = - crate::graph_rehydrate::GraphSurfaceRehydrator::new(es, st, session_id, cgs) - .resolve_row_source_rows( - row_source, - Some(crate::plasm_plan::PLAN_RENDER_MAX_ROWS), - ) - .await?; - return eval_compute_from_rows(compute, &rows, cross_binding_rows); - } - if compute_needs_full_materialize(&compute.op) { - let rows = - crate::graph_rehydrate::GraphSurfaceRehydrator::new(es, st, session_id, cgs) - .rehydrate_rows( - std::sync::Arc::clone(hot_snapshot), - entity_type, - *logical_count, - ) - .await?; - return eval_compute_from_rows(compute, &rows, cross_binding_rows); - } - eval_compute_streaming( - compute, - es, - st, - session_id, - entity_type, - cgs, - std::sync::Arc::clone(hot_snapshot), - ) - .await - } - } -} - -pub(crate) async fn eval_compute_streaming( - compute: &ComputeTemplate, - es: &ExecuteSession, - st: &PlasmHostState, - session_id: &str, - entity_type: &str, - cgs: &CGS, - hot_snapshot: std::sync::Arc<[plasm_runtime::CachedEntity]>, -) -> Result, String> { - let mut out = Vec::new(); - let limit = match &compute.op { - ComputeOp::Limit { count } => Some(*count), - _ => None, + let cap = if matches!(&compute.op, ComputeOp::Render { .. }) { + Some(crate::plasm_plan::PLAN_RENDER_MAX_ROWS) + } else { + None }; - crate::graph_rehydrate::GraphSurfaceRehydrator::new(es, st, session_id, cgs) - .stream_entity_rows(hot_snapshot, entity_type, |row| { - match &compute.op { - ComputeOp::Filter { predicates } => { - if predicates.iter().all(|p| predicate_matches(row, p)) { - out.push(row.clone()); - } - } - ComputeOp::Limit { .. } => out.push(row.clone()), - ComputeOp::Project { fields } => { - let mut obj = serde_json::Map::new(); - for (name, path) in fields { - obj.insert( - name.as_str().to_string(), - value_at_path(row, path) - .cloned() - .unwrap_or(serde_json::Value::Null), - ); - } - out.push(serde_json::Value::Object(obj)); - } - _ => {} - } - limit.is_some_and(|cap| out.len() >= cap) - }) + let rows = crate::graph_rehydrate::GraphSurfaceRehydrator::new(es, st, session_id, cgs) + .resolve_row_source_rows(row_source, cap) .await?; - if let ComputeOp::Limit { count } = &compute.op { - out.truncate(*count); - } - Ok(out) + eval_compute_from_rows(compute, &rows, cross_binding_rows) } pub(crate) fn eval_compute_from_rows( @@ -113,194 +34,27 @@ pub(crate) fn eval_compute_from_rows( rows: &[serde_json::Value], cross_binding_rows: &BTreeMap>, ) -> Result, String> { - match &compute.op { - ComputeOp::Project { fields } => rows - .iter() - .map(|row| { - let mut out = serde_json::Map::new(); - for (name, path) in fields { - out.insert( - name.as_str().to_string(), - value_at_path(row, path) - .cloned() - .unwrap_or(serde_json::Value::Null), - ); - } - Ok(serde_json::Value::Object(out)) - }) - .collect(), - ComputeOp::Filter { predicates } => Ok(rows - .iter() - .filter(|row| predicates.iter().all(|p| predicate_matches(row, p))) - .cloned() - .collect()), - ComputeOp::GroupBy { keys, aggregates } => group_rows(rows, keys, aggregates), - ComputeOp::Aggregate { aggregates } => aggregate_rows(rows, aggregates), - ComputeOp::Sort { key, descending } => { - let mut sorted = rows.to_vec(); - sorted - .sort_by(|a, b| cmp_json_sort_values(value_at_path(a, key), value_at_path(b, key))); - if *descending { - sorted.reverse(); - } - Ok(sorted) - } - ComputeOp::Limit { count } => Ok(rows.iter().take(*count).cloned().collect()), - ComputeOp::DedupeBy { keys } => dedupe_rows(rows, keys), - ComputeOp::Render { + match eval_compute_ops(std::slice::from_ref(&compute.op), rows)? { + ComputeEvalOutcome::Rows(out) => Ok(out), + ComputeEvalOutcome::Render { + rows, columns, - template, column_aliases, - render_bindings, - } => render_compute(&RenderComputeInput { - primary_rows: rows, - columns: &RenderColumns::from_op_parts(columns.clone(), column_aliases.clone()), template, - collection_alias: compute.collection_alias.as_ref(), + collection_alias, render_bindings, + } => render_compute(&RenderComputeInput { + primary_rows: &rows, + columns: &RenderColumns::from_op_parts(columns, column_aliases), + template: &template, + collection_alias: collection_alias + .as_ref() + .or(compute.collection_alias.as_ref()), + render_bindings: &render_bindings, binding_rows: cross_binding_rows, }), } } -pub(crate) fn dedupe_rows( - rows: &[serde_json::Value], - keys: &[FieldPath], -) -> Result, String> { - use std::collections::HashSet; - let mut seen = HashSet::new(); - let mut out = Vec::new(); - for row in rows { - let composite = if keys.is_empty() { - serde_json::to_string(row).unwrap_or_default() - } else { - let parts: Vec = keys - .iter() - .map(|k| { - value_at_path(row, k) - .map(json_scalar_display) - .unwrap_or_default() - }) - .collect(); - serde_json::to_string(&parts).unwrap_or_default() - }; - if seen.insert(composite) { - out.push(row.clone()); - } - } - Ok(out) -} - -pub(crate) fn group_rows( - rows: &[serde_json::Value], - keys: &[FieldPath], - aggregates: &[crate::plasm_plan::AggregateSpec], -) -> Result, String> { - if keys.is_empty() { - return Err("group_by requires at least one key".into()); - } - let mut groups: BTreeMap> = BTreeMap::new(); - for row in rows { - let parts: Vec = keys - .iter() - .map(|k| { - value_at_path(row, k) - .map(json_scalar_display) - .unwrap_or_default() - }) - .collect(); - let composite = serde_json::to_string(&parts).unwrap_or_default(); - groups.entry(composite).or_default().push(row); - } - let mut out = Vec::new(); - for (composite, group_rows) in groups { - let parts: Vec = serde_json::from_str(&composite).unwrap_or_default(); - let mut obj = serde_json::Map::new(); - for (key_path, part) in keys.iter().zip(parts.iter()) { - obj.insert(key_path.dotted(), serde_json::Value::String(part.clone())); - } - append_aggregates(&mut obj, &group_rows, aggregates)?; - out.push(serde_json::Value::Object(obj)); - } - Ok(out) -} - -pub(crate) fn aggregate_rows( - rows: &[serde_json::Value], - aggregates: &[crate::plasm_plan::AggregateSpec], -) -> Result, String> { - let refs = rows.iter().collect::>(); - let mut obj = serde_json::Map::new(); - append_aggregates(&mut obj, &refs, aggregates)?; - Ok(vec![serde_json::Value::Object(obj)]) -} - -pub(crate) fn append_aggregates( - obj: &mut serde_json::Map, - rows: &[&serde_json::Value], - aggregates: &[crate::plasm_plan::AggregateSpec], -) -> Result<(), String> { - for agg in aggregates { - let value = match agg.function { - AggregateFunction::Count => serde_json::json!(rows.len()), - AggregateFunction::Sum => { - serde_json::json!(aggregate_numbers(rows, agg.field.as_ref()) - .iter() - .sum::()) - } - AggregateFunction::Avg => { - let nums = aggregate_numbers(rows, agg.field.as_ref()); - serde_json::json!(if nums.is_empty() { - 0.0 - } else { - nums.iter().sum::() / nums.len() as f64 - }) - } - AggregateFunction::Min => aggregate_numbers(rows, agg.field.as_ref()) - .into_iter() - .reduce(f64::min) - .map(|n| serde_json::json!(n)) - .unwrap_or(serde_json::Value::Null), - AggregateFunction::Max => aggregate_numbers(rows, agg.field.as_ref()) - .into_iter() - .reduce(f64::max) - .map(|n| serde_json::json!(n)) - .unwrap_or(serde_json::Value::Null), - AggregateFunction::First => rows - .first() - .and_then(|row| { - agg.field - .as_ref() - .and_then(|f| value_at_path(row, f)) - .cloned() - }) - .unwrap_or(serde_json::Value::Null), - AggregateFunction::Last => rows - .last() - .and_then(|row| { - agg.field - .as_ref() - .and_then(|f| value_at_path(row, f)) - .cloned() - }) - .unwrap_or(serde_json::Value::Null), - }; - obj.insert(agg.name.as_str().to_string(), value); - } - Ok(()) -} - -pub(crate) fn aggregate_numbers( - rows: &[&serde_json::Value], - field: Option<&FieldPath>, -) -> Vec { - rows.iter() - .filter_map(|row| { - field - .and_then(|f| value_at_path(row, f)) - .and_then(json_number) - }) - .collect() -} pub(crate) struct RenderComputeInput<'a> { pub primary_rows: &'a [serde_json::Value], @@ -404,7 +158,6 @@ impl Object for RenderBindingValue { } fn get_value(self: &Arc, key: &minijinja::Value) -> Option { - // Attribute access (`items.title`) delegates to the first row. if let Some(name) = key.as_str() { return self .rows @@ -412,7 +165,6 @@ impl Object for RenderBindingValue { .and_then(|row| row.get_attr(name).ok()) .filter(|value| !value.is_undefined()); } - // Sequence index access (`items[0]`) and `{% for … %}` iteration. usize::try_from(key.clone()) .ok() .and_then(|idx| self.rows.get(idx).cloned()) @@ -458,10 +210,6 @@ pub(crate) fn binding_rows_for_render( Ok(out) } -pub(crate) fn json_number(v: &serde_json::Value) -> Option { - v.as_f64().or_else(|| v.as_i64().map(|n| n as f64)) -} - pub(crate) fn json_scalar_display(v: &serde_json::Value) -> String { match v { serde_json::Value::String(s) => s.clone(), @@ -484,37 +232,6 @@ pub(crate) fn json_plasm_literal_display(v: &serde_json::Value) -> String { } } -pub(crate) fn sort_display_key(v: Option<&serde_json::Value>) -> String { - v.map(json_scalar_display).unwrap_or_default() -} - -/// Compare two JSON cell values for deterministic `.sort(...)` ordering. -/// -/// When both values are numeric (JSON numbers or strings that parse as integers/floats), ordering is -/// numeric so multi-digit values sort correctly (`87` before `300`). Otherwise ordering follows the -/// legacy string collation used by [`sort_display_key`] (including missing/`null` → empty string). -pub(crate) fn cmp_json_sort_values( - a: Option<&serde_json::Value>, - b: Option<&serde_json::Value>, -) -> std::cmp::Ordering { - match (a, b) { - (Some(va), Some(vb)) => { - if let (Some(na), Some(nb)) = (json_number(va), json_number(vb)) { - return na.total_cmp(&nb); - } - if let (Some(sa), Some(sb)) = (va.as_str(), vb.as_str()) { - if let (Ok(ia), Ok(ib)) = (sa.parse::(), sb.parse::()) { - return ia.cmp(&ib); - } - if let (Ok(fa), Ok(fb)) = (sa.parse::(), sb.parse::()) { - return fa.total_cmp(&fb); - } - } - sort_display_key(Some(va)).cmp(&sort_display_key(Some(vb))) - } - _ => sort_display_key(a).cmp(&sort_display_key(b)), - } -} pub(crate) fn compute_fingerprint(node: &ValidatedPlanNode, rows: &[serde_json::Value]) -> String { use sha2::{Digest, Sha256}; let mut hasher = Sha256::new(); diff --git a/crates/plasm-agent-core/src/plasm_plan_run/dry_render.rs b/crates/plasm-agent-core/src/plasm_plan_run/dry_render.rs index d2cc7503..13ec9a7e 100644 --- a/crates/plasm-agent-core/src/plasm_plan_run/dry_render.rs +++ b/crates/plasm-agent-core/src/plasm_plan_run/dry_render.rs @@ -173,6 +173,15 @@ pub(crate) fn render_compute_template(compute: &ComputeTemplate) -> String { ) } } + ComputeOp::With { columns } => format!( + "with {} [{}]", + compute.source, + columns + .iter() + .map(|c| c.name.as_str()) + .collect::>() + .join(", ") + ), ComputeOp::Render { columns, template, .. } => format!( diff --git a/crates/plasm-agent-core/src/plasm_plan_run/materialize.rs b/crates/plasm-agent-core/src/plasm_plan_run/materialize.rs index 0c8fd4ab..6cb96922 100644 --- a/crates/plasm-agent-core/src/plasm_plan_run/materialize.rs +++ b/crates/plasm-agent-core/src/plasm_plan_run/materialize.rs @@ -485,16 +485,6 @@ pub(crate) async fn materialized_rows( .await } -pub(crate) fn compute_needs_full_materialize(op: &ComputeOp) -> bool { - matches!( - op, - ComputeOp::Sort { .. } - | ComputeOp::GroupBy { .. } - | ComputeOp::Aggregate { .. } - | ComputeOp::DedupeBy { .. } - ) -} - #[must_use] pub(crate) fn execution_result_from_fanout_fold( fold: super::plan_fanout_parallel::PlanLineExecutionFold, diff --git a/crates/plasm-agent-core/src/plasm_plan_run/mod.rs b/crates/plasm-agent-core/src/plasm_plan_run/mod.rs index 58733923..7e81864d 100644 --- a/crates/plasm-agent-core/src/plasm_plan_run/mod.rs +++ b/crates/plasm-agent-core/src/plasm_plan_run/mod.rs @@ -27,8 +27,8 @@ use crate::http_execute::{ use crate::plan_dry_display; pub use crate::plan_dry_display::PlanDryReview; use crate::plasm_plan::{ - AggregateFunction, BindingName, ComputeOp, ComputeTemplate, EffectClass, FieldPath, InputAlias, - Plan, PlanExprTemplate, PlanNodeId, PlanNodeKind, PlanResultUse, PlanValue, QualifiedEntityKey, + BindingName, ComputeOp, ComputeTemplate, EffectClass, InputAlias, Plan, PlanExprTemplate, + PlanNodeId, PlanNodeKind, PlanResultUse, PlanValue, QualifiedEntityKey, RelationSourceCardinality, ValidatedForEachNode, ValidatedPlan, ValidatedPlanDataInput, ValidatedPlanExprTemplate, ValidatedPlanNode, ValidatedPlanState, ValidatedRelationTraversalNode, PLAN_RENDER_MAX_OUTPUT_CHARS, PLAN_RENDER_MAX_ROWS, @@ -102,8 +102,7 @@ pub(crate) use parse::{ entry_scoped_execute_session, propagate_row_identities, row_identities_from_entities, }; pub(crate) use row_json::{ - cached_entity_row_json, predicate_matches, value_at_dotted, value_at_field_path, - value_at_segments, + cached_entity_row_json, predicate_matches, value_at_dotted, value_at_segments, }; #[cfg(test)] diff --git a/crates/plasm-agent-core/src/plasm_plan_run/orchestrator.rs b/crates/plasm-agent-core/src/plasm_plan_run/orchestrator.rs index bf72c95e..89742ad8 100644 --- a/crates/plasm-agent-core/src/plasm_plan_run/orchestrator.rs +++ b/crates/plasm-agent-core/src/plasm_plan_run/orchestrator.rs @@ -12,6 +12,7 @@ use plasm_core::plasm_monad::{PlasmStepPayload, StepId}; use plasm_core::PlasmReturn; use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; +use tracing::Instrument; #[allow(clippy::too_many_arguments)] pub async fn run_plasm_comp( @@ -57,6 +58,7 @@ pub async fn run_plasm_comp( execution_scope, mcp_result_policy, )) + .instrument(crate::spans::plan_live_run()) .await } @@ -313,7 +315,10 @@ pub(crate) async fn run_executable_plan_phased( let bind = Arc::clone(&bind); let rows_progress_step = rows_progress_parallel.clone(); let execution_scope_step = execution_scope_parallel.clone(); + let parent_span = tracing::Span::current(); joins.push(async move { + let step_span = + crate::spans::plan_step_materialize(&parent_span, step_id.as_str()); let mat_ctx = PlanStepMaterializeCtx { es: &es, st: &st, @@ -336,6 +341,7 @@ pub(crate) async fn run_executable_plan_phased( bind.as_ref(), &materialized_snap, )) + .instrument(step_span) .await }); } diff --git a/crates/plasm-agent-core/src/plasm_plan_run/parse.rs b/crates/plasm-agent-core/src/plasm_plan_run/parse.rs index 166fb741..bc894d25 100644 --- a/crates/plasm-agent-core/src/plasm_plan_run/parse.rs +++ b/crates/plasm-agent-core/src/plasm_plan_run/parse.rs @@ -452,6 +452,7 @@ pub(crate) fn propagate_row_identities( match op { ComputeOp::Limit { count } => Ok(mat.row_identities.iter().take(*count).cloned().collect()), ComputeOp::Project { .. } => Ok(mat.row_identities.iter().take(out_len).cloned().collect()), + ComputeOp::With { .. } => Ok(mat.row_identities.iter().take(out_len).cloned().collect()), ComputeOp::Filter { predicates } => { let Some(rows) = mat.row_source.inline_rows() else { return Ok(Vec::new()); diff --git a/crates/plasm-agent-core/src/plasm_plan_run/row_json.rs b/crates/plasm-agent-core/src/plasm_plan_run/row_json.rs index 7b1b72fd..aa72a8da 100644 --- a/crates/plasm-agent-core/src/plasm_plan_run/row_json.rs +++ b/crates/plasm-agent-core/src/plasm_plan_run/row_json.rs @@ -1,7 +1,6 @@ //! Row JSON helpers. use super::*; -use crate::plasm_plan::FieldPath; pub(crate) fn cached_entity_row_json(entity: &CachedEntity, cgs: &CGS) -> serde_json::Value { entity_to_row_json(entity, Some(cgs)) @@ -18,13 +17,6 @@ pub(crate) fn value_at_segments<'a>( Some(cur) } -pub(crate) fn value_at_field_path<'a>( - row: &'a serde_json::Value, - path: &FieldPath, -) -> Option<&'a serde_json::Value> { - value_at_segments(row, path.segments()) -} - pub(crate) fn value_at_dotted<'a>( row: &'a serde_json::Value, path: &str, diff --git a/crates/plasm-agent-core/src/plasm_plan_run/tests/dry_run.rs b/crates/plasm-agent-core/src/plasm_plan_run/tests/dry_run.rs index f1450a69..598b460f 100644 --- a/crates/plasm-agent-core/src/plasm_plan_run/tests/dry_run.rs +++ b/crates/plasm-agent-core/src/plasm_plan_run/tests/dry_run.rs @@ -6,23 +6,6 @@ use plasm_core::TeachingExposureSession; use std::path::PathBuf; use std::sync::Arc; -#[test] -fn cmp_json_sort_values_orders_multi_digit_numbers_numerically() { - use std::cmp::Ordering; - let n87 = serde_json::json!(87); - let n300 = serde_json::json!(300); - assert_eq!( - cmp_json_sort_values(Some(&n87), Some(&n300)), - Ordering::Less - ); - let s87 = serde_json::json!("87"); - let s300 = serde_json::json!("300"); - assert_eq!( - cmp_json_sort_values(Some(&s87), Some(&s300)), - Ordering::Less - ); -} - #[test] fn singleton_input_zero_row_error_is_actionable() { let err = singleton_input_row_count_error("src", "_", 0, "staged expression rendering"); @@ -39,40 +22,6 @@ fn singleton_input_multi_row_error_mentions_ambiguity_remedy() { assert!(err.contains(".singleton()"), "{err}"); } -#[test] -fn cmp_json_sort_values_string_collates_non_numeric_strings_lexically() { - use std::cmp::Ordering; - let apple = serde_json::json!("apple"); - let banana = serde_json::json!("banana"); - assert_eq!( - cmp_json_sort_values(Some(&apple), Some(&banana)), - Ordering::Less - ); -} - -/// Regression: `.sort(score)` must not stringify numbers and compare lexicographically (where -/// `87` sorts after `300`). Keeps parity with [`eval_compute`] `ComputeOp::Sort` staging. -#[test] -fn plan_sort_compute_orders_integer_scores_numerically() { - let key = FieldPath::from_dotted("score").expect("score path"); - let mut rows = [ - serde_json::json!({"id": "n300", "score": 300}), - serde_json::json!({"id": "n87", "score": 87}), - serde_json::json!({"id": "n100", "score": 100}), - ]; - rows.sort_by(|a, b| { - cmp_json_sort_values(value_at_field_path(a, &key), value_at_field_path(b, &key)) - }); - assert_eq!(rows[0]["id"], "n87"); - assert_eq!(rows[1]["id"], "n100"); - assert_eq!(rows[2]["id"], "n300"); - - rows.reverse(); - assert_eq!(rows[0]["id"], "n300"); - assert_eq!(rows[1]["id"], "n100"); - assert_eq!(rows[2]["id"], "n87"); -} - fn github_repository_commit_session() -> ExecuteSession { let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let cgs = Arc::new(load_schema(&root.join("../../apis/github")).expect("load github")); diff --git a/crates/plasm-agent-core/src/session_bindings.rs b/crates/plasm-agent-core/src/session_bindings.rs index a885648c..58910255 100644 --- a/crates/plasm-agent-core/src/session_bindings.rs +++ b/crates/plasm-agent-core/src/session_bindings.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::sync::Arc; +use tracing::Instrument; use auth_framework::storage::AuthStorage; @@ -46,16 +47,24 @@ pub async fn tenant_bindings_for_entries( let scope = BindingScope::new(cfg.tenant_id.clone(), cfg.id, eid.clone()); let storage = Arc::clone(&storage); let eid = eid.clone(); - futs.push(async move { - let values = binding_store::load_binding_values_scoped(&storage, repo, &scope).await?; - match values { - Some(vals) if crate::binding_slots::bindings_complete_for_entry(&eid, &vals) => { - Ok(Some((eid, SessionBindingMap::from_values(scope, vals)))) + let parent_span = tracing::Span::current(); + let bind_span = crate::spans::session_load_binding(&parent_span, eid.as_str()); + futs.push( + async move { + let values = + binding_store::load_binding_values_scoped(&storage, repo, &scope).await?; + match values { + Some(vals) + if crate::binding_slots::bindings_complete_for_entry(&eid, &vals) => + { + Ok(Some((eid, SessionBindingMap::from_values(scope, vals)))) + } + Some(_) => Err(BindingLoadError::Incomplete(eid)), + None => Err(BindingLoadError::NotConfigured(eid)), } - Some(_) => Err(BindingLoadError::Incomplete(eid)), - None => Err(BindingLoadError::NotConfigured(eid)), } - }); + .instrument(bind_span), + ); } let results = futures_util::future::join_all(futs).await; let mut out = HashMap::new(); diff --git a/crates/plasm-agent-core/src/span_graph_tests.rs b/crates/plasm-agent-core/src/span_graph_tests.rs new file mode 100644 index 00000000..410fd372 --- /dev/null +++ b/crates/plasm-agent-core/src/span_graph_tests.rs @@ -0,0 +1,3 @@ +//! Span-graph contracts for plasm-agent-core. +//! +//! Live call-site lock: `http_execute::routes::tests::http_request_parents_execute_run_post_on_handler`. diff --git a/crates/plasm-agent-core/src/spans.rs b/crates/plasm-agent-core/src/spans.rs index 2b9d8300..155007f1 100644 --- a/crates/plasm-agent-core/src/spans.rs +++ b/crates/plasm-agent-core/src/spans.rs @@ -231,6 +231,91 @@ pub fn security_incoming_http(principal: bool, tenant_id: &str) -> Span { ) } +// --- OAuth link (outbound hosted_kv) ----------------------------------------- + +#[inline] +pub(crate) fn oauth_link_start() -> Span { + tracing::info_span!("plasm_agent.oauth_link.start", oauth.phase = "start",) +} + +#[inline] +pub(crate) fn oauth_link_device_start() -> Span { + tracing::info_span!( + "plasm_agent.oauth_link.device_start", + oauth.phase = "device_start", + ) +} + +#[inline] +pub(crate) fn oauth_link_device_poll() -> Span { + tracing::info_span!( + "plasm_agent.oauth_link.device_poll", + oauth.phase = "device_poll", + ) +} + +#[inline] +pub(crate) fn oauth_link_token_exchange(entry_id: &str) -> Span { + tracing::info_span!( + "plasm_agent.oauth_link.token_exchange", + oauth.phase = "token_exchange", + entry_id = %entry_id, + ) +} + +/// CPU work offloaded onto the catalog/blocking compute pool. +#[inline] +pub(crate) fn blocking_compute(label: &'static str) -> Span { + tracing::debug_span!("plasm_agent.blocking_compute", label = label) +} + +// --- Plan dry / live / discover / MCP dispatch -------------------------------- + +#[inline] +pub(crate) fn plan_dry_run(source_len: usize) -> Span { + tracing::info_span!("plasm_agent.plan.dry_run", source_len = source_len) +} + +#[inline] +pub(crate) fn plan_live_run() -> Span { + tracing::info_span!("plasm_agent.plan.live_run") +} + +/// One parallel plan-step materialize arm (parent must be the request/plan span). +#[inline] +pub(crate) fn plan_step_materialize(parent: &Span, step_id: &str) -> Span { + tracing::debug_span!( + parent: parent, + "plasm_agent.plan.step_materialize", + step_id = %step_id, + ) +} + +/// One parallel session-binding load arm (parent must be the request/plan span). +#[inline] +pub(crate) fn session_load_binding(parent: &Span, entry_id: &str) -> Span { + tracing::debug_span!( + parent: parent, + "plasm_agent.session.load_binding", + entry_id = %entry_id, + ) +} + +#[inline] +pub(crate) fn execute_run_post() -> Span { + tracing::info_span!("plasm_agent.execute.run_post") +} + +#[inline] +pub(crate) fn discover_query() -> Span { + tracing::info_span!("plasm_agent.discover.query") +} + +#[inline] +pub(crate) fn mcp_call_tool(tool: &str) -> Span { + tracing::info_span!("plasm_agent.mcp.call_tool", tool = %tool) +} + // --- Billing / audit (trace sink envelope, durable rows) ---------------------- /// Building an audit batch for the trace sink (`mcp_trace_segment` rows; payload is `plasm_trace::TraceEvent` JSON). diff --git a/crates/plasm-agent-core/src/tenant_binding.rs b/crates/plasm-agent-core/src/tenant_binding.rs index 7d9ec80c..0f9d2c22 100644 --- a/crates/plasm-agent-core/src/tenant_binding.rs +++ b/crates/plasm-agent-core/src/tenant_binding.rs @@ -1,8 +1,9 @@ //! Postgres-backed mapping from incoming-auth `subject` (e.g. `github:`) to tenant + shell slugs. +use crate::traced_pg::PgPool; use sha2::{Digest, Sha256}; use sqlx::postgres::PgPoolOptions; -use sqlx::{PgPool, Row}; +use sqlx::Row; /// Same URL resolution as outbound OAuth provider pull (Phoenix `DATABASE_URL` in k8s). pub fn tenant_binding_database_url() -> Option { @@ -114,10 +115,11 @@ impl TenantBindingStore { .connect(database_url) .await?; Self::ensure_schema(&pool).await?; + let pool = crate::traced_pg::wrap(pool); Ok(Self { pool }) } - async fn ensure_schema(pool: &PgPool) -> Result<(), sqlx::Error> { + async fn ensure_schema(pool: &sqlx::PgPool) -> Result<(), sqlx::Error> { sqlx::query( r#" CREATE TABLE IF NOT EXISTS plasm_incoming_subject_binding ( diff --git a/crates/plasm-agent-core/src/trace_sink_emit.rs b/crates/plasm-agent-core/src/trace_sink_emit.rs index 7f0d923b..898d012e 100644 --- a/crates/plasm-agent-core/src/trace_sink_emit.rs +++ b/crates/plasm-agent-core/src/trace_sink_emit.rs @@ -121,7 +121,7 @@ impl TraceIngestClient for EnvTraceIngestClient { return; } }; - tokio::spawn(async move { post_events_json(base, body).await }.instrument(emit_span)); + tokio::spawn(post_events_json(base, body).instrument(emit_span)); } } diff --git a/crates/plasm-agent-core/src/traced_pg.rs b/crates/plasm-agent-core/src/traced_pg.rs new file mode 100644 index 00000000..6213a10a --- /dev/null +++ b/crates/plasm-agent-core/src/traced_pg.rs @@ -0,0 +1,15 @@ +//! OpenTelemetry-instrumented Postgres pool (`sqlx-tracing` full cutover). +//! +//! All durable sqlx repositories in this crate hold [`PgPool`] from this module, not raw +//! `sqlx::PgPool`. Construct via [`wrap`] after `PgPoolOptions::connect`. + +use sqlx::Postgres; + +/// Traced Postgres connection pool (queries become `sqlx.*` child spans). +pub type PgPool = sqlx_tracing::Pool; + +/// Wrap a raw sqlx pool so every executor call is instrumented. +#[inline] +pub fn wrap(pool: sqlx::PgPool) -> PgPool { + sqlx_tracing::Pool::from(pool) +} diff --git a/crates/plasm-agent-core/tests/mcp_readiness_pending_auth.rs b/crates/plasm-agent-core/tests/mcp_readiness_pending_auth.rs index 749936db..1e55ef1f 100644 --- a/crates/plasm-agent-core/tests/mcp_readiness_pending_auth.rs +++ b/crates/plasm-agent-core/tests/mcp_readiness_pending_auth.rs @@ -12,7 +12,7 @@ use plasm_agent_core::binding_store::entry_secret_present_for_upsert; use plasm_agent_core::mcp_config_readiness::catalog_entry_readiness_gaps; use plasm_agent_core::mcp_config_repository::McpConfigRepository; use plasm_agent_core::mcp_runtime_config::McpRuntimeConfig; -use sqlx::PgPool; +use plasm_agent_core::traced_pg::PgPool; use support::postgres::{integration_postgres_url, INTEGRATION_POSTGRES_URL_ENV}; use uuid::Uuid; diff --git a/crates/plasm-core/Cargo.toml b/crates/plasm-core/Cargo.toml index 0c867eed..c97d7b7d 100644 --- a/crates/plasm-core/Cargo.toml +++ b/crates/plasm-core/Cargo.toml @@ -9,6 +9,8 @@ readme = "README.md" default = ["ranked_capability_gate"] # Non-read capabilities also require membership in an optional sorted capability-name list when that list is non-empty. ranked_capability_gate = [] +# Subscriber for the `dump_prompt` binary only — library code must not own a global subscriber. +dump-prompt = ["dep:tracing-subscriber"] [dependencies] bm25 = { workspace = true } @@ -22,7 +24,7 @@ chrono = { workspace = true } chrono-english = { workspace = true } rust_decimal = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true, features = ["env-filter"] } +tracing-subscriber = { workspace = true, features = ["env-filter"], optional = true } sha2 = { workspace = true } hex = { workspace = true } base64 = { workspace = true } @@ -30,13 +32,21 @@ rustc-hash = "2" riptoken = "0.3.0" rayon = { workspace = true } minijinja = { version = "2.19.0", default-features = false, features = ["builtins", "serde"] } + [dev-dependencies] +plasm-otel = { path = "../plasm-otel", features = ["testing"] } insta = { workspace = true } proptest = { workspace = true } tempfile = { workspace = true } criterion = { workspace = true } plasm-discovery = { path = "../plasm-discovery" } plasm-discovery-eval = { path = "../plasm-discovery-eval" } +tracing-subscriber = { workspace = true, features = ["env-filter"] } + +[[bin]] +name = "dump_prompt" +path = "src/bin/dump_prompt.rs" +required-features = ["dump-prompt"] [[bench]] name = "schema_load" diff --git a/crates/plasm-core/src/catalog_il.rs b/crates/plasm-core/src/catalog_il.rs index 95736452..f228b64b 100644 --- a/crates/plasm-core/src/catalog_il.rs +++ b/crates/plasm-core/src/catalog_il.rs @@ -69,6 +69,8 @@ pub fn cgs_to_catalog_il_bytes(cgs: &CGS) -> Result, String> { /// Decode compiled JSON IL bytes into a CGS and run full validation. pub fn load_catalog_il_bytes(bytes: &[u8]) -> Result { + let span = crate::spans::catalog_load_il(bytes.len()); + let _guard = span.enter(); let cgs: CGS = serde_json::from_slice(bytes).map_err(|e| format!("CGS JSON decode failed: {e}"))?; cgs.validate() diff --git a/crates/plasm-core/src/discovery/mod.rs b/crates/plasm-core/src/discovery/mod.rs index 074cfabf..803bc71c 100644 --- a/crates/plasm-core/src/discovery/mod.rs +++ b/crates/plasm-core/src/discovery/mod.rs @@ -829,6 +829,8 @@ fn cap_passes_filters(query: &CapabilityQuery, entry_id: &str, cap: &CapabilityS impl CgsDiscovery for InMemoryCgsRegistry { fn discover(&self, query: &CapabilityQuery) -> Result { + let span = crate::spans::discovery_discover(); + let _guard = span.enter(); let query_text = collect_query_text(query); let query_tokens = collect_query_tokens(query); let has_explicit_expand = query @@ -1036,6 +1038,9 @@ impl CgsDiscovery for InMemoryCgsRegistry { "cgs discovery completed" ); + tracing::Span::current().record("candidate_count", candidates.len()); + tracing::Span::current().record("result_count", candidates.len()); + let catalog_route = catalog_route .as_ref() .map(|set| { diff --git a/crates/plasm-core/src/expr_parser/mod.rs b/crates/plasm-core/src/expr_parser/mod.rs index 32c24c14..3118cb93 100644 --- a/crates/plasm-core/src/expr_parser/mod.rs +++ b/crates/plasm-core/src/expr_parser/mod.rs @@ -424,6 +424,8 @@ pub fn parse_with_remainder( input: &str, cgs: &CGS, ) -> Result<(ParsedExpr, ParseRemainder), ParseError> { + let span = crate::spans::parse_program(input.len()); + let _guard = span.enter(); let mut p = Parser::new(input, cgs); let mut parsed = p.parse_expr()?; parsed.expr = crate::expr_sugar::rewrite_id_field_brace_query_to_get(parsed.expr, cgs); @@ -440,6 +442,8 @@ pub fn parse_with_remainder( /// Trailing text (after whitespace) is **ignored** so callers can paste noisy LLM output /// without failing the whole line. pub fn parse(input: &str, cgs: &CGS) -> Result { + let span = crate::spans::parse_program(input.len()); + let _guard = span.enter(); let mut p = Parser::new(input, cgs); let mut parsed = p.parse_expr()?; parsed.expr = crate::expr_sugar::rewrite_id_field_brace_query_to_get(parsed.expr, cgs); @@ -522,6 +526,8 @@ fn parse_with_cgs_layers_program_opts( for_each_row_context: bool, apply_id_field_get_rewrite: bool, ) -> Result { + let span = crate::spans::parse_program(input.len()); + let _guard = span.enter(); if layers.is_empty() { return Err(ParseError { kind: ParseErrorKind::Other { diff --git a/crates/plasm-core/src/expr_parser/postfix.rs b/crates/plasm-core/src/expr_parser/postfix.rs index f37d084e..3956cb10 100644 --- a/crates/plasm-core/src/expr_parser/postfix.rs +++ b/crates/plasm-core/src/expr_parser/postfix.rs @@ -22,6 +22,7 @@ pub enum PlasmPostfixOp { GroupBy { args: String }, Dedupe { keys: String }, Distinct { keys: Option }, + With { body: String }, Projection { fields: String }, } @@ -325,6 +326,14 @@ pub fn peel_postfix_suffixes(rhs: &str) -> Result<(String, Vec), }); cur = p; progressed = true; + } else if let Some((p, body)) = strip_trailing_brace_block(t, "with")? { + ops_rev.push(PlasmPostfixOp::With { body }); + cur = p; + progressed = true; + } else if let Some((p, body)) = strip_trailing_method_call(t, "with")? { + ops_rev.push(PlasmPostfixOp::With { body }); + cur = p; + progressed = true; } else if let Some((p, fields)) = strip_trailing_projection(t)? { ops_rev.push(PlasmPostfixOp::Projection { fields }); cur = p; @@ -684,6 +693,28 @@ mod tests { assert_eq!(ops3, vec![PlasmPostfixOp::Distinct { keys: None }]); } + #[test] + fn peel_with_brace_body() { + let (p, ops) = peel_postfix_suffixes("issues.with{age_days: (now - updated_at)}").unwrap(); + assert_eq!(p, "issues"); + assert_eq!( + ops, + vec![PlasmPostfixOp::With { + body: "age_days: (now - updated_at)".into() + }] + ); + } + + #[test] + fn peel_does_not_treat_join_or_open_as_row_compute() { + let (p, ops) = peel_postfix_suffixes("issues.join(comments)").unwrap(); + assert!(ops.is_empty(), "join is not a postfix verb, got {ops:?}"); + assert!(p.contains("join")); + let (p2, ops2) = peel_postfix_suffixes("issues.open(labels)").unwrap(); + assert!(ops2.is_empty(), "open is not a postfix verb, got {ops2:?}"); + assert!(p2.contains("open")); + } + #[test] fn render_tail_cross_binding_labels_before_heredoc() { let r = try_parse_render_tail("pika,repos < Result { /// Run post-assemble normalization, validation, and string-semantics checks. pub fn finalize_cgs_load(cgs: &mut CGS) -> Result<(), String> { + let span = crate::spans::schema_validate(cgs.entities.len(), cgs.capabilities.len()); + let _guard = span.enter(); let legacy_via_param = std::mem::take(&mut cgs.pending_legacy_via_param_patches); cgs.normalize_relation_materialization(&legacy_via_param); debug!( @@ -452,7 +454,7 @@ pub fn finalize_cgs_load(cgs: &mut CGS) -> Result<(), String> { let sem_violations = cgs.string_semantics_violations(); if !sem_violations.is_empty() { for msg in &sem_violations { - error!(target: "plasm_core::cgs", "{}", msg); + error!(target: "plasm_core::cgs", violation = %msg, "string_semantics violation"); } return Err(format!( "CGS load requires string_semantics on every string field and string capability parameter ({} issue(s); first: {})", @@ -1048,7 +1050,7 @@ fn normalize_blob_field_type( /// without a `data_class` (plan-flow cannot label that data). fn warn_unlabeled_output_data(cgs: &CGS) { for msg in cgs.unlabeled_output_data_warnings() { - warn!(target: "plasm_core::loader", "{msg}"); + warn!(target: "plasm_core::loader", violation = %msg, "unlabeled output data"); } } diff --git a/crates/plasm-core/src/plasm_monad/mod.rs b/crates/plasm-core/src/plasm_monad/mod.rs index ed0d5d9e..8afb49db 100644 --- a/crates/plasm-core/src/plasm_monad/mod.rs +++ b/crates/plasm-core/src/plasm_monad/mod.rs @@ -21,12 +21,13 @@ pub use operators::{ plasm_parallel_return, plasm_pure_step, }; pub use payload::{ - AggregateFunction, AggregateSpec, BindingName, ComputeOp, ComputeTemplate, DeriveKind, + AggregateFunction, AggregateSpec, ArithOp, BindingName, ComputeOp, ComputeTemplate, DeriveKind, DerivePayload, DeriveTemplate, EffectTemplate, FieldPath, FlatMapEffectPayload, FlatMapRelationPayload, InputCardinality, InvokePayload, MapPayload, OutputName, PlanDataInput, PlanExprIr, PlanExprTemplate, PlanInputBinding, PlanPredicate, PlanPredicateOp, PlanQualifiedEntityKey, PlanRelationTraversal, PlanResultUse, PlasmDataValue, PlasmStepPayload, PurePayload, RelationCardinality, RelationName, RelationSourceCardinality, - SyntheticFieldSchema, SyntheticResultSchema, SyntheticValueKind, + SyntheticFieldSchema, SyntheticResultSchema, SyntheticValueKind, WithColumn, WithExpr, + WithExprError, WithLiteral, }; pub use step::{EffectBarrier, EffectClass, PlasmStep, PlasmStepKind, ResultShape, SurfaceKind}; diff --git a/crates/plasm-core/src/plasm_monad/payload/compute.rs b/crates/plasm-core/src/plasm_monad/payload/compute.rs index 3e33763c..fa4891d7 100644 --- a/crates/plasm-core/src/plasm_monad/payload/compute.rs +++ b/crates/plasm-core/src/plasm_monad/payload/compute.rs @@ -1,5 +1,6 @@ use super::atoms::{FieldPath, OutputName}; use super::value::PlanPredicate; +use super::with_expr::WithColumn; use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -47,6 +48,9 @@ pub enum ComputeOp { #[serde(default, skip_serializing_if = "Vec::is_empty")] keys: Vec, }, + With { + columns: Vec, + }, Render { columns: Vec, template: String, @@ -109,6 +113,10 @@ pub enum SyntheticValueKind { String, Array, Object, + Money, + Temporal, + EntityRef, + Duration, Unknown, } diff --git a/crates/plasm-core/src/plasm_monad/payload/mod.rs b/crates/plasm-core/src/plasm_monad/payload/mod.rs index d88b6a04..1a52b871 100644 --- a/crates/plasm-core/src/plasm_monad/payload/mod.rs +++ b/crates/plasm-core/src/plasm_monad/payload/mod.rs @@ -5,6 +5,7 @@ mod relation; mod step_payload; mod templates; mod value; +mod with_expr; pub use crate::identity::RelationName; pub use atoms::{BindingName, FieldPath, OutputName, PlanQualifiedEntityKey}; @@ -23,3 +24,4 @@ pub use value::{ InputCardinality, PlanDataInput, PlanInputBinding, PlanPredicate, PlanPredicateOp, PlanResultUse, PlasmDataValue, }; +pub use with_expr::{ArithOp, WithColumn, WithExpr, WithExprError, WithLiteral}; diff --git a/crates/plasm-core/src/plasm_monad/payload/step_payload.rs b/crates/plasm-core/src/plasm_monad/payload/step_payload.rs index 39963513..7c1d8dbb 100644 --- a/crates/plasm-core/src/plasm_monad/payload/step_payload.rs +++ b/crates/plasm-core/src/plasm_monad/payload/step_payload.rs @@ -168,6 +168,7 @@ fn compute_op_label(op: &super::compute::ComputeOp) -> String { ComputeOp::Sort { .. } => "sort".into(), ComputeOp::Limit { count } => format!("limit {count}"), ComputeOp::DedupeBy { .. } => "dedupe_by".into(), + ComputeOp::With { .. } => "with".into(), ComputeOp::Render { .. } => "render".into(), } } diff --git a/crates/plasm-core/src/plasm_monad/payload/with_expr.rs b/crates/plasm-core/src/plasm_monad/payload/with_expr.rs new file mode 100644 index 00000000..a4b60460 --- /dev/null +++ b/crates/plasm-core/src/plasm_monad/payload/with_expr.rs @@ -0,0 +1,65 @@ +//! `.with` expression AST stored on hashed [`super::ComputeOp::With`]. + +use super::atoms::{FieldPath, OutputName}; +use super::value::PlanPredicateOp; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct WithColumn { + pub name: OutputName, + pub expr: WithExpr, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WithExpr { + Field(FieldPath), + Literal(WithLiteral), + Arith { + op: ArithOp, + lhs: Box, + rhs: Box, + }, + /// Catalog-plane clock token (`now` → UTC). A catalog field named `now` loses. + Now, + Len { + field: FieldPath, + }, + When { + lhs: Box, + op: PlanPredicateOp, + rhs: Box, + then: Box, + else_: Box, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ArithOp { + Add, + Sub, + Mul, + Div, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WithLiteral { + Null, + Bool(bool), + Integer(i64), + Number(String), + String(String), +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum WithExprError { + #[error("empty .with body")] + EmptyBody, + #[error("invalid .with column `{0}`")] + BadColumn(String), + #[error("invalid .with expression: {0}")] + Parse(String), +} diff --git a/crates/plasm-core/src/prompt_render/assets/plasm_tool.txt b/crates/plasm-core/src/prompt_render/assets/plasm_tool.txt index b7b2dbe4..f4d84b30 100644 --- a/crates/plasm-core/src/prompt_render/assets/plasm_tool.txt +++ b/crates/plasm-core/src/prompt_render/assets/plasm_tool.txt @@ -41,7 +41,7 @@ TSV table semantics: Core surface: - Get identity: `e#(id)` (parens). Query/filter: `e#{field=…}` (braces). Search when taught: `e#~$` / `e#~"text"`. -- Postfix from TSV left column: `.filter{…}` `.sort` `.limit` `.group_by` `.aggregate` `[field,…]`. +- Postfix from TSV left column: `.filter{…}` `.sort` `.limit` `.group_by` `.aggregate` `.with{k: expr}` `[field,…]`. - Inline rows when delivery is `inline` (≤25); `snapshot_only` / `(in artifact)` need artifact read. Copy **`artifact_uri`** from the step — never plan `run_step` / `dict_ref`. - `page(...)` is HTTP-execute only — not an MCP tool argument. diff --git a/crates/plasm-core/src/prompt_render/assets/program_param.txt b/crates/plasm-core/src/prompt_render/assets/program_param.txt index 9be72c1a..3edd0eb8 100644 --- a/crates/plasm-core/src/prompt_render/assets/program_param.txt +++ b/crates/plasm-core/src/prompt_render/assets/program_param.txt @@ -5,7 +5,7 @@ Symbols from the `plasm_context` teaching TSV: `e#` entity, `m#` method, `r#` re Shape: - Get by identity: `e#(id)` (parens). Query/filter: `e#{field="val"}` (braces). Relation nav: `producer.r#`. - Search when taught: `e#~"text"` or `e#~$` (bare `e#~` is a parse error). -- Postfix from the TSV left column: `.filter{…}` `.sort(field,desc)` `.limit(n)` `.group_by` `.aggregate`; `[field,…]` projects. +- Postfix from the TSV left column: `.filter{…}` `.sort(field,desc)` `.limit(n)` `.group_by` `.aggregate` `.with{k: expr}` `.dedupe(field)` `.distinct`; `[field,…]` projects. - Write: `source => e#.m#(param=…)`; create: `e#.m#(param=…)`. Prior write **`provides`** fields usable as later args; `[field,…]` on creates projects them. - After `repo = e#(…)`, prefer **`repo.m#(…)`** / **`repo.r#`**. - Multi-write / batch-read shape: see the `plasm` tool description. diff --git a/crates/plasm-core/src/row_composition.rs b/crates/plasm-core/src/row_composition.rs index 45babdc2..b4795fa2 100644 --- a/crates/plasm-core/src/row_composition.rs +++ b/crates/plasm-core/src/row_composition.rs @@ -71,6 +71,7 @@ pub enum RowSuffix { GroupBy { args: String }, Dedupe { keys: String }, Distinct { keys: Option }, + With { body: String }, Singleton, PageSize { n: u32 }, } @@ -92,6 +93,7 @@ impl RowSuffix { PlasmPostfixOp::GroupBy { args } => Ok(Self::GroupBy { args: args.clone() }), PlasmPostfixOp::Dedupe { keys } => Ok(Self::Dedupe { keys: keys.clone() }), PlasmPostfixOp::Distinct { keys } => Ok(Self::Distinct { keys: keys.clone() }), + PlasmPostfixOp::With { body } => Ok(Self::With { body: body.clone() }), PlasmPostfixOp::Singleton => Ok(Self::Singleton), PlasmPostfixOp::PageSize(n) => Ok(Self::PageSize { n: *n as u32 }), } diff --git a/crates/plasm-core/src/row_plan/collect.rs b/crates/plasm-core/src/row_plan/collect.rs new file mode 100644 index 00000000..4413bf2b --- /dev/null +++ b/crates/plasm-core/src/row_plan/collect.rs @@ -0,0 +1,64 @@ +//! Collect barriers — the only legal materialize points. + +use crate::plasm_monad::{OutputName, StepId}; +use serde::{Deserialize, Serialize}; +use std::num::NonZeroUsize; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CollectReason { + ProgramReturn { + step: StepId, + }, + PageContinue { + step: StepId, + page: PageCursor, + }, + InvokeArg { + consumer: StepId, + hole: String, + }, + Render { + step: StepId, + spec: RenderCollectSpec, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RenderCollectSpec { + pub columns: Vec, + pub column_aliases: std::collections::BTreeMap, + pub template: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub collection_alias: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub render_bindings: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PageCursor { + pub token: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CollectCardinality { + List, + Single, + Page { size: PageSize }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct PageSize(NonZeroUsize); + +impl PageSize { + pub fn new(n: usize) -> Option { + NonZeroUsize::new(n).map(Self) + } + + #[must_use] + pub fn get(self) -> usize { + self.0.get() + } +} diff --git a/crates/plasm-core/src/row_plan/engine.rs b/crates/plasm-core/src/row_plan/engine.rs new file mode 100644 index 00000000..c1098e3b --- /dev/null +++ b/crates/plasm-core/src/row_plan/engine.rs @@ -0,0 +1,80 @@ +//! Engine ports. Implementations live in `plasm-runtime`. No `polars` types here. + +use crate::plasm_monad::StepId; +use crate::value::Value; +use indexmap::IndexMap; + +use super::collect::CollectReason; +use super::error::RowComputeError; +use super::ids::{EnginePlanId, FixtureScanId, FrameId, GraphSnapshotId}; +use super::plan::RowPlan; +use super::schema::PlasmFrameSchema; +use crate::identity::EntityName; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ScanSource { + Fixture { + id: FixtureScanId, + schema: PlasmFrameSchema, + }, + Inline { + schema: PlasmFrameSchema, + }, + Graph { + entity: EntityName, + snapshot: GraphSnapshotId, + schema: PlasmFrameSchema, + }, +} + +pub struct IngestBatch<'a> { + pub rows: &'a [IndexMap], +} + +#[derive(Debug, Clone, PartialEq)] +pub struct CollectedFrame { + pub schema: PlasmFrameSchema, + pub rows: Vec>, +} + +pub trait IngestRows { + fn ingest( + &mut self, + source: &ScanSource, + batch: IngestBatch<'_>, + ) -> Result; +} + +pub trait CompileRowPlan { + fn compile(&self, plan: &RowPlan) -> Result; +} + +pub trait CollectRows { + fn collect( + &self, + id: EnginePlanId, + reason: CollectReason, + ) -> Result; +} + +/// Convenience bound for the single phase-1 adapter (not object-safe). +pub trait RowComputeEngine: IngestRows + CompileRowPlan + CollectRows {} + +impl RowComputeEngine for T where T: IngestRows + CompileRowPlan + CollectRows {} + +impl CollectedFrame { + #[must_use] + pub fn empty(schema: PlasmFrameSchema) -> Self { + Self { + schema, + rows: Vec::new(), + } + } +} + +impl CollectReason { + #[must_use] + pub fn program_return(step: StepId) -> Self { + Self::ProgramReturn { step } + } +} diff --git a/crates/plasm-core/src/row_plan/error.rs b/crates/plasm-core/src/row_plan/error.rs new file mode 100644 index 00000000..eb55db29 --- /dev/null +++ b/crates/plasm-core/src/row_plan/error.rs @@ -0,0 +1,125 @@ +//! Typed row-compute errors — no stringly engine failures. + +use crate::identity::EntityName; +use crate::money::{CrossCurrencyError, MoneyError}; +use crate::plasm_monad::ArithOp; +use thiserror::Error; + +use super::schema::LogicalColumnType; + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum RowComputeError { + #[error(transparent)] + Type(#[from] RowTypeError), + #[error(transparent)] + Money(#[from] MoneyError), + #[error("cannot compare money in {left} to money in {right}")] + CrossCurrency { left: String, right: String }, + #[error(transparent)] + Schema(#[from] FrameSchemaError), + #[error(transparent)] + Collect(#[from] CollectError), + #[error(transparent)] + Expr(#[from] crate::plasm_monad::WithExprError), + #[error(transparent)] + Predicate(#[from] RowFilterError), + #[error(transparent)] + Scan(#[from] ScanError), + #[error(transparent)] + Fusion(#[from] FusionError), +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum RowTypeError { + #[error("arithmetic `{op:?}` is not defined for {lhs:?} and {rhs:?}")] + ArithDomain { + op: ArithOp, + lhs: LogicalColumnType, + rhs: LogicalColumnType, + }, + #[error("when() branches have mismatched types {then:?} vs {else_:?}")] + WhenBranchMismatch { + then: LogicalColumnType, + else_: LogicalColumnType, + }, + #[error("temporal arithmetic requires a temporal value, got {got:?}")] + TemporalArithNotTemporal { got: LogicalColumnType }, + #[error("money must not be stored as Utf8")] + MoneyStoredAsUtf8, + #[error("project spec cannot be used as a .with column")] + ProjectIntoWith, + #[error(".with must preserve entity identity")] + WithBreaksEntityShape, +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum FrameSchemaError { + #[error("unknown column `{0}`")] + UnknownColumn(String), + #[error("empty pipeline is illegal")] + EmptyPipeline, + #[error("limit count must be non-zero")] + ZeroLimit, + #[error("group_by requires at least one key")] + EmptyGroupKeys, + #[error("with requires at least one column")] + EmptyWith, +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum CollectError { + #[error("collect is only legal at a program-return, page, invoke-arg, or render barrier")] + CollectNotAtBarrier, + #[error("render row cap exceeded: got {got}, max {max}")] + RenderRowCap { got: usize, max: usize }, + #[error("silent page exhaust is forbidden")] + PageExhaustSilent, +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum RowFilterError { + #[error("row filter requires at least one predicate")] + Empty, + #[error("row filter cannot be rewritten as a catalog filter")] + CrossPlanePushdown, +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum ScanError { + #[error("unbound frame")] + UnboundFrame, + #[error("fixture scan `{0}` is not loaded")] + MissingFixture(u64), + #[error("entity `{0}` is not in the graph snapshot")] + MissingGraphEntity(EntityName), +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum FusionError { + #[error("sort and limit must not be commuted")] + CommuteSortLimit, + #[error("optimizer must not rewrite row filters into catalog filters")] + CrossPlanePushdown, + #[error("join cannot be constructed from the surface")] + JoinFromSurface, + #[error("render is a collect barrier, not a pipeline node")] + RenderInPipeline, + #[error("derive remap cannot fold into a row-compute pipeline")] + DeriveInPipeline, +} + +impl From for RowComputeError { + fn from(e: CrossCurrencyError) -> Self { + Self::CrossCurrency { + left: e.left().to_string(), + right: e.right().to_string(), + } + } +} + +impl RowComputeError { + #[must_use] + pub fn temporal_arith_not_temporal(got: LogicalColumnType) -> Self { + Self::Type(RowTypeError::TemporalArithNotTemporal { got }) + } +} diff --git a/crates/plasm-core/src/row_plan/expr.rs b/crates/plasm-core/src/row_plan/expr.rs new file mode 100644 index 00000000..734eb54e --- /dev/null +++ b/crates/plasm-core/src/row_plan/expr.rs @@ -0,0 +1,11 @@ +//! Projection spec — distinct from `.with` columns. + +use crate::plasm_monad::payload::{FieldPath, OutputName}; +use serde::{Deserialize, Serialize}; + +pub use crate::plasm_monad::{ArithOp, WithColumn, WithExpr, WithExprError, WithLiteral}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProjectSpec { + pub fields: std::collections::BTreeMap, +} diff --git a/crates/plasm-core/src/row_plan/filter.rs b/crates/plasm-core/src/row_plan/filter.rs new file mode 100644 index 00000000..5ed542f9 --- /dev/null +++ b/crates/plasm-core/src/row_plan/filter.rs @@ -0,0 +1,50 @@ +//! Catalog vs row filter newtypes — cannot be substituted. + +use crate::plasm_monad::payload::PlanPredicate; +use serde::{Deserialize, Serialize}; + +use super::error::RowFilterError; + +/// Fetch-plane predicates (`e1{…}`). Not a row-compute input. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CatalogFilter(Vec); + +impl CatalogFilter { + #[must_use] + pub fn new(predicates: Vec) -> Self { + Self(predicates) + } + + #[must_use] + pub fn predicates(&self) -> &[PlanPredicate] { + &self.0 + } +} + +/// Row-plane AND-filter. No conversion to [`CatalogFilter`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RowFilter { + predicates: Vec, +} + +impl RowFilter { + pub fn new(predicates: Vec) -> Result { + if predicates.is_empty() { + return Err(RowFilterError::Empty); + } + Ok(Self { predicates }) + } + + #[must_use] + pub fn predicates(&self) -> &[PlanPredicate] { + &self.predicates + } +} + +impl TryFrom> for RowFilter { + type Error = RowFilterError; + + fn try_from(predicates: Vec) -> Result { + Self::new(predicates) + } +} diff --git a/crates/plasm-core/src/row_plan/fold.rs b/crates/plasm-core/src/row_plan/fold.rs new file mode 100644 index 00000000..9126d809 --- /dev/null +++ b/crates/plasm-core/src/row_plan/fold.rs @@ -0,0 +1,109 @@ +//! Fold hashed `ComputeOp` constructors into a fused [`RowPlan`]. + +use crate::plasm_monad::{ComputeOp, StepId}; + +use super::collect::{CollectCardinality, CollectReason, RenderCollectSpec}; +use super::error::{FrameSchemaError, FusionError, RowComputeError}; +use super::expr::ProjectSpec; +use super::filter::RowFilter; +use super::ids::{FrameId, RowNodeId, SurfaceMeaningId}; +use super::plan::{Pipeline, PlanNode, RowPlan, TypedAggregate}; +use std::num::NonZeroUsize; + +/// Fold a linear Map-spine `ComputeOp` chain. `Render` is a collect barrier, not a node. +pub fn fold_compute_ops( + ops: &[ComputeOp], + source: FrameId, + step: StepId, + cardinality: CollectCardinality, +) -> Result { + let meaning = SurfaceMeaningId::from_bytes( + &serde_json::to_vec(ops).unwrap_or_else(|_| ops.len().to_le_bytes().to_vec()), + ); + let mut pipeline = Pipeline::new(); + let mut collect = CollectReason::ProgramReturn { step: step.clone() }; + for (i, op) in ops.iter().enumerate() { + let id = RowNodeId::new(i as u64 + 1); + match op { + ComputeOp::Render { + columns, + template, + column_aliases, + render_bindings, + } => { + if i + 1 != ops.len() { + return Err(FusionError::RenderInPipeline.into()); + } + collect = CollectReason::Render { + step, + spec: RenderCollectSpec { + columns: columns.clone(), + column_aliases: column_aliases.clone(), + template: template.clone(), + collection_alias: None, + render_bindings: render_bindings.clone(), + }, + }; + break; + } + other => pipeline.push(id, plan_node_from_compute(other)?)?, + } + } + Ok(RowPlan::new( + source, + pipeline, + collect, + cardinality, + meaning, + )?) +} + +pub fn plan_node_from_compute(op: &ComputeOp) -> Result { + match op { + ComputeOp::Filter { predicates } => { + let filter = RowFilter::new(predicates.clone())?; + Ok(PlanNode::Filter(filter)) + } + ComputeOp::Sort { key, descending } => Ok(PlanNode::Sort { + key: key.clone(), + descending: *descending, + }), + ComputeOp::Limit { count } => { + let count = NonZeroUsize::new(*count).ok_or(FrameSchemaError::ZeroLimit)?; + Ok(PlanNode::Limit { count }) + } + ComputeOp::DedupeBy { keys } => Ok(PlanNode::Dedupe { keys: keys.clone() }), + ComputeOp::Project { fields } => Ok(PlanNode::Project(ProjectSpec { + fields: fields.clone(), + })), + ComputeOp::With { columns } => { + if columns.is_empty() { + return Err(FrameSchemaError::EmptyWith.into()); + } + Ok(PlanNode::With { + columns: columns.clone(), + }) + } + ComputeOp::GroupBy { keys, aggregates } => { + if keys.is_empty() { + return Err(FrameSchemaError::EmptyGroupKeys.into()); + } + let aggs = aggregates + .iter() + .map(TypedAggregate::from_spec) + .collect::, _>>()?; + Ok(PlanNode::GroupBy { + keys: keys.clone(), + aggs, + }) + } + ComputeOp::Aggregate { aggregates } => { + let aggs = aggregates + .iter() + .map(TypedAggregate::from_spec) + .collect::, _>>()?; + Ok(PlanNode::Aggregate { aggs }) + } + ComputeOp::Render { .. } => Err(FusionError::RenderInPipeline.into()), + } +} diff --git a/crates/plasm-core/src/row_plan/ids.rs b/crates/plasm-core/src/row_plan/ids.rs new file mode 100644 index 00000000..256e2001 --- /dev/null +++ b/crates/plasm-core/src/row_plan/ids.rs @@ -0,0 +1,67 @@ +//! Opaque identifiers for frames, fused nodes, and engine handles. +//! +//! None of these appear on hashed [`crate::PlasmComp`]. + +use serde::{Deserialize, Serialize}; + +macro_rules! u64_id { + ($(#[$meta:meta])* $name:ident) => { + $(#[$meta])* + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] + #[serde(transparent)] + pub struct $name(u64); + + impl $name { + #[must_use] + pub const fn new(raw: u64) -> Self { + Self(raw) + } + + #[must_use] + pub const fn as_u64(self) -> u64 { + self.0 + } + } + }; +} + +u64_id! { + /// Session-local ingested frame. + FrameId +} + +u64_id! { + /// Node inside a fused [`super::RowPlan`] pipeline. + RowNodeId +} + +u64_id! { + /// Adapter-private compiled plan handle. Never stored on `PlasmComp`. + EnginePlanId +} + +u64_id! { + /// Language-matrix / unit-test scan. + FixtureScanId +} + +u64_id! { + /// Graph-backed scan (hot snapshot identity). + GraphSnapshotId +} + +u64_id! { + /// Hash of the surface `ComputeOp` chain (written order), not the fused engine plan. + SurfaceMeaningId +} + +impl SurfaceMeaningId { + #[must_use] + pub fn from_bytes(bytes: &[u8]) -> Self { + use sha2::{Digest, Sha256}; + let digest = Sha256::digest(bytes); + let mut raw = [0u8; 8]; + raw.copy_from_slice(&digest[..8]); + Self(u64::from_be_bytes(raw)) + } +} diff --git a/crates/plasm-core/src/row_plan/mod.rs b/crates/plasm-core/src/row_plan/mod.rs new file mode 100644 index 00000000..8d3b68ac --- /dev/null +++ b/crates/plasm-core/src/row_plan/mod.rs @@ -0,0 +1,207 @@ +//! Fused row-compute IR and engine ports. +//! +//! [`ComputeOp`](crate::ComputeOp) remains the hashed PlasmComp constructor. This module is the +//! execute-time IR. Polars types do not appear here. + +mod collect; +mod engine; +mod error; +mod expr; +mod filter; +mod fold; +mod ids; +mod plan; +mod schema; +mod with_parse; + +pub use collect::{CollectCardinality, CollectReason, PageCursor, PageSize, RenderCollectSpec}; +pub use engine::{ + CollectRows, CollectedFrame, CompileRowPlan, IngestBatch, IngestRows, RowComputeEngine, + ScanSource, +}; +pub use error::{ + CollectError, FrameSchemaError, FusionError, RowComputeError, RowFilterError, RowTypeError, + ScanError, +}; +pub use expr::{ArithOp, ProjectSpec, WithColumn, WithExpr, WithExprError, WithLiteral}; +pub use filter::{CatalogFilter, RowFilter}; +pub use fold::{fold_compute_ops, plan_node_from_compute}; +pub use ids::{EnginePlanId, FixtureScanId, FrameId, GraphSnapshotId, RowNodeId, SurfaceMeaningId}; +pub use plan::{MoneyAggLaw, NumericAgg, Pipeline, PlanNode, RowPlan, TypedAggregate}; +pub use schema::{ + ColumnName, FrameShape, IdentityPreservation, LogicalColumn, LogicalColumnType, + MoneyColumnLayout, PlasmFrameSchema, RemapReason, +}; +pub use with_parse::parse_with_body; + +#[cfg(test)] +mod tests { + use super::*; + use crate::plasm_monad::payload::PlasmDataValue; + use crate::plasm_monad::{ComputeOp, FieldPath, OutputName, PlanPredicate, PlanPredicateOp}; + + #[test] + fn plan_node_has_no_render_or_join_variants() { + let names: Vec<&str> = vec![ + "Filter", + "Sort", + "Limit", + "Dedupe", + "Distinct", + "Project", + "With", + "GroupBy", + "Aggregate", + ]; + assert!(!names.contains(&"Render")); + assert!(!names.contains(&"EquiJoin")); + assert!(!names.contains(&"Join")); + } + + #[test] + fn with_parse_now_minus_and_mul() { + let cols = parse_with_body("age_days: (now - updated_at), notional: quantity * price") + .expect("parse"); + assert_eq!(cols.len(), 2); + match &cols[0].expr { + WithExpr::Arith { + op: ArithOp::Sub, + lhs, + rhs, + } => { + assert!(matches!(lhs.as_ref(), WithExpr::Now)); + assert!(matches!(rhs.as_ref(), WithExpr::Field(_))); + } + other => panic!("expected now - field, got {other:?}"), + } + assert!(matches!( + cols[1].expr, + WithExpr::Arith { + op: ArithOp::Mul, + .. + } + )); + } + + #[test] + fn with_parse_div_concat_when_and_field_minus_field() { + let cols = parse_with_body( + "cycle: (updated_at - created_at), rate: qty / n, name: first + last, blank: when(len(title)=0, 1, 0), stale: when(now - updated_at > 14, 1, 0)", + ) + .expect("parse"); + assert_eq!(cols.len(), 5); + assert!(matches!( + &cols[0].expr, + WithExpr::Arith { + op: ArithOp::Sub, + lhs, + rhs, + } if matches!(lhs.as_ref(), WithExpr::Field(_)) && matches!(rhs.as_ref(), WithExpr::Field(_)) + )); + assert!(matches!( + cols[1].expr, + WithExpr::Arith { + op: ArithOp::Div, + .. + } + )); + assert!(matches!( + cols[2].expr, + WithExpr::Arith { + op: ArithOp::Add, + .. + } + )); + match &cols[3].expr { + WithExpr::When { lhs, op, rhs, .. } => { + assert!(matches!(lhs.as_ref(), WithExpr::Len { .. })); + assert_eq!(*op, PlanPredicateOp::Eq); + assert!(matches!( + rhs.as_ref(), + WithExpr::Literal(WithLiteral::Integer(0)) + )); + } + other => panic!("expected when(len), got {other:?}"), + } + match &cols[4].expr { + WithExpr::When { lhs, op, .. } => { + assert!(matches!( + lhs.as_ref(), + WithExpr::Arith { + op: ArithOp::Sub, + .. + } + )); + assert_eq!(*op, PlanPredicateOp::Gt); + } + other => panic!("expected when(now - field), got {other:?}"), + } + } + + #[test] + fn fold_render_is_collect_barrier() { + let op = ComputeOp::Render { + columns: vec![OutputName::new("title").unwrap()], + template: "{{ r.title }}".into(), + column_aliases: Default::default(), + render_bindings: vec![], + }; + let plan = fold_compute_ops( + &[op], + FrameId::new(1), + crate::plasm_monad::StepId::new("out").unwrap(), + CollectCardinality::List, + ) + .unwrap(); + assert!(matches!(plan.collect(), CollectReason::Render { .. })); + assert!(plan.nodes().is_empty()); + } + + #[test] + fn fold_filter_does_not_become_catalog_filter() { + let pred = PlanPredicate { + field_path: FieldPath::from_dotted("owner").unwrap(), + op: PlanPredicateOp::Eq, + value: PlasmDataValue::Literal { + value: serde_json::json!("alice"), + }, + }; + let node = plan_node_from_compute(&ComputeOp::Filter { + predicates: vec![pred], + }) + .unwrap(); + assert!(matches!(node, PlanNode::Filter(_))); + } + + #[test] + fn with_body_rejects_empty() { + assert!(parse_with_body("").is_err()); + assert!(parse_with_body(" ").is_err()); + } + + #[test] + fn with_body_rejects_hop_summary_and_unknown_calls() { + let err = parse_with_body("n: count(r1)").unwrap_err().to_string(); + assert!(err.contains("count"), "{err}"); + assert!(parse_with_body("x: open(labels)").is_err()); + assert!(parse_with_body("x: rank(score)").is_err()); + let age = parse_with_body("x: age_days(updated_at)") + .unwrap_err() + .to_string(); + assert!(age.contains("age_days"), "{age}"); + assert!(age.contains("len"), "{age}"); + assert!(age.contains("when"), "{age}"); + assert!(!age.contains("age_days, len"), "{age}"); + let empty = parse_with_body("x: empty(title)").unwrap_err().to_string(); + assert!(empty.contains("empty"), "{empty}"); + assert!(parse_with_body("x: datediff(updated_at)").is_err()); + assert!(parse_with_body("x: col(updated_at)").is_err()); + } + + #[test] + fn fusion_error_join_from_surface_is_named() { + let msg = FusionError::JoinFromSurface.to_string(); + assert!(msg.contains("join")); + assert!(msg.contains("surface")); + } +} diff --git a/crates/plasm-core/src/row_plan/plan.rs b/crates/plasm-core/src/row_plan/plan.rs new file mode 100644 index 00000000..5291be95 --- /dev/null +++ b/crates/plasm-core/src/row_plan/plan.rs @@ -0,0 +1,215 @@ +//! Fused row-compute IR. EquiJoin and Render are not pipeline nodes. + +use crate::plasm_monad::payload::{AggregateSpec, FieldPath}; +use crate::plasm_monad::OutputName; +use serde::{Deserialize, Serialize}; +use std::num::NonZeroUsize; + +use super::collect::{CollectCardinality, CollectReason}; +use super::error::{FrameSchemaError, FusionError}; +use super::expr::{ProjectSpec, WithColumn}; +use super::filter::RowFilter; +use super::ids::{FrameId, RowNodeId, SurfaceMeaningId}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RowPlan { + source: FrameId, + nodes: Pipeline, + collect: CollectReason, + cardinality: CollectCardinality, + meaning: SurfaceMeaningId, +} + +impl RowPlan { + pub fn new( + source: FrameId, + nodes: Pipeline, + collect: CollectReason, + cardinality: CollectCardinality, + meaning: SurfaceMeaningId, + ) -> Result { + if nodes.is_empty() && !matches!(collect, CollectReason::Render { .. }) { + // Identity collect (return ingested rows) is legal. + } + Ok(Self { + source, + nodes, + collect, + cardinality, + meaning, + }) + } + + #[must_use] + pub fn source(&self) -> FrameId { + self.source + } + + #[must_use] + pub fn nodes(&self) -> &Pipeline { + &self.nodes + } + + #[must_use] + pub fn collect(&self) -> &CollectReason { + &self.collect + } + + #[must_use] + pub fn cardinality(&self) -> CollectCardinality { + self.cardinality + } + + #[must_use] + pub fn meaning(&self) -> SurfaceMeaningId { + self.meaning + } +} + +/// Append-only pipeline. Written order is meaning; no swap/insert. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct Pipeline(Vec<(RowNodeId, PlanNode)>); + +impl Pipeline { + #[must_use] + pub fn new() -> Self { + Self(Vec::new()) + } + + pub fn push(&mut self, id: RowNodeId, node: PlanNode) -> Result<(), FusionError> { + self.0.push((id, node)); + Ok(()) + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + #[must_use] + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PlanNode { + Filter(RowFilter), + Sort { + key: FieldPath, + descending: bool, + }, + Limit { + count: NonZeroUsize, + }, + Dedupe { + keys: Vec, + }, + Distinct { + keys: Vec, + }, + Project(ProjectSpec), + With { + columns: Vec, + }, + GroupBy { + keys: Vec, + aggs: Vec, + }, + Aggregate { + aggs: Vec, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TypedAggregate { + Count { + name: OutputName, + }, + Numeric { + name: OutputName, + fn_: NumericAgg, + field: FieldPath, + }, + MoneySum { + name: OutputName, + field: FieldPath, + currency: MoneyAggLaw, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NumericAgg { + Sum, + Avg, + Min, + Max, + First, + Last, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MoneyAggLaw { + RequireUniform, + CurrencyIsGroupKey, +} + +impl TypedAggregate { + pub fn from_spec(spec: &AggregateSpec) -> Result { + use crate::plasm_monad::AggregateFunction; + match spec.function { + AggregateFunction::Count => Ok(Self::Count { + name: spec.name.clone(), + }), + AggregateFunction::Sum => { + let field = spec + .field + .clone() + .ok_or(FrameSchemaError::UnknownColumn("sum field".into()))?; + Ok(Self::Numeric { + name: spec.name.clone(), + fn_: NumericAgg::Sum, + field, + }) + } + AggregateFunction::Avg => map_numeric(spec, NumericAgg::Avg), + AggregateFunction::Min => map_numeric(spec, NumericAgg::Min), + AggregateFunction::Max => map_numeric(spec, NumericAgg::Max), + AggregateFunction::First => map_numeric(spec, NumericAgg::First), + AggregateFunction::Last => map_numeric(spec, NumericAgg::Last), + } + } + + #[must_use] + pub fn as_money_sum(spec: &AggregateSpec) -> Option { + use crate::plasm_monad::AggregateFunction; + if spec.function != AggregateFunction::Sum { + return None; + } + spec.field.clone().map(|field| Self::MoneySum { + name: spec.name.clone(), + field, + currency: MoneyAggLaw::RequireUniform, + }) + } +} + +fn map_numeric(spec: &AggregateSpec, fn_: NumericAgg) -> Result { + let field = spec + .field + .clone() + .ok_or(FrameSchemaError::UnknownColumn(format!("{fn_:?} field")))?; + Ok(TypedAggregate::Numeric { + name: spec.name.clone(), + fn_, + field, + }) +} diff --git a/crates/plasm-core/src/row_plan/schema.rs b/crates/plasm-core/src/row_plan/schema.rs new file mode 100644 index 00000000..06dac95b --- /dev/null +++ b/crates/plasm-core/src/row_plan/schema.rs @@ -0,0 +1,141 @@ +//! Logical frame schema. Physical Polars dtypes stay behind the runtime adapter. + +use crate::identity::EntityName; +use crate::plasm_monad::payload::FieldPath; +use crate::TemporalWireFormat; +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; + +use crate::plasm_monad::OutputName; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct ColumnName(OutputName); + +impl ColumnName { + pub fn new(name: impl Into) -> Result { + Ok(Self(OutputName::new(name.into())?)) + } + + #[must_use] + pub fn as_str(&self) -> &str { + self.0.as_str() + } + + #[must_use] + pub fn as_output_name(&self) -> &OutputName { + &self.0 + } +} + +impl From for ColumnName { + fn from(name: OutputName) -> Self { + Self(name) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PlasmFrameSchema { + shape: FrameShape, + columns: IndexMap, +} + +impl PlasmFrameSchema { + #[must_use] + pub fn new(shape: FrameShape, columns: IndexMap) -> Self { + Self { shape, columns } + } + + #[must_use] + pub fn opaque_object() -> Self { + Self { + shape: FrameShape::Remapped { + reason: RemapReason::Project, + }, + columns: IndexMap::new(), + } + } + + #[must_use] + pub fn shape(&self) -> &FrameShape { + &self.shape + } + + #[must_use] + pub fn columns(&self) -> &IndexMap { + &self.columns + } + + #[must_use] + pub fn with_intact_identity(mut self) -> Self { + if let FrameShape::Entity { identity, .. } = &mut self.shape { + *identity = IdentityPreservation::Intact; + } + self + } + + pub fn insert_column(&mut self, name: ColumnName, col: LogicalColumn) { + self.columns.insert(name.as_str().to_string(), col); + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FrameShape { + Entity { + entity: EntityName, + identity: IdentityPreservation, + }, + Remapped { + reason: RemapReason, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IdentityPreservation { + Intact, + Projected, + Aggregated, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RemapReason { + Project, + GroupBy, + Aggregate, + Derive, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LogicalColumn { + pub ty: LogicalColumnType, + pub nullable: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LogicalColumnType { + Null, + Boolean, + Integer, + Number, + String, + Duration, + Temporal { format: TemporalWireFormat }, + Money { currency: MoneyColumnLayout }, + EntityRef { target: EntityName }, + Array, + Object, + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MoneyColumnLayout { + Uniform { currency: String }, + PerRow, +} diff --git a/crates/plasm-core/src/row_plan/with_parse.rs b/crates/plasm-core/src/row_plan/with_parse.rs new file mode 100644 index 00000000..f68937a1 --- /dev/null +++ b/crates/plasm-core/src/row_plan/with_parse.rs @@ -0,0 +1,232 @@ +//! Parse `.with{name: expr, …}` bodies. + +use crate::plasm_monad::payload::{FieldPath, PlanPredicateOp}; +use crate::plasm_monad::{OutputName, WithColumn, WithExpr, WithExprError, WithLiteral}; + +use super::expr::ArithOp; + +pub fn parse_with_body(body: &str) -> Result, WithExprError> { + let body = body.trim(); + if body.is_empty() { + return Err(WithExprError::EmptyBody); + } + let mut columns = Vec::new(); + for part in split_top_level_comma(body) { + let part = part.trim(); + let Some((name, expr)) = part.split_once(':') else { + return Err(WithExprError::Parse(format!( + "expected `name: expr`, got `{part}`" + ))); + }; + let name = OutputName::new(name.trim().to_string()).map_err(WithExprError::BadColumn)?; + let expr = parse_with_expr(expr.trim())?; + columns.push(WithColumn { name, expr }); + } + if columns.is_empty() { + return Err(WithExprError::EmptyBody); + } + Ok(columns) +} + +fn split_top_level_comma(s: &str) -> Vec<&str> { + let mut out = Vec::new(); + let mut depth = 0i32; + let mut start = 0usize; + for (i, c) in s.char_indices() { + match c { + '(' | '{' => depth += 1, + ')' | '}' => depth -= 1, + ',' if depth == 0 => { + out.push(&s[start..i]); + start = i + 1; + } + _ => {} + } + } + out.push(&s[start..]); + out +} + +fn parse_with_expr(s: &str) -> Result { + let s = s.trim(); + parse_arith(s) +} + +fn parse_arith(s: &str) -> Result { + if let Some((lhs, rhs)) = split_top_bin(s, '+') { + return Ok(WithExpr::Arith { + op: ArithOp::Add, + lhs: Box::new(parse_arith(lhs)?), + rhs: Box::new(parse_arith(rhs)?), + }); + } + if let Some((lhs, rhs)) = split_top_bin(s, '-') { + if !lhs.trim().is_empty() { + return Ok(WithExpr::Arith { + op: ArithOp::Sub, + lhs: Box::new(parse_arith(lhs)?), + rhs: Box::new(parse_arith(rhs)?), + }); + } + } + if let Some((op, lhs, rhs)) = split_top_muldiv(s) { + return Ok(WithExpr::Arith { + op, + lhs: Box::new(parse_arith(lhs)?), + rhs: Box::new(parse_arith(rhs)?), + }); + } + parse_atom(s) +} + +fn split_top_bin(s: &str, op: char) -> Option<(&str, &str)> { + let mut depth = 0i32; + for (i, c) in s.char_indices().rev() { + match c { + ')' | '}' => depth += 1, + '(' | '{' => depth -= 1, + c if c == op && depth == 0 && i > 0 => { + return Some((s[..i].trim(), s[i + op.len_utf8()..].trim())); + } + _ => {} + } + } + None +} + +fn split_top_muldiv(s: &str) -> Option<(ArithOp, &str, &str)> { + let mut depth = 0i32; + for (i, c) in s.char_indices().rev() { + match c { + ')' | '}' => depth += 1, + '(' | '{' => depth -= 1, + '*' | '/' if depth == 0 && i > 0 => { + let op = if c == '*' { ArithOp::Mul } else { ArithOp::Div }; + return Some((op, s[..i].trim(), s[i + 1..].trim())); + } + _ => {} + } + } + None +} + +fn parse_atom(s: &str) -> Result { + let s = s.trim(); + if let Some(inner) = strip_wrapping_parens(s) { + return parse_with_expr(inner); + } + if s.eq_ignore_ascii_case("null") { + return Ok(WithExpr::Literal(WithLiteral::Null)); + } + if s.eq_ignore_ascii_case("true") { + return Ok(WithExpr::Literal(WithLiteral::Bool(true))); + } + if s.eq_ignore_ascii_case("false") { + return Ok(WithExpr::Literal(WithLiteral::Bool(false))); + } + if s.eq_ignore_ascii_case("now") { + return Ok(WithExpr::Now); + } + if let Some(inner) = s.strip_prefix('"').and_then(|t| t.strip_suffix('"')) { + return Ok(WithExpr::Literal(WithLiteral::String(inner.to_string()))); + } + if let Some(rest) = s.strip_prefix("len(").and_then(|t| t.strip_suffix(')')) { + return Ok(WithExpr::Len { + field: FieldPath::from_dotted(rest.trim()).map_err(WithExprError::Parse)?, + }); + } + if let Some(rest) = s.strip_prefix("when(").and_then(|t| t.strip_suffix(')')) { + return parse_when(rest); + } + if let Ok(i) = s.parse::() { + return Ok(WithExpr::Literal(WithLiteral::Integer(i))); + } + if s.parse::().is_ok() { + return Ok(WithExpr::Literal(WithLiteral::Number(s.to_string()))); + } + if let Some(idx) = s.find('(') { + if s.ends_with(')') { + let fname = &s[..idx]; + return Err(WithExprError::Parse(format!( + "unknown .with function `{fname}` (known calls: len, when; `now` is a word, not a call)" + ))); + } + } + FieldPath::from_dotted(s) + .map(WithExpr::Field) + .map_err(WithExprError::Parse) +} + +/// Outer `(`…`)` only when that pair wraps the whole atom (`(now - t)`, not `(a)+(b)`). +fn strip_wrapping_parens(s: &str) -> Option<&str> { + if !s.starts_with('(') || !s.ends_with(')') { + return None; + } + let mut depth = 0i32; + for (i, c) in s.char_indices() { + match c { + '(' => depth += 1, + ')' => { + depth -= 1; + if depth == 0 { + if i + 1 == s.len() { + return Some(s[1..i].trim()); + } + return None; + } + } + _ => {} + } + } + None +} + +fn parse_when(args: &str) -> Result { + let parts = split_top_level_comma(args); + if parts.len() != 3 { + return Err(WithExprError::Parse( + "when(pred, then, else) requires three arguments".into(), + )); + } + let (lhs, op, rhs) = split_when_cmp(parts[0].trim())?; + Ok(WithExpr::When { + lhs: Box::new(parse_with_expr(lhs)?), + op, + rhs: Box::new(parse_with_expr(rhs)?), + then: Box::new(parse_with_expr(parts[1])?), + else_: Box::new(parse_with_expr(parts[2])?), + }) +} + +fn split_when_cmp(s: &str) -> Result<(&str, PlanPredicateOp, &str), WithExprError> { + let ops: [(&str, PlanPredicateOp); 6] = [ + (">=", PlanPredicateOp::Gte), + ("<=", PlanPredicateOp::Lte), + ("!=", PlanPredicateOp::Ne), + ("=", PlanPredicateOp::Eq), + (">", PlanPredicateOp::Gt), + ("<", PlanPredicateOp::Lt), + ]; + let mut depth = 0i32; + for (i, c) in s.char_indices() { + match c { + '(' | '{' => depth += 1, + ')' | '}' => depth -= 1, + _ if depth == 0 && i > 0 => { + for (sym, op) in ops { + if s[i..].starts_with(sym) { + let lhs = s[..i].trim(); + let rhs = s[i + sym.len()..].trim(); + if !lhs.is_empty() && !rhs.is_empty() { + return Ok((lhs, op, rhs)); + } + } + } + } + _ => {} + } + } + Err(WithExprError::Parse(format!( + "when() predicate must be a comparison, got `{s}`" + ))) +} diff --git a/crates/plasm-core/src/snapshots/plasm_core__prompt_render__tests__plasm_tool_description.snap b/crates/plasm-core/src/snapshots/plasm_core__prompt_render__tests__plasm_tool_description.snap index 31f3b8f8..7e2b2153 100644 --- a/crates/plasm-core/src/snapshots/plasm_core__prompt_render__tests__plasm_tool_description.snap +++ b/crates/plasm-core/src/snapshots/plasm_core__prompt_render__tests__plasm_tool_description.snap @@ -1,5 +1,6 @@ --- -source: crates/plasm-core/src/prompt_render/tests.rs +source: plasm-oss/crates/plasm-core/src/prompt_render/tests.rs +assertion_line: 1972 expression: "super::PLASM_TOOL_DESCRIPTION" --- **Plasm** — **`logical_session_ref`** + **`program`**. Clean read-only plans execute inline (rows in tool **`content`**). Plans with any mutation / review gate return a **`run_ref`** (`pcN`, 10 min TTL) for **`plasm_run`**; do **not** echo the program. @@ -45,7 +46,7 @@ TSV table semantics: Core surface: - Get identity: `e#(id)` (parens). Query/filter: `e#{field=…}` (braces). Search when taught: `e#~$` / `e#~"text"`. -- Postfix from TSV left column: `.filter{…}` `.sort` `.limit` `.group_by` `.aggregate` `[field,…]`. +- Postfix from TSV left column: `.filter{…}` `.sort` `.limit` `.group_by` `.aggregate` `.with{k: expr}` `[field,…]`. - Inline rows when delivery is `inline` (≤25); `snapshot_only` / `(in artifact)` need artifact read. Copy **`artifact_uri`** from the step — never plan `run_step` / `dict_ref`. - `page(...)` is HTTP-execute only — not an MCP tool argument. diff --git a/crates/plasm-core/src/span_graph_tests.rs b/crates/plasm-core/src/span_graph_tests.rs new file mode 100644 index 00000000..f8b66b2b --- /dev/null +++ b/crates/plasm-core/src/span_graph_tests.rs @@ -0,0 +1,38 @@ +//! Force-flush span-graph contracts for plasm-core hot-path spans. + +#![cfg(test)] + +use std::path::PathBuf; + +use plasm_otel::span_capture::{find_span, is_descendant, with_captured_spans}; + +fn fixtures_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../..") + .join("fixtures/schemas") +} + +#[test] +fn schema_load_path_parents_assemble_and_parse_program_exists() { + let dir = fixtures_root().join("overshow_tools"); + assert!(dir.is_dir(), "expected fixture dir at {}", dir.display()); + + let ((), spans) = with_captured_spans(|| { + let cgs = crate::loader::load_schema(&dir).expect("load schema"); + // Even on parse failure the parse.program span must still be entered. + let _ = crate::expr_parser::parse("not a valid program {{{", &cgs); + }); + + let load = find_span(&spans, "plasm_core.schema.load_path").expect("load_path span"); + let assemble = find_span(&spans, "plasm_core.schema.assemble").expect("assemble span"); + assert!( + is_descendant(assemble, load, &spans), + "assemble must be under load_path (got {:?})", + spans.iter().map(|s| s.name.as_ref()).collect::>() + ); + assert!( + find_span(&spans, "plasm_core.parse.program").is_some(), + "parse.program span missing; got {:?}", + spans.iter().map(|s| s.name.as_ref()).collect::>() + ); +} diff --git a/crates/plasm-core/src/spans.rs b/crates/plasm-core/src/spans.rs index e7dc4b77..1abfd258 100644 --- a/crates/plasm-core/src/spans.rs +++ b/crates/plasm-core/src/spans.rs @@ -78,3 +78,38 @@ pub(crate) fn prompt_domain_bundle_exposure_federated( cache.hit = tracing::field::Empty, ) } + +// --- Parse / typecheck / discovery / catalog --------------------------------- + +#[inline] +pub(crate) fn parse_program(source_len: usize) -> Span { + tracing::debug_span!("plasm_core.parse.program", source_len = source_len) +} + +#[inline] +pub(crate) fn typecheck_expr() -> Span { + tracing::debug_span!("plasm_core.typecheck.expr") +} + +#[inline] +pub(crate) fn discovery_discover() -> Span { + tracing::debug_span!( + "plasm_core.discovery.discover", + candidate_count = tracing::field::Empty, + result_count = tracing::field::Empty, + ) +} + +#[inline] +pub(crate) fn catalog_load_il(byte_len: usize) -> Span { + tracing::debug_span!("plasm_core.catalog.load_il", byte_len = byte_len) +} + +#[inline] +pub(crate) fn schema_validate(entity_count: usize, capability_count: usize) -> Span { + tracing::debug_span!( + "plasm_core.schema.validate", + entity_count = entity_count, + capability_count = capability_count, + ) +} diff --git a/crates/plasm-core/src/type_checker.rs b/crates/plasm-core/src/type_checker.rs index fcb78442..cceda48a 100644 --- a/crates/plasm-core/src/type_checker.rs +++ b/crates/plasm-core/src/type_checker.rs @@ -74,6 +74,8 @@ fn union_query_and_search_params(cgs: &CGS, entity: &str) -> Vec Result<(), TypeError> { + let span = crate::spans::typecheck_expr(); + let _guard = span.enter(); match expr { Expr::Query(query) => type_check_query(query, cgs), Expr::Get(get) => type_check_get(get, cgs), diff --git a/crates/plasm-core/src/typed_row.rs b/crates/plasm-core/src/typed_row.rs index 994bd14f..1f7bbb28 100644 --- a/crates/plasm-core/src/typed_row.rs +++ b/crates/plasm-core/src/typed_row.rs @@ -22,6 +22,8 @@ pub enum TypedFieldValue { Object(IndexMap), /// Normalized `entity_ref` payload when [`FieldType::EntityRef`] applies and the wire shape parses. EntityRef(EntityRefPayload), + /// Fowler money — must not collapse to [`TypedFieldValue::Json`]. + Money(crate::money::MoneyValue), PlasmInputRef(PlasmInputRef), /// Arbitrary subtree (`FieldType::Json`, `Blob`, attachment blobs) stored verbatim. Json(Value), @@ -39,6 +41,10 @@ impl TypedFieldValue { pub fn from_value_in_field(field_type: &FieldType, v: Value) -> Self { match field_type { FieldType::Json | FieldType::Blob => Self::Json(v), + FieldType::Money => match v { + Value::Money(m) => Self::Money(m), + other => Self::from(other), + }, FieldType::EntityRef { .. } => match EntityRefPayload::try_from_value(&v) { Ok(p) => Self::EntityRef(p), Err(_) => Self::from(v), @@ -61,6 +67,7 @@ impl TypedFieldValue { Value::Object(m.iter().map(|(k, v)| (k.clone(), v.to_value())).collect()) } TypedFieldValue::EntityRef(p) => p.to_value(), + TypedFieldValue::Money(m) => Value::Money(m.clone()), TypedFieldValue::PlasmInputRef(r) => Value::PlasmInputRef(r.clone()), TypedFieldValue::Json(v) => v.clone(), } @@ -107,7 +114,7 @@ impl From for TypedFieldValue { Value::Float(f) => TypedFieldValue::Float(f), Value::String(s) | Value::PhraseIdent(s) => TypedFieldValue::String(s), Value::Array(a) => TypedFieldValue::Array(a.into_iter().map(Self::from).collect()), - Value::Money(_) => TypedFieldValue::Json(v), + Value::Money(m) => TypedFieldValue::Money(m), Value::Object(m) => { TypedFieldValue::Object(m.into_iter().map(|(k, v)| (k, Self::from(v))).collect()) } @@ -190,4 +197,19 @@ mod tests { other => panic!("expected Json variant: {other:?}"), } } + + #[test] + fn money_does_not_dump_to_json() { + let m = + crate::money::MoneyValue::new(rust_decimal::Decimal::new(1250, 2), Some("USD".into())); + let v = Value::Money(m.clone()); + let tf = TypedFieldValue::from(v); + match tf { + TypedFieldValue::Money(got) => assert_eq!(got, m), + other => panic!("expected Money variant: {other:?}"), + } + let tf2 = TypedFieldValue::from_value_in_field(&FieldType::Money, Value::Money(m.clone())); + assert!(matches!(tf2, TypedFieldValue::Money(_))); + assert_eq!(tf2.to_value(), Value::Money(m)); + } } diff --git a/crates/plasm-e2e/tests/plasm_language_matrix.rs b/crates/plasm-e2e/tests/plasm_language_matrix.rs index 64301690..395e1dcf 100644 --- a/crates/plasm-e2e/tests/plasm_language_matrix.rs +++ b/crates/plasm-e2e/tests/plasm_language_matrix.rs @@ -6,11 +6,11 @@ //! ## Coverage contract (keep in sync when extending the language) //! //! Each [`MatrixRow`] should exercise a **distinct** user-visible construct or sugar called out in -//! [`docs/plasm-language-definition.md`](../../../../docs/plasm-language-definition.md): +//! [`doc-site/docs/reference/plasm-language-definition.md`](../../../../doc-site/docs/reference/plasm-language-definition.md): //! //! - Entity roots: bare query, search `~`, get `(id)`, brace predicates `{field=value}`, comparisons. //! - Postfix: `.limit`, `.sort(field[, dir])` including `asc`/`desc`, `.aggregate` (named + sugar), -//! `.group_by`, `.singleton()`, `.page_size`, bracket projection `[…]`. +//! `.group_by`, `.with{k: expr}`, `.singleton()`, `.page_size`, bracket projection `[…]`. //! - Programs: bindings, node-ref continuation, parallel final roots, `compile_plasm_expression` //! (single-line surface) vs multi-line DAG programs. //! - Relations: `from_parent_get`, `query_scoped`, opaque `r#` nav (not `p#`), one-cardinality `r#`, @@ -97,6 +97,7 @@ const REQUIRED_FEATURE_TAGS: &[&str] = &[ "for_each_effect", "domain_symbol_e1", "postfix_group_by", + "postfix_with", "postfix_group_by_aggregate_chain", "postfix_row_filter", "postfix_group_by_sugar", @@ -614,6 +615,14 @@ fn assert_planning_ir( return Err(format!("expected Filter compute, got {:?}", computes)); } } + "lang_with_mul" | "lang_with_div" | "lang_with_concat" | "lang_with_when_len" => { + if !computes + .iter() + .any(|c| matches!(c.op, ComputeOp::With { .. })) + { + return Err(format!("expected With compute, got {:?}", computes)); + } + } "lang_group_by" => { let Some(ComputeTemplate { op: ComputeOp::GroupBy { keys, aggregates }, @@ -1848,6 +1857,46 @@ newbranch, newfile"#, min_node_results: 1, expect_markdown_substrings: &["```tsv", "alice"], }, + MatrixRow { + id: "lang_with_mul", + program: "items = LangItem\nboosted = items.with{boost: score * 2}.limit(3)\nboosted[id,boost]", + surface_line: false, + federated: false, + features: &["postfix_with", "bindings_assignment", "postfix_limit"], + min_node_results: 1, + expect_markdown_substrings: &["```tsv", "boost"], + }, + MatrixRow { + id: "lang_with_div", + program: "items = LangItem\nhalved = items.with{half: score / 2}.limit(3)\nhalved[id,half]", + surface_line: false, + federated: false, + features: &["postfix_with", "bindings_assignment", "postfix_limit"], + min_node_results: 1, + expect_markdown_substrings: &["```tsv", "half"], + }, + MatrixRow { + id: "lang_with_concat", + program: r#"items = LangItem.filter{owner="alice"} +tagged = items.with{tag: owner + owner}.limit(1) +tagged[tag]"#, + surface_line: false, + federated: false, + features: &["postfix_with", "bindings_assignment", "postfix_limit", "postfix_row_filter"], + min_node_results: 1, + expect_markdown_substrings: &["```tsv", "alicealice"], + }, + MatrixRow { + id: "lang_with_when_len", + program: r#"items = LangItem.filter{owner="alice"} +labeled = items.with{label: when(len(owner)>0, owner, title)}.limit(1) +labeled[label]"#, + surface_line: false, + federated: false, + features: &["postfix_with", "bindings_assignment", "postfix_limit", "postfix_row_filter"], + min_node_results: 1, + expect_markdown_substrings: &["```tsv", "alice"], + }, MatrixRow { id: "lang_relation_lines", program: r#"LangItem("i1").lines[id,note]"#, diff --git a/crates/plasm-otel/Cargo.toml b/crates/plasm-otel/Cargo.toml index c130f186..98c02be3 100644 --- a/crates/plasm-otel/Cargo.toml +++ b/crates/plasm-otel/Cargo.toml @@ -6,6 +6,11 @@ license = "MIT OR Apache-2.0" description = "Shared OpenTelemetry OTLP bootstrap (traces, metrics, logs) from standard OTEL_* env vars" readme = "README.md" +[features] +default = [] +# In-memory span exporter + capture helpers for span-graph tests in dependent crates. +testing = ["opentelemetry_sdk/testing"] + [dependencies] anyhow = { workspace = true } opentelemetry = { workspace = true } @@ -17,3 +22,8 @@ http = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } tracing-opentelemetry = { workspace = true } +axum = { workspace = true } + +[dev-dependencies] +opentelemetry_sdk = { workspace = true, features = ["testing"] } +tokio = { workspace = true } diff --git a/crates/plasm-otel/README.md b/crates/plasm-otel/README.md index 64944b7e..954a0f1b 100644 --- a/crates/plasm-otel/README.md +++ b/crates/plasm-otel/README.md @@ -48,6 +48,18 @@ HTTP servers use `**tower_http::trace::TraceLayer`**, whose default request span If you set `**RUST_LOG**` yourself (for example to `info` only), add `**tower_http=debug**` (or `trace`) for request spans to reach OTLP, and the same `**on_request`/`on_response**` overrides if you want to avoid that log noise. +## Semantic span names (stable contract) + +Application spans use **product semantics**, not Rust module paths, so refactors do not churn dashboards. Prefixes: + +| Prefix | Crate | Examples | +|--------|-------|----------| +| `plasm_agent.*` | `plasm-agent-core` | `plasm_agent.execute.expression`, `plasm_agent.mcp.tool.plasm`, `plasm_agent.plan.live_run` | +| `plasm_core.*` | `plasm-core` | `plasm_core.schema.load_path`, `plasm_core.parse.program`, `plasm_core.typecheck.expr` | +| `plasm_runtime.*` | `plasm-runtime` | `plasm_runtime.execute.query`, `plasm_runtime.http.compiled_request`, `plasm_runtime.projection.hydrate` | + +Parent/child relationships are asserted in crate `span_graph_tests` (force-flush capture via `plasm_otel::span_capture` under the `testing` feature). Async work must use `.instrument(span)` (or explicit `parent:`) so nested spans link under the request root — see module docs in each crate’s `spans.rs`. + ## Fallback If OTLP wiring fails at startup, falls back to **stderr** `tracing` formatting only (same as a pure console setup). \ No newline at end of file diff --git a/crates/plasm-otel/src/lib.rs b/crates/plasm-otel/src/lib.rs index a1352369..c9dce782 100644 --- a/crates/plasm-otel/src/lib.rs +++ b/crates/plasm-otel/src/lib.rs @@ -3,6 +3,9 @@ mod trace_context; mod tui_capture; +#[cfg(any(feature = "testing", test))] +pub mod span_capture; + pub use trace_context::{install_w3c_trace_context_propagator, tower_http_trace_parent_span}; pub use tui_capture::{layer as tui_capture_layer, TuiCaptureLayer, TuiLogCallback, TuiLogRecord}; @@ -362,3 +365,6 @@ where } } } + +#[cfg(test)] +mod span_graph_tests; diff --git a/crates/plasm-otel/src/span_capture.rs b/crates/plasm-otel/src/span_capture.rs new file mode 100644 index 00000000..aad9601d --- /dev/null +++ b/crates/plasm-otel/src/span_capture.rs @@ -0,0 +1,85 @@ +//! In-memory OpenTelemetry span capture for force_flush parent/child assertions. +//! +//! Enabled with the `testing` feature (or crate unit tests). + +use opentelemetry::trace::TracerProvider as _; +use opentelemetry_sdk::trace::{ + InMemorySpanExporter, InMemorySpanExporterBuilder, SdkTracerProvider, SimpleSpanProcessor, + SpanData, +}; +use tracing::Subscriber; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::Registry; + +/// Captured finished spans plus the provider used to flush them. +pub struct SpanCapture { + exporter: InMemorySpanExporter, + provider: SdkTracerProvider, +} + +impl SpanCapture { + /// Build a tracer provider with a simple in-memory exporter (sync on span end). + /// + /// Uses a provider-local tracer for the subscriber layer (no global tracer provider + /// mutation — safe under parallel `cargo test`). + pub fn install() -> Self { + let exporter = InMemorySpanExporterBuilder::new().build(); + let provider = SdkTracerProvider::builder() + .with_span_processor(SimpleSpanProcessor::new(exporter.clone())) + .build(); + Self { exporter, provider } + } + + /// Registry + `tracing-opentelemetry` layer bound to this capture's tracer. + pub fn subscriber(&self) -> impl Subscriber + Send + Sync + 'static { + let tracer = self.provider.tracer("plasm-otel-span-capture"); + let otel = tracing_opentelemetry::layer().with_tracer(tracer); + Registry::default().with(otel) + } + + /// Force-flush then return finished [`SpanData`] rows. + pub fn force_flush_spans(&self) -> Vec { + let _ = self.provider.force_flush(); + self.exporter + .get_finished_spans() + .expect("in-memory span exporter lock") + } +} + +/// Run `f` under a thread-local subscriber that exports OTel spans; return result + finished spans. +pub fn with_captured_spans(f: F) -> (R, Vec) +where + F: FnOnce() -> R, +{ + let capture = SpanCapture::install(); + let subscriber = capture.subscriber(); + let result = tracing::subscriber::with_default(subscriber, f); + let spans = capture.force_flush_spans(); + (result, spans) +} + +/// True when `child` lists `parent` as its parent span id. +pub fn is_child_of(child: &SpanData, parent: &SpanData) -> bool { + child.parent_span_id == parent.span_context.span_id() +} + +/// True when `child` is a descendant of `ancestor` by walking `parent_span_id`. +pub fn is_descendant(child: &SpanData, ancestor: &SpanData, spans: &[SpanData]) -> bool { + let mut current = child; + for _ in 0..16 { + if is_child_of(current, ancestor) { + return true; + } + let parent_id = current.parent_span_id; + match spans.iter().find(|s| s.span_context.span_id() == parent_id) { + Some(p) => current = p, + None => return false, + } + } + false +} + +/// Find the first finished span whose name equals `name`. +pub fn find_span<'a>(spans: &'a [SpanData], name: &str) -> Option<&'a SpanData> { + spans.iter().find(|s| s.name.as_ref() == name) +} diff --git a/crates/plasm-otel/src/span_graph_tests.rs b/crates/plasm-otel/src/span_graph_tests.rs new file mode 100644 index 00000000..f97af50e --- /dev/null +++ b/crates/plasm-otel/src/span_graph_tests.rs @@ -0,0 +1,48 @@ +//! Span-graph contracts for `plasm-otel` HTTP MakeSpan + capture helper. + +#![cfg(test)] + +use crate::span_capture::{find_span, is_child_of, with_captured_spans}; +use crate::tower_http_trace_parent_span; +use axum::http::Request; + +#[test] +fn http_request_span_name_and_semantic_fields() { + let req = Request::builder() + .method("POST") + .uri("/execute/abc/session-1") + .body(()) + .unwrap(); + let (_, spans) = with_captured_spans(|| { + let span = tower_http_trace_parent_span(&req); + let _g = span.entered(); + }); + let http = find_span(&spans, "plasm_agent.http.request").expect("http request span"); + assert!( + http.attributes + .iter() + .any(|kv| kv.key.as_str() == "http.method"), + "expected http.method attribute, got {:?}", + http.attributes + ); + assert!( + http.attributes + .iter() + .any(|kv| kv.key.as_str() == "http.route"), + "expected http.route attribute, got {:?}", + http.attributes + ); +} + +#[test] +fn capture_helper_records_parent_child() { + let (_, spans) = with_captured_spans(|| { + let parent = tracing::info_span!("test.parent"); + let _g = parent.enter(); + let child = tracing::info_span!("test.child"); + let _c = child.enter(); + }); + let parent = find_span(&spans, "test.parent").expect("parent"); + let child = find_span(&spans, "test.child").expect("child"); + assert!(is_child_of(child, parent)); +} diff --git a/crates/plasm-otel/src/trace_context.rs b/crates/plasm-otel/src/trace_context.rs index 1ed45eaa..45349e23 100644 --- a/crates/plasm-otel/src/trace_context.rs +++ b/crates/plasm-otel/src/trace_context.rs @@ -3,6 +3,7 @@ //! Call [`crate::install_w3c_trace_context_propagator`] during OTLP init when traces are enabled //! so [`tower_http_trace_parent_span`] can extract `traceparent` / `tracestate` from request headers. +use axum::extract::MatchedPath; use http::Request; use opentelemetry::global; use opentelemetry_http::HeaderExtractor; @@ -12,22 +13,29 @@ use tracing_opentelemetry::OpenTelemetrySpanExt; /// /// Must run **after** [`opentelemetry::global::set_tracer_provider`] when exporting traces, and /// **before** serving HTTP so incoming distributed traces (e.g. Phoenix `Req` + `traceparent`) -/// become parents of the tower-http `request` span. +/// become parents of the tower-http request span. pub fn install_w3c_trace_context_propagator() { global::set_text_map_propagator(opentelemetry_sdk::propagation::TraceContextPropagator::new()); } -/// [`tower_http::trace::MakeSpan`] implementation matching the default `request` span fields at -/// [`tracing::Level::DEBUG`], with the OpenTelemetry parent taken from W3C headers when present. +/// [`tower_http::trace::MakeSpan`] for Axum: static name `plasm_agent.http.request`, semantic +/// `http.method` / `http.route` (matched template when available; path without query otherwise), +/// and OpenTelemetry parent from W3C headers when present. +/// +/// Apply with [`.route_layer`](axum::Router::route_layer) so [`MatchedPath`] is populated. pub fn tower_http_trace_parent_span(request: &Request) -> tracing::Span { let parent_cx = global::get_text_map_propagator(|propagator| { propagator.extract(&HeaderExtractor(request.headers())) }); + let route = request + .extensions() + .get::() + .map(|p| p.as_str()) + .unwrap_or_else(|| request.uri().path()); let span = tracing::debug_span!( - "request", - method = %request.method(), - uri = %request.uri(), - version = ?request.version(), + "plasm_agent.http.request", + http.method = %request.method(), + http.route = %route, ); let _ = span.set_parent(parent_cx); span diff --git a/crates/plasm-runtime/Cargo.toml b/crates/plasm-runtime/Cargo.toml index b0856b67..d87fcfe1 100644 --- a/crates/plasm-runtime/Cargo.toml +++ b/crates/plasm-runtime/Cargo.toml @@ -34,8 +34,12 @@ oauth2 = { workspace = true } url = { workspace = true } opentelemetry = { workspace = true } minijinja = { version = "2.19.0", default-features = false, features = ["builtins", "serde"] } +polars = { workspace = true } +chrono = { workspace = true } +rust_decimal = { workspace = true } [dev-dependencies] +plasm-otel = { path = "../plasm-otel", features = ["testing"] } proptest = { workspace = true } insta = { workspace = true } tempfile = { workspace = true } diff --git a/crates/plasm-runtime/src/execution/http_exec.rs b/crates/plasm-runtime/src/execution/http_exec.rs index 02ec3bf4..8844fafe 100644 --- a/crates/plasm-runtime/src/execution/http_exec.rs +++ b/crates/plasm-runtime/src/execution/http_exec.rs @@ -1,12 +1,25 @@ //! HTTP transport helpers (compiled request + absolute URL GET). use super::*; +use crate::http_transport::compiled_method_label; +use tracing::Instrument; impl ExecutionEngine { /// Execute request and capture `Link` header (`rel="next"`) when present. pub(crate) async fn execute_http_request_full( &self, request: &CompiledRequest, + ) -> Result<(serde_json::Value, Option), RuntimeError> { + let method = compiled_method_label(&request.method); + let url_len = request.url_path().len(); + self.execute_http_request_full_inner(request) + .instrument(crate::spans::http_compiled_request(method, url_len)) + .await + } + + async fn execute_http_request_full_inner( + &self, + request: &CompiledRequest, ) -> Result<(serde_json::Value, Option), RuntimeError> { let base_url = self.effective_http_base_for_request(); let auth = self.resolve_auth_http().await?; @@ -19,6 +32,15 @@ impl ExecutionEngine { pub(crate) async fn get_json_absolute( &self, url: &str, + ) -> Result<(serde_json::Value, Option), RuntimeError> { + self.get_json_absolute_inner(url) + .instrument(crate::spans::http_absolute_get(url.len())) + .await + } + + async fn get_json_absolute_inner( + &self, + url: &str, ) -> Result<(serde_json::Value, Option), RuntimeError> { let auth = self.resolve_auth_http().await?; self.transport.get_json_absolute(url, auth).await diff --git a/crates/plasm-runtime/src/execution/mod.rs b/crates/plasm-runtime/src/execution/mod.rs index 24377818..fba14772 100644 --- a/crates/plasm-runtime/src/execution/mod.rs +++ b/crates/plasm-runtime/src/execution/mod.rs @@ -990,6 +990,15 @@ impl ExecutionEngine { async fn execute_operation_full( &self, operation: &CompiledOperation, + ) -> Result<(serde_json::Value, Option), RuntimeError> { + self.execute_operation_full_inner(operation) + .instrument(crate::spans::execute_operation()) + .await + } + + async fn execute_operation_full_inner( + &self, + operation: &CompiledOperation, ) -> Result<(serde_json::Value, Option), RuntimeError> { let fp = crate::RequestFingerprint::from_operation(operation); let out = match operation { @@ -1224,6 +1233,20 @@ impl ExecutionEngine { mode: ExecutionMode, consume: StreamConsumeOpts, ambient: &ViewAmbientContext, + ) -> Result { + self.execute_query_inner(query, cgs, mat, mode, consume, ambient) + .instrument(crate::spans::execute_query()) + .await + } + + async fn execute_query_inner( + &self, + query: &QueryExpr, + cgs: &CGS, + mat: &mut SessionMaterialization, + mode: ExecutionMode, + consume: StreamConsumeOpts, + ambient: &ViewAmbientContext, ) -> Result { let mut stream = self.query_to_stream(query, cgs, mat, mode, consume.clone(), None, ambient)?; @@ -1411,6 +1434,19 @@ impl ExecutionEngine { mat: &mut SessionMaterialization, mode: ExecutionMode, ambient: &ViewAmbientContext, + ) -> Result { + self.execute_get_inner(get, cgs, mat, mode, ambient) + .instrument(crate::spans::execute_get()) + .await + } + + async fn execute_get_inner( + &self, + get: &GetExpr, + cgs: &CGS, + mat: &mut SessionMaterialization, + mode: ExecutionMode, + ambient: &ViewAmbientContext, ) -> Result { // Satisfy from cache only when we already hold a detail payload. if let Some(entity) = mat.get(&get.reference) { @@ -4085,6 +4121,84 @@ mod tests { assert_eq!(last.lock().unwrap().as_deref(), Some("http://right-host")); } + /// Sync test: `with_captured_spans` + nested `block_on` must not run under `#[tokio::test]`. + #[test] + fn execute_operation_parents_http_compiled_request_on_live_get() { + use crate::auth::ResolvedAuth; + use crate::http_transport::HttpTransport; + use async_trait::async_trait; + use plasm_compile::CompiledRequest; + use plasm_otel::span_capture::{find_span, is_child_of, with_captured_spans}; + use std::sync::Arc; + + struct OkTransport; + + #[async_trait] + impl HttpTransport for OkTransport { + async fn send_compiled_http( + &self, + _base_url: &str, + _request: &CompiledRequest, + _auth: Option, + ) -> Result<(serde_json::Value, Option), RuntimeError> { + Ok((serde_json::json!({"id": "1", "name": "n"}), None)) + } + + async fn get_json_absolute( + &self, + _url: &str, + _auth: Option, + ) -> Result<(serde_json::Value, Option), RuntimeError> { + Ok((serde_json::json!({}), None)) + } + } + + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let ((), spans) = with_captured_spans(|| { + rt.block_on(async { + let config = ExecutionConfig { + base_url: Some("http://example.test".to_string()), + ..ExecutionConfig::default() + }; + let engine = + ExecutionEngine::new_with_transport(config, Arc::new(OkTransport), None); + let cgs = create_test_cgs(); + let mut cache = SessionMaterialization::new(); + let expr = Expr::Get(GetExpr::new("Account", "1")); + let _ = engine + .execute( + &expr, + &cgs, + &mut cache, + None, + StreamConsumeOpts::default(), + ExecuteOptions::default(), + ) + .await; + }); + }); + + let parent = + find_span(&spans, "plasm_runtime.execute.operation").expect("execute.operation"); + let child = find_span(&spans, "plasm_runtime.http.compiled_request") + .expect("http.compiled_request"); + assert!( + is_child_of(child, parent), + "compiled_request must be child of execute.operation; spans={:?}", + spans.iter().map(|s| s.name.as_ref()).collect::>() + ); + assert!( + child + .attributes + .iter() + .any(|kv| kv.key.as_str() == "http.method"), + "expected http.method on compiled_request" + ); + } + #[tokio::test] async fn execute_http_uses_session_auth_resolver_override_when_engine_has_none() { use crate::auth::ResolvedAuth; diff --git a/crates/plasm-runtime/src/http_resilience.rs b/crates/plasm-runtime/src/http_resilience.rs index 57b8ee36..04ac1e2a 100644 --- a/crates/plasm-runtime/src/http_resilience.rs +++ b/crates/plasm-runtime/src/http_resilience.rs @@ -15,7 +15,7 @@ use std::sync::Arc; use std::sync::OnceLock; use std::time::{Duration, Instant}; use tokio::sync::{Mutex, Semaphore, SemaphorePermit}; -use tracing::debug; +use tracing::{debug, Instrument}; /// Retry and concurrency policy for outbound HTTP. #[derive(Debug, Clone)] @@ -199,6 +199,38 @@ impl ResilientHttpTransport { } } } + + async fn run_with_retries( + &self, + url: &str, + method: &'static str, + host: &str, + started: Instant, + mut attempt_fn: F, + ) -> Result<(serde_json::Value, Option), RuntimeError> + where + F: FnMut() -> Fut, + Fut: std::future::Future>, + { + let mut attempt = 0u32; + async { + loop { + attempt += 1; + tracing::Span::current().record("attempt", attempt); + let outcome = attempt_fn().await; + match self + .process_attempt(url, method, host, attempt, started, outcome) + .await + { + Ok(Some(ok)) => break Ok(ok), + Ok(None) => continue, + Err(e) => break Err(e), + } + } + } + .instrument(crate::spans::http_retry()) + .await + } } fn finalize_retryable_failure( @@ -276,22 +308,12 @@ impl HttpTransport for ResilientHttpTransport { ) .await?; - let mut attempt = 0u32; - let result = loop { - attempt += 1; - let outcome = self - .inner - .compiled_http_attempt(base_url, request, auth.clone()) - .await; - match self - .process_attempt(&url, method, &host, attempt, started, outcome) - .await - { - Ok(Some(ok)) => break Ok(ok), - Ok(None) => continue, - Err(e) => break Err(e), - } - }; + let result = self + .run_with_retries(&url, method, &host, started, || { + self.inner + .compiled_http_attempt(base_url, request, auth.clone()) + }) + .await; let elapsed = started.elapsed(); crate::runtime_metrics::record_outbound_http_request(method, &url, result.is_ok(), elapsed); crate::live_run_telemetry::record_live_http_trace( @@ -318,19 +340,11 @@ impl HttpTransport for ResilientHttpTransport { ) .await?; - let mut attempt = 0u32; - let result = loop { - attempt += 1; - let outcome = self.inner.absolute_get_attempt(url, auth.clone()).await; - match self - .process_attempt(url, "GET", &host, attempt, started, outcome) - .await - { - Ok(Some(ok)) => break Ok(ok), - Ok(None) => continue, - Err(e) => break Err(e), - } - }; + let result = self + .run_with_retries(url, "GET", &host, started, || { + self.inner.absolute_get_attempt(url, auth.clone()) + }) + .await; let elapsed = started.elapsed(); crate::runtime_metrics::record_outbound_http_request("GET", url, result.is_ok(), elapsed); crate::live_run_telemetry::record_live_http_trace( diff --git a/crates/plasm-runtime/src/lib.rs b/crates/plasm-runtime/src/lib.rs index 860e26f9..8922a73f 100644 --- a/crates/plasm-runtime/src/lib.rs +++ b/crates/plasm-runtime/src/lib.rs @@ -123,6 +123,7 @@ pub mod paginated_collect; pub mod preflight; pub mod query_index; pub mod replay; +pub mod row_compute; pub mod row_predicate; pub mod runtime_error_render; pub mod session_graph_cache; @@ -149,6 +150,8 @@ pub use view_preflight::{ mod cancel_signal; mod live_run_telemetry; mod runtime_metrics; +#[cfg(test)] +mod span_graph_tests; mod spans; pub use api_error_detail::{ @@ -198,6 +201,7 @@ pub use oauth_client::{ pub use oauth_token_debug::TokenEndpointResponseSummary; pub use query_index::{QueryCacheKey, QueryIndex}; pub use replay::*; +pub use row_compute::{eval_compute_ops, ComputeEvalOutcome, PolarsAdapter}; pub use row_predicate::{ json_matches_predicate, json_predicate_matches, JsonRowPredicate, JsonRowPredicateOp, }; diff --git a/crates/plasm-runtime/src/row_compute/adapter.rs b/crates/plasm-runtime/src/row_compute/adapter.rs new file mode 100644 index 00000000..f237d0f2 --- /dev/null +++ b/crates/plasm-runtime/src/row_compute/adapter.rs @@ -0,0 +1,113 @@ +//! Three sync engine ports. Polars types do not escape this module. + +use super::eval::apply_stored_plan; +use super::json_frame::{collect_json, ingest_json_rows, FrameState}; +use indexmap::IndexMap; +use plasm_core::{ + CollectReason, CollectRows, CollectedFrame, CompileRowPlan, EnginePlanId, FrameId, IngestBatch, + IngestRows, PlasmFrameSchema, RowComputeError, RowPlan, ScanError, ScanSource, Value, +}; +use std::cell::{Cell, RefCell}; +use std::collections::HashMap; + +/// Polars-backed row engine. Handles are session-local and never stored on `PlasmComp`. +pub struct PolarsAdapter { + frames: RefCell>, + plans: RefCell>, + next_frame: Cell, + next_engine: Cell, +} + +impl Default for PolarsAdapter { + fn default() -> Self { + Self::new() + } +} + +impl PolarsAdapter { + #[must_use] + pub fn new() -> Self { + Self { + frames: RefCell::new(HashMap::new()), + plans: RefCell::new(HashMap::new()), + next_frame: Cell::new(1), + next_engine: Cell::new(1), + } + } + + fn json_from_values(rows: &[IndexMap]) -> Vec { + rows.iter() + .map(|row| { + let mut map = serde_json::Map::new(); + for (k, v) in row { + map.insert(k.clone(), plasm_core::plasm_value_to_json(v)); + } + serde_json::Value::Object(map) + }) + .collect() + } +} + +impl IngestRows for PolarsAdapter { + fn ingest( + &mut self, + _source: &ScanSource, + batch: IngestBatch<'_>, + ) -> Result { + let json_rows = Self::json_from_values(batch.rows); + let state = ingest_json_rows(&json_rows).map_err(|_| ScanError::UnboundFrame)?; + let id = FrameId::new(self.next_frame.get()); + self.next_frame.set(id.as_u64() + 1); + self.frames.borrow_mut().insert(id, state); + Ok(id) + } +} + +impl CompileRowPlan for PolarsAdapter { + fn compile(&self, plan: &RowPlan) -> Result { + let id = EnginePlanId::new(self.next_engine.get()); + self.next_engine.set(id.as_u64() + 1); + self.plans.borrow_mut().insert(id, plan.clone()); + Ok(id) + } +} + +impl CollectRows for PolarsAdapter { + fn collect( + &self, + id: EnginePlanId, + _reason: CollectReason, + ) -> Result { + let plans = self.plans.borrow(); + let plan = plans.get(&id).ok_or(ScanError::UnboundFrame)?; + let frames = self.frames.borrow(); + let mut state = frames + .get(&plan.source()) + .cloned() + .ok_or(ScanError::UnboundFrame)?; + drop(frames); + apply_stored_plan(plan, &mut state).map_err(|_| ScanError::UnboundFrame)?; + let rows_json = collect_json(&state).map_err(|_| ScanError::UnboundFrame)?; + let rows = rows_json + .into_iter() + .map(|v| match v { + serde_json::Value::Object(map) => map + .into_iter() + .map(|(k, val)| (k, plasm_core::json_value_to_plasm_value(&val))) + .collect(), + other => { + let mut m = IndexMap::new(); + m.insert( + "value".into(), + plasm_core::json_value_to_plasm_value(&other), + ); + m + } + }) + .collect(); + Ok(CollectedFrame { + schema: PlasmFrameSchema::opaque_object(), + rows, + }) + } +} diff --git a/crates/plasm-runtime/src/row_compute/eval.rs b/crates/plasm-runtime/src/row_compute/eval.rs new file mode 100644 index 00000000..81ce674b --- /dev/null +++ b/crates/plasm-runtime/src/row_compute/eval.rs @@ -0,0 +1,382 @@ +//! Apply a fused [`RowPlan`] on an ingested frame. + +use super::json_frame::{collect_json, ingest_json_rows, ColKind, FrameState}; +use super::money::finalize_money_sums; +use super::nodes::apply_node; +use chrono::Utc; +use plasm_core::plasm_monad::{ComputeOp, WithExpr}; +use plasm_core::{ + fold_compute_ops, CollectCardinality, CollectReason, FrameId, PlanNode, RowPlan, StepId, + TypedAggregate, +}; +use polars::prelude::*; + +/// Engine collect before host Minijinja (Render is not a PlanNode). +#[derive(Debug, Clone)] +pub enum ComputeEvalOutcome { + Rows(Vec), + Render { + rows: Vec, + columns: Vec, + column_aliases: std::collections::BTreeMap, + template: String, + collection_alias: Option, + render_bindings: Vec, + }, +} + +pub fn eval_compute_ops( + ops: &[ComputeOp], + rows: &[serde_json::Value], +) -> Result { + let step = StepId::new("row").map_err(|e| e.to_string())?; + let plan = fold_compute_ops(ops, FrameId::new(1), step, CollectCardinality::List) + .map_err(|e| e.to_string())?; + let mut state = ingest_json_rows(rows).map_err(|e| e.to_string())?; + apply_stored_plan(&plan, &mut state).map_err(|e| e.to_string())?; + let collected = collect_json(&state).map_err(|e| e.to_string())?; + match plan.collect() { + CollectReason::Render { spec, .. } => Ok(ComputeEvalOutcome::Render { + rows: collected, + columns: spec.columns.clone(), + column_aliases: spec.column_aliases.clone(), + template: spec.template.clone(), + collection_alias: spec.collection_alias.clone(), + render_bindings: spec.render_bindings.clone(), + }), + _ => Ok(ComputeEvalOutcome::Rows(collected)), + } +} + +pub(super) fn apply_stored_plan(plan: &RowPlan, state: &mut FrameState) -> PolarsResult<()> { + let now = Utc::now(); + ensure_plan_columns(plan, state)?; + let mut lf = state.df.clone().lazy(); + for (_, node) in plan.nodes().iter() { + lf = apply_node(lf, node, state, now)?; + } + state.df = lf.collect()?; + finalize_money_sums(state)?; + Ok(()) +} + +fn ensure_plan_columns(plan: &RowPlan, state: &mut FrameState) -> PolarsResult<()> { + let mut names = Vec::new(); + for (_, node) in plan.nodes().iter() { + collect_node_columns(node, &mut names); + } + let height = state.df.height(); + for name in names { + if state.df.column(&name).is_ok() { + continue; + } + let series = Series::full_null(PlSmallStr::from_str(&name), height, &DataType::Null); + state.df.with_column(series)?; + if !state.visible.iter().any(|v| v == &name) { + state.visible.push(name.clone()); + } + state.kinds.entry(name).or_insert(ColKind::Json); + } + Ok(()) +} + +fn collect_node_columns(node: &PlanNode, names: &mut Vec) { + match node { + PlanNode::Filter(filter) => { + for p in filter.predicates() { + names.push(p.field_path.dotted()); + } + } + PlanNode::Sort { key, .. } => names.push(key.dotted()), + PlanNode::GroupBy { keys, aggs } => { + names.extend(keys.iter().map(|k| k.dotted())); + for agg in aggs { + collect_agg_columns(agg, names); + } + } + PlanNode::Aggregate { aggs } => { + for agg in aggs { + collect_agg_columns(agg, names); + } + } + PlanNode::Project(spec) => { + names.extend(spec.fields.values().map(|p| p.dotted())); + } + PlanNode::With { columns } => { + for col in columns { + collect_with_columns(&col.expr, names); + } + } + PlanNode::Limit { .. } => {} + PlanNode::Dedupe { keys } | PlanNode::Distinct { keys } => { + names.extend(keys.iter().map(|k| k.dotted())); + } + } +} + +fn collect_agg_columns(agg: &TypedAggregate, names: &mut Vec) { + match agg { + TypedAggregate::Count { .. } => {} + TypedAggregate::Numeric { field, .. } | TypedAggregate::MoneySum { field, .. } => { + names.push(field.dotted()); + } + } +} + +fn collect_with_columns(expr: &WithExpr, names: &mut Vec) { + match expr { + WithExpr::Field(p) | WithExpr::Len { field: p } => { + names.push(p.dotted()); + } + WithExpr::Literal(_) | WithExpr::Now => {} + WithExpr::Arith { lhs, rhs, .. } => { + collect_with_columns(lhs, names); + collect_with_columns(rhs, names); + } + WithExpr::When { + lhs, + rhs, + then, + else_, + .. + } => { + collect_with_columns(lhs, names); + collect_with_columns(rhs, names); + collect_with_columns(then, names); + collect_with_columns(else_, names); + } + } +} + +#[cfg(test)] +mod tests { + use super::{eval_compute_ops, ComputeEvalOutcome}; + use plasm_core::parse_with_body; + use plasm_core::plasm_monad::{ComputeOp, FieldPath, PlanPredicateOp, PlasmDataValue}; + use rust_decimal::Decimal; + use std::str::FromStr; + + #[test] + fn filter_sort_limit_roundtrip() { + let rows = vec![ + serde_json::json!({"owner":"alice","score":10}), + serde_json::json!({"owner":"bob","score":30}), + serde_json::json!({"owner":"alice","score":20}), + ]; + let pred = plasm_core::PlanPredicate { + field_path: FieldPath::from_dotted("owner").unwrap(), + op: PlanPredicateOp::Eq, + value: PlasmDataValue::Literal { + value: serde_json::json!("alice"), + }, + }; + let ops = vec![ + ComputeOp::Filter { + predicates: vec![pred], + }, + ComputeOp::Sort { + key: FieldPath::from_dotted("score").unwrap(), + descending: true, + }, + ComputeOp::Limit { count: 1 }, + ]; + let ComputeEvalOutcome::Rows(out) = eval_compute_ops(&ops, &rows).unwrap() else { + panic!("rows"); + }; + assert_eq!(out.len(), 1); + assert_eq!(out[0]["score"], serde_json::json!(20)); + } + + #[test] + fn with_mul_adds_column() { + let rows = vec![serde_json::json!({"quantity": 2, "price": 5})]; + let columns = parse_with_body("notional: quantity * price").unwrap(); + let ops = vec![ComputeOp::With { columns }]; + let ComputeEvalOutcome::Rows(out) = eval_compute_ops(&ops, &rows).unwrap() else { + panic!("rows"); + }; + assert_eq!(out[0]["notional"], serde_json::json!(10)); + assert_eq!(out[0]["quantity"], serde_json::json!(2)); + } + + #[test] + fn with_now_minus_field_is_nonnegative_int_days() { + let rows = vec![ + serde_json::json!({"id": "old", "updated_at": "2020-01-01T00:00:00Z"}), + serde_json::json!({"id": "new", "updated_at": "2024-06-01T00:00:00Z"}), + ]; + let columns = parse_with_body("age_days: (now - updated_at)").unwrap(); + let ops = vec![ComputeOp::With { columns }]; + let ComputeEvalOutcome::Rows(out) = eval_compute_ops(&ops, &rows).unwrap() else { + panic!("rows"); + }; + let older = out.iter().find(|r| r["id"] == "old").unwrap(); + let newer = out.iter().find(|r| r["id"] == "new").unwrap(); + let age_old = older["age_days"].as_i64().expect("age int"); + let age_new = newer["age_days"].as_i64().expect("age int"); + assert!(age_old >= 0 && age_new >= 0, "ages {age_old} {age_new}"); + assert!( + age_old > age_new, + "older row must have larger age: {age_old} vs {age_new}" + ); + } + + #[test] + fn with_field_minus_field_is_int_days() { + let rows = vec![serde_json::json!({ + "created_at": "2020-01-01T00:00:00Z", + "updated_at": "2020-01-11T00:00:00Z", + })]; + let columns = parse_with_body("cycle: (updated_at - created_at)").unwrap(); + let ComputeEvalOutcome::Rows(out) = + eval_compute_ops(&[ComputeOp::With { columns }], &rows).unwrap() + else { + panic!("rows"); + }; + assert_eq!(out[0]["cycle"], serde_json::json!(10)); + } + + #[test] + fn with_div_is_float() { + let rows = vec![serde_json::json!({"quantity": 10, "price": 4})]; + let columns = parse_with_body("rate: quantity / price").unwrap(); + let ComputeEvalOutcome::Rows(out) = + eval_compute_ops(&[ComputeOp::With { columns }], &rows).unwrap() + else { + panic!("rows"); + }; + assert_eq!(out[0]["rate"].as_f64().unwrap(), 2.5); + } + + #[test] + fn with_string_plus_concat() { + let rows = vec![serde_json::json!({"first": "al", "last": "ice"})]; + let columns = parse_with_body("name: first + last").unwrap(); + let ComputeEvalOutcome::Rows(out) = + eval_compute_ops(&[ComputeOp::With { columns }], &rows).unwrap() + else { + panic!("rows"); + }; + assert_eq!(out[0]["name"], serde_json::json!("alice")); + } + + #[test] + fn with_when_len_and_temporal_cmp() { + let rows = vec![ + serde_json::json!({ + "title": "", + "created_at": "2020-01-01T00:00:00Z", + "updated_at": "2020-01-02T00:00:00Z", + }), + serde_json::json!({ + "title": "ok", + "created_at": "2020-01-01T00:00:00Z", + "updated_at": "2020-01-20T00:00:00Z", + }), + ]; + let columns = parse_with_body( + "blank: when(len(title)=0, 1, 0), long: when(updated_at - created_at > 5, 1, 0)", + ) + .unwrap(); + let ComputeEvalOutcome::Rows(out) = + eval_compute_ops(&[ComputeOp::With { columns }], &rows).unwrap() + else { + panic!("rows"); + }; + assert_eq!(out[0]["blank"], serde_json::json!(1)); + assert_eq!(out[0]["long"], serde_json::json!(0)); + assert_eq!(out[1]["blank"], serde_json::json!(0)); + assert_eq!(out[1]["long"], serde_json::json!(1)); + } + + #[test] + fn with_when_now_minus_gt() { + let rows = vec![ + serde_json::json!({"id": "old", "updated_at": "2020-01-01T00:00:00Z"}), + serde_json::json!({"id": "future", "updated_at": "2099-01-01T00:00:00Z"}), + ]; + let columns = parse_with_body("stale: when(now - updated_at > 14, 1, 0)").unwrap(); + let ComputeEvalOutcome::Rows(out) = + eval_compute_ops(&[ComputeOp::With { columns }], &rows).unwrap() + else { + panic!("rows"); + }; + let old = out.iter().find(|r| r["id"] == "old").unwrap(); + let future = out.iter().find(|r| r["id"] == "future").unwrap(); + assert_eq!(old["stale"], serde_json::json!(1)); + assert_eq!(future["stale"], serde_json::json!(0)); + } + + #[test] + fn group_by_count() { + let rows = vec![ + serde_json::json!({"owner":"a","score":1}), + serde_json::json!({"owner":"a","score":2}), + serde_json::json!({"owner":"b","score":3}), + ]; + let ops = vec![ComputeOp::GroupBy { + keys: vec![FieldPath::from_dotted("owner").unwrap()], + aggregates: vec![plasm_core::AggregateSpec { + name: plasm_core::OutputName::new("n").unwrap(), + function: plasm_core::AggregateFunction::Count, + field: None, + }], + }]; + let ComputeEvalOutcome::Rows(out) = eval_compute_ops(&ops, &rows).unwrap() else { + panic!("rows"); + }; + assert_eq!(out.len(), 2); + } + + #[test] + fn money_sum_same_currency() { + let rows = vec![ + serde_json::json!({"symbol":"A","fee":{"__plasm_money":"1.50","currency":"USD"}}), + serde_json::json!({"symbol":"A","fee":{"__plasm_money":"2.50","currency":"USD"}}), + serde_json::json!({"symbol":"B","fee":{"__plasm_money":"4.00","currency":"USD"}}), + ]; + let ops = vec![ComputeOp::GroupBy { + keys: vec![FieldPath::from_dotted("symbol").unwrap()], + aggregates: vec![plasm_core::AggregateSpec { + name: plasm_core::OutputName::new("fees").unwrap(), + function: plasm_core::AggregateFunction::Sum, + field: Some(FieldPath::from_dotted("fee").unwrap()), + }], + }]; + let ComputeEvalOutcome::Rows(out) = eval_compute_ops(&ops, &rows).unwrap() else { + panic!("rows"); + }; + assert_eq!(out.len(), 2, "out={out:?}"); + let a = out.iter().find(|r| r["symbol"] == "A").unwrap(); + let got = a["fees"]["__plasm_money"].as_str().expect("money amount"); + assert_eq!( + Decimal::from_str(got).unwrap(), + Decimal::from_str("4.00").unwrap(), + "row={a:?}" + ); + assert_eq!(a["fees"]["currency"], "USD"); + assert!(a.get("__ccy_n").is_none()); + assert!(a.get("__ccy_n_fees").is_none()); + } + + #[test] + fn money_sum_rejects_cross_currency() { + let rows = vec![ + serde_json::json!({"symbol":"A","fee":{"__plasm_money":"1.00","currency":"USD"}}), + serde_json::json!({"symbol":"A","fee":{"__plasm_money":"1.00","currency":"EUR"}}), + ]; + let ops = vec![ComputeOp::GroupBy { + keys: vec![FieldPath::from_dotted("symbol").unwrap()], + aggregates: vec![plasm_core::AggregateSpec { + name: plasm_core::OutputName::new("fees").unwrap(), + function: plasm_core::AggregateFunction::Sum, + field: Some(FieldPath::from_dotted("fee").unwrap()), + }], + }]; + let err = eval_compute_ops(&ops, &rows).unwrap_err(); + assert!( + err.contains("currency") || err.contains("money"), + "expected cross-currency error, got {err}" + ); + } +} diff --git a/crates/plasm-runtime/src/row_compute/expressions.rs b/crates/plasm-runtime/src/row_compute/expressions.rs new file mode 100644 index 00000000..b464c0a0 --- /dev/null +++ b/crates/plasm-runtime/src/row_compute/expressions.rs @@ -0,0 +1,316 @@ +//! Lower row-plan predicates and `.with` expressions into Polars expressions. + +use super::json_frame::{col_expr, ColKind, FrameState, MONEY_AMOUNT, MONEY_CCY}; +use chrono::{DateTime, Utc}; +use plasm_core::plasm_monad::{ + FieldPath, PlanPredicate, PlanPredicateOp, PlasmDataValue, WithExpr, WithLiteral, +}; +use plasm_core::{normalize_temporal_value, ArithOp, TemporalWireFormat}; +use polars::prelude::*; + +pub(super) fn pred_expr(p: &PlanPredicate) -> PolarsResult { + let lhs = col_expr(&p.field_path); + let rhs = data_lit(&p.value)?; + Ok(match p.op { + PlanPredicateOp::Eq => lhs.eq(rhs), + PlanPredicateOp::Ne => lhs.neq(rhs), + PlanPredicateOp::Lt => lhs.lt(rhs), + PlanPredicateOp::Lte => lhs.lt_eq(rhs), + PlanPredicateOp::Gt => lhs.gt(rhs), + PlanPredicateOp::Gte => lhs.gt_eq(rhs), + PlanPredicateOp::Contains => lhs.cast(DataType::String).str().contains(rhs, false), + PlanPredicateOp::In => lhs.is_in(rhs), + PlanPredicateOp::Exists => lhs.is_not_null(), + }) +} + +fn data_lit(v: &PlasmDataValue) -> PolarsResult { + match v { + PlasmDataValue::Literal { value } => json_lit(value), + PlasmDataValue::Array { items } => { + let lits: Result, _> = items.iter().map(data_lit).collect(); + Ok(concat_list(lits?)?) + } + other => Err(PolarsError::ComputeError( + format!("unsupported row-filter value {other:?}").into(), + )), + } +} + +fn json_lit(v: &serde_json::Value) -> PolarsResult { + Ok(match v { + serde_json::Value::Null => lit(NULL), + serde_json::Value::Bool(b) => lit(*b), + serde_json::Value::Number(n) => { + if let Some(i) = n.as_i64() { + lit(i) + } else if let Some(f) = n.as_f64() { + lit(f) + } else { + lit(n.to_string()) + } + } + serde_json::Value::String(s) => lit(s.as_str()), + serde_json::Value::Array(items) => { + let lits: Result, _> = items.iter().map(json_lit).collect(); + concat_list(lits?)? + } + serde_json::Value::Object(_) => lit(v.to_string()), + }) +} + +pub(super) fn with_expr( + expr: &WithExpr, + state: &FrameState, + now: DateTime, +) -> PolarsResult { + match expr { + WithExpr::Field(path) => Ok(col_expr(path)), + WithExpr::Now => Ok(lit(now.to_rfc3339())), + WithExpr::Literal(litv) => Ok(match litv { + WithLiteral::Null => lit(NULL), + WithLiteral::Bool(b) => lit(*b), + WithLiteral::Integer(i) => lit(*i), + WithLiteral::Number(s) => { + if let Ok(i) = s.parse::() { + lit(i) + } else if let Ok(f) = s.parse::() { + lit(f) + } else { + lit(s.as_str()) + } + } + WithLiteral::String(s) => lit(s.as_str()), + }), + WithExpr::Arith { op, lhs, rhs } + if *op == ArithOp::Sub && is_temporal_sub(lhs, rhs, state) => + { + temporal_sub_days(now, lhs, rhs) + } + WithExpr::Arith { op, lhs, rhs } => { + let l_kind = infer_with_kind(lhs, state); + let r_kind = infer_with_kind(rhs, state); + let l = with_expr(lhs, state, now)?; + let r = with_expr(rhs, state, now)?; + arith_expr(*op, l, r, l_kind, r_kind) + } + WithExpr::Len { field } => Ok(col_expr(field) + .cast(DataType::String) + .str() + .len_chars() + .cast(DataType::Int64)), + WithExpr::When { + lhs, + op, + rhs, + then, + else_, + } => { + let l = with_expr(lhs, state, now)?; + let r = with_expr(rhs, state, now)?; + Ok(when(cmp_exprs(*op, l, r)) + .then(with_expr(then, state, now)?) + .otherwise(with_expr(else_, state, now)?)) + } + } +} + +fn is_now(expr: &WithExpr) -> bool { + matches!(expr, WithExpr::Now) +} + +fn is_temporal_operand(expr: &WithExpr, state: &FrameState) -> bool { + match expr { + WithExpr::Now => true, + WithExpr::Literal(WithLiteral::String(_)) => true, + WithExpr::Field(p) => matches!( + state.kinds.get(&p.dotted()), + Some(ColKind::Str | ColKind::Temporal) + ), + _ => false, + } +} + +fn is_temporal_sub(lhs: &WithExpr, rhs: &WithExpr, state: &FrameState) -> bool { + is_now(lhs) + || is_now(rhs) + || (is_temporal_operand(lhs, state) && is_temporal_operand(rhs, state)) +} + +fn utc_from_raw(raw: &str) -> Option> { + normalize_temporal_value( + plasm_core::Value::String(raw.to_string()), + TemporalWireFormat::Rfc3339, + ) + .ok() + .and_then(|v| match v { + plasm_core::Value::String(iso) => DateTime::parse_from_rfc3339(&iso) + .ok() + .map(|dt| dt.with_timezone(&Utc)), + _ => None, + }) +} + +const MS_PER_DAY: i64 = 86_400_000; + +fn col_to_epoch_millis(field: &FieldPath) -> Expr { + col_expr(field).map( + move |s| { + let out: Vec> = match s.dtype() { + DataType::String => s + .str() + .map(|ca| { + ca.into_iter() + .map(|opt| { + opt.and_then(|raw| { + utc_from_raw(raw).map(|dt| dt.timestamp_millis()) + }) + }) + .collect() + }) + .unwrap_or_default(), + _ => vec![None; s.len()], + }; + Ok(Some(Column::new(s.name().clone(), out))) + }, + GetOutput::from_type(DataType::Int64), + ) +} + +fn temporal_millis_expr(expr: &WithExpr, now: DateTime) -> PolarsResult { + match expr { + WithExpr::Now => Ok(lit(now.timestamp_millis())), + WithExpr::Field(field) => Ok(col_to_epoch_millis(field)), + WithExpr::Literal(WithLiteral::String(s)) => Ok(match utc_from_raw(s) { + Some(dt) => lit(dt.timestamp_millis()), + None => lit(NULL).cast(DataType::Int64), + }), + _ => Err(PolarsError::ComputeError( + "temporal subtraction requires temporal fields or `now`".into(), + )), + } +} + +fn temporal_sub_days(now: DateTime, lhs: &WithExpr, rhs: &WithExpr) -> PolarsResult { + let l = temporal_millis_expr(lhs, now)?; + let r = temporal_millis_expr(rhs, now)?; + Ok((l - r) / lit(MS_PER_DAY)) +} + +fn cmp_exprs(op: PlanPredicateOp, l: Expr, r: Expr) -> Expr { + match op { + PlanPredicateOp::Eq => l.eq(r), + PlanPredicateOp::Ne => l.neq(r), + PlanPredicateOp::Lt => l.lt(r), + PlanPredicateOp::Lte => l.lt_eq(r), + PlanPredicateOp::Gt => l.gt(r), + PlanPredicateOp::Gte => l.gt_eq(r), + PlanPredicateOp::Contains => l.cast(DataType::String).str().contains(r, false), + PlanPredicateOp::In => l.is_in(r), + PlanPredicateOp::Exists => l.is_not_null(), + } +} + +fn arith_expr( + op: ArithOp, + l: Expr, + r: Expr, + l_kind: ColKind, + r_kind: ColKind, +) -> PolarsResult { + let money_l = l_kind == ColKind::Money; + let money_r = r_kind == ColKind::Money; + if !money_l && !money_r { + let string_add = op == ArithOp::Add + && (l_kind == ColKind::Str || r_kind == ColKind::Str) + && l_kind != ColKind::Temporal + && r_kind != ColKind::Temporal; + if string_add { + return Ok(l.cast(DataType::String) + r.cast(DataType::String)); + } + let coerce = op == ArithOp::Div + || matches!(l_kind, ColKind::Str | ColKind::Json | ColKind::Float) + || matches!(r_kind, ColKind::Str | ColKind::Json | ColKind::Float); + let l = if coerce { l.cast(DataType::Float64) } else { l }; + let r = if coerce { r.cast(DataType::Float64) } else { r }; + return Ok(match op { + ArithOp::Add => l + r, + ArithOp::Sub => l - r, + ArithOp::Mul => l * r, + ArithOp::Div => l / r, + }); + } + + let l_amt = if money_l { + l.clone() + .struct_() + .field_by_name(MONEY_AMOUNT) + .cast(DataType::Decimal(Some(38), Some(8))) + } else { + l.clone().cast(DataType::Decimal(Some(38), Some(8))) + }; + let r_amt = if money_r { + r.clone() + .struct_() + .field_by_name(MONEY_AMOUNT) + .cast(DataType::Decimal(Some(38), Some(8))) + } else { + r.clone().cast(DataType::Decimal(Some(38), Some(8))) + }; + let amount = match op { + ArithOp::Add => l_amt + r_amt, + ArithOp::Sub => l_amt - r_amt, + ArithOp::Mul => l_amt * r_amt, + ArithOp::Div => l_amt / r_amt, + }; + let ccy = if money_l { + l.struct_().field_by_name(MONEY_CCY) + } else { + r.struct_().field_by_name(MONEY_CCY) + }; + Ok(as_struct(vec![ + amount.cast(DataType::String).alias(MONEY_AMOUNT), + ccy.alias(MONEY_CCY), + ])) +} + +pub(super) fn infer_with_kind(expr: &WithExpr, state: &FrameState) -> ColKind { + match expr { + WithExpr::Field(p) => state + .kinds + .get(&p.dotted()) + .copied() + .unwrap_or(ColKind::Json), + WithExpr::Now => ColKind::Temporal, + WithExpr::Literal(WithLiteral::Bool(_)) => ColKind::Bool, + WithExpr::Literal(WithLiteral::Integer(_)) => ColKind::Int, + WithExpr::Literal(WithLiteral::Number(_)) => ColKind::Float, + WithExpr::Literal(WithLiteral::String(_)) => ColKind::Str, + WithExpr::Literal(WithLiteral::Null) => ColKind::Json, + WithExpr::Len { .. } => ColKind::Int, + WithExpr::Arith { op, lhs, rhs } => { + if *op == ArithOp::Sub && is_temporal_sub(lhs, rhs, state) { + return ColKind::Int; + } + let l = infer_with_kind(lhs, state); + let r = infer_with_kind(rhs, state); + if *op == ArithOp::Add + && (l == ColKind::Str || r == ColKind::Str) + && l != ColKind::Temporal + && r != ColKind::Temporal + && l != ColKind::Money + && r != ColKind::Money + { + return ColKind::Str; + } + if l == ColKind::Money || r == ColKind::Money { + ColKind::Money + } else if *op == ArithOp::Div || l == ColKind::Float || r == ColKind::Float { + ColKind::Float + } else { + l + } + } + WithExpr::When { then, .. } => infer_with_kind(then, state), + } +} diff --git a/crates/plasm-runtime/src/row_compute/json_frame.rs b/crates/plasm-runtime/src/row_compute/json_frame.rs new file mode 100644 index 00000000..1a169735 --- /dev/null +++ b/crates/plasm-runtime/src/row_compute/json_frame.rs @@ -0,0 +1,379 @@ +//! JSON object rows ↔ Polars DataFrame. Nested objects stay JSON strings; dotted paths +//! are extra columns used only for FieldPath access. + +use indexmap::IndexMap; +use plasm_core::money::MoneyValue; +use plasm_core::{json_value_to_plasm_value, Value}; +use polars::prelude::*; +use rust_decimal::Decimal; +use std::str::FromStr; + +pub(super) const IDX_COL: &str = "__plasm_idx"; +pub(super) const MONEY_AMOUNT: &str = "__amount"; +pub(super) const MONEY_CCY: &str = "__ccy"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum ColKind { + Bool, + Int, + Float, + Str, + Temporal, + Money, + Json, +} + +#[derive(Debug, Clone)] +pub(super) struct FrameState { + pub df: DataFrame, + pub visible: Vec, + pub kinds: IndexMap, + /// Output names of money `sum` aggregates pending reconstruct + currency check. + pub money_sum_names: Vec, +} + +pub(super) fn ingest_json_rows(rows: &[serde_json::Value]) -> PolarsResult { + let mut visible = Vec::new(); + let mut seen = std::collections::HashSet::new(); + for row in rows { + if let serde_json::Value::Object(map) = row { + for k in map.keys() { + if seen.insert(k.clone()) { + visible.push(k.clone()); + } + } + } + } + let mut kinds: IndexMap = IndexMap::new(); + let mut columns: IndexMap>> = IndexMap::new(); + for key in &visible { + columns.insert(key.clone(), Vec::with_capacity(rows.len())); + } + let mut extra: IndexMap>> = IndexMap::new(); + + for (i, row) in rows.iter().enumerate() { + let obj = match row { + serde_json::Value::Object(m) => m, + other => { + let mut m = serde_json::Map::new(); + m.insert("value".into(), other.clone()); + for key in &visible { + let cell = json_to_any(m.get(key).unwrap_or(&serde_json::Value::Null)); + columns.get_mut(key).unwrap().push(cell); + } + flatten_into("", other, &mut extra, i, rows.len()); + continue; + } + }; + for key in &visible { + let v = obj.get(key).unwrap_or(&serde_json::Value::Null); + let cell = json_to_any(v); + let kind = kind_of(&cell); + kinds + .entry(key.clone()) + .and_modify(|k| *k = promote(*k, kind)) + .or_insert(kind); + columns.get_mut(key).unwrap().push(cell); + flatten_into(key, v, &mut extra, i, rows.len()); + } + } + + let mut series: Vec = Vec::new(); + let idx: Vec = (0..rows.len() as u32).collect(); + series.push(Column::new(PlSmallStr::from_static(IDX_COL), idx)); + for (key, vals) in columns { + series.push(series_from_any(key.as_str(), vals, kinds.get(&key).copied())?.into()); + } + for (key, vals) in extra { + if visible.iter().any(|v| v == &key) { + continue; + } + let kind = vals.iter().find_map(|v| { + let k = kind_of(v); + if k == ColKind::Json && matches!(v, AnyValue::Null) { + None + } else { + Some(k) + } + }); + kinds + .entry(key.clone()) + .or_insert(kind.unwrap_or(ColKind::Json)); + series.push(series_from_any(key.as_str(), vals, kinds.get(&key).copied())?.into()); + } + Ok(FrameState { + df: DataFrame::new(series)?, + visible, + kinds, + money_sum_names: Vec::new(), + }) +} + +fn flatten_into( + prefix: &str, + v: &serde_json::Value, + extra: &mut IndexMap>>, + row_i: usize, + n: usize, +) { + let serde_json::Value::Object(map) = v else { + return; + }; + for (k, child) in map { + let path = if prefix.is_empty() { + k.clone() + } else { + format!("{prefix}.{k}") + }; + let slot = extra + .entry(path.clone()) + .or_insert_with(|| vec![AnyValue::Null; n]); + if slot.len() < n { + slot.resize(n, AnyValue::Null); + } + slot[row_i] = json_to_any(child); + flatten_into(&path, child, extra, row_i, n); + } +} + +fn json_to_any(v: &serde_json::Value) -> AnyValue<'static> { + match json_value_to_plasm_value(v) { + Value::Null => AnyValue::Null, + Value::Bool(b) => AnyValue::Boolean(b), + Value::Integer(i) => AnyValue::Int64(i), + Value::Float(f) => AnyValue::Float64(f), + Value::String(s) | Value::PhraseIdent(s) => AnyValue::StringOwned(s.into()), + Value::Money(m) => money_any(&m), + Value::Array(_) | Value::Object(_) | Value::UnionCtor { .. } | Value::PlasmInputRef(_) => { + AnyValue::StringOwned(v.to_string().into()) + } + } +} + +fn money_any(m: &MoneyValue) -> AnyValue<'static> { + let amount = m.amount().to_string(); + let ccy = m.currency().unwrap_or("").to_string(); + AnyValue::StructOwned(Box::new(( + vec![ + AnyValue::StringOwned(amount.into()), + AnyValue::StringOwned(ccy.into()), + ], + vec![ + Field::new(PlSmallStr::from_static(MONEY_AMOUNT), DataType::String), + Field::new(PlSmallStr::from_static(MONEY_CCY), DataType::String), + ], + ))) +} + +fn kind_of(v: &AnyValue<'_>) -> ColKind { + match v { + AnyValue::Null => ColKind::Json, + AnyValue::Boolean(_) => ColKind::Bool, + AnyValue::Int64(_) | AnyValue::Int32(_) | AnyValue::UInt32(_) | AnyValue::UInt64(_) => { + ColKind::Int + } + AnyValue::Float64(_) | AnyValue::Float32(_) => ColKind::Float, + AnyValue::StructOwned(_) | AnyValue::Struct(_, _, _) => ColKind::Money, + AnyValue::String(_) | AnyValue::StringOwned(_) => ColKind::Str, + _ => ColKind::Json, + } +} + +fn promote(a: ColKind, b: ColKind) -> ColKind { + if a == b { + return a; + } + if a == ColKind::Json { + return b; + } + if b == ColKind::Json { + return a; + } + match (a, b) { + (ColKind::Int, ColKind::Float) | (ColKind::Float, ColKind::Int) => ColKind::Float, + (ColKind::Money, _) | (_, ColKind::Money) => ColKind::Money, + _ => ColKind::Str, + } +} + +fn series_from_any( + name: &str, + vals: Vec>, + kind: Option, +) -> PolarsResult { + let name = PlSmallStr::from_str(name); + match kind.unwrap_or(ColKind::Json) { + ColKind::Bool => { + let data: Vec> = vals + .into_iter() + .map(|v| match v { + AnyValue::Boolean(b) => Some(b), + AnyValue::Null => None, + _ => None, + }) + .collect(); + Ok(Series::new(name, data)) + } + ColKind::Int => { + let data: Vec> = vals + .into_iter() + .map(|v| match v { + AnyValue::Int64(i) => Some(i), + AnyValue::Int32(i) => Some(i as i64), + AnyValue::UInt32(i) => Some(i as i64), + AnyValue::Null => None, + _ => None, + }) + .collect(); + Ok(Series::new(name, data)) + } + ColKind::Float => { + let data: Vec> = vals + .into_iter() + .map(|v| match v { + AnyValue::Float64(f) => Some(f), + AnyValue::Int64(i) => Some(i as f64), + AnyValue::Null => None, + _ => None, + }) + .collect(); + Ok(Series::new(name, data)) + } + ColKind::Str | ColKind::Json | ColKind::Temporal => { + let data: Vec> = vals + .into_iter() + .map(|v| match v { + AnyValue::Null => None, + AnyValue::StringOwned(s) => Some(s.as_str().to_string()), + AnyValue::String(s) => Some(s.to_string()), + AnyValue::Boolean(b) => Some(b.to_string()), + AnyValue::Int64(i) => Some(i.to_string()), + AnyValue::Float64(f) => Some(f.to_string()), + other => Some(other.to_string()), + }) + .collect(); + Ok(Series::new(name, data)) + } + ColKind::Money => Series::from_any_values_and_dtype( + name, + &vals, + &DataType::Struct(vec![ + Field::new(PlSmallStr::from_static(MONEY_AMOUNT), DataType::String), + Field::new(PlSmallStr::from_static(MONEY_CCY), DataType::String), + ]), + true, + ), + } +} + +pub(super) fn collect_json(state: &FrameState) -> PolarsResult> { + let df = &state.df; + let n = df.height(); + let mut out = Vec::with_capacity(n); + for row_idx in 0..n { + let mut map = serde_json::Map::new(); + for key in &state.visible { + if key == IDX_COL { + continue; + } + let Some(s) = df.column(key).ok() else { + continue; + }; + map.insert( + key.clone(), + any_to_json(s.get(row_idx)?, state.kinds.get(key).copied()), + ); + } + out.push(serde_json::Value::Object(map)); + } + Ok(out) +} + +fn any_to_json(v: AnyValue<'_>, kind: Option) -> serde_json::Value { + if kind == Some(ColKind::Money) { + if let Some(m) = money_from_any(&v) { + return money_tagged_json(&m); + } + } + match v { + AnyValue::Null => serde_json::Value::Null, + AnyValue::Boolean(b) => serde_json::Value::Bool(b), + AnyValue::Int64(i) => serde_json::json!(i), + AnyValue::Int32(i) => serde_json::json!(i), + AnyValue::UInt32(i) => serde_json::json!(i), + AnyValue::UInt64(i) => serde_json::json!(i), + AnyValue::Float64(f) => serde_json::Number::from_f64(f) + .map(serde_json::Value::Number) + .unwrap_or(serde_json::Value::Null), + AnyValue::Float32(f) => serde_json::Number::from_f64(f as f64) + .map(serde_json::Value::Number) + .unwrap_or(serde_json::Value::Null), + AnyValue::String(s) => parse_json_or_string(s), + AnyValue::StringOwned(s) => parse_json_or_string(s.as_str()), + AnyValue::StructOwned(boxed) => { + if let Some(m) = money_from_struct(&boxed.0, &boxed.1) { + return money_tagged_json(&m); + } + serde_json::Value::Null + } + other => serde_json::Value::String(other.to_string()), + } +} + +fn parse_json_or_string(s: &str) -> serde_json::Value { + let t = s.trim(); + if (t.starts_with('{') && t.ends_with('}')) || (t.starts_with('[') && t.ends_with(']')) { + if let Ok(v) = serde_json::from_str(s) { + return v; + } + } + serde_json::Value::String(s.to_string()) +} + +fn money_tagged_json(m: &MoneyValue) -> serde_json::Value { + let mut map = serde_json::Map::new(); + map.insert( + "__plasm_money".into(), + serde_json::Value::String(m.amount().to_string()), + ); + if let Some(c) = m.currency() { + map.insert("currency".into(), serde_json::Value::String(c.to_string())); + } + serde_json::Value::Object(map) +} + +fn money_from_any(v: &AnyValue<'_>) -> Option { + match v { + AnyValue::StructOwned(boxed) => money_from_struct(&boxed.0, &boxed.1), + _ => None, + } +} + +fn money_from_struct(vals: &[AnyValue<'_>], fields: &[Field]) -> Option { + let mut amount = None; + let mut ccy = None; + for (field, val) in fields.iter().zip(vals.iter()) { + match field.name().as_str() { + MONEY_AMOUNT => { + amount = match val { + AnyValue::String(s) => Decimal::from_str(s).ok(), + AnyValue::StringOwned(s) => Decimal::from_str(s.as_str()).ok(), + _ => None, + } + } + MONEY_CCY => { + ccy = match val { + AnyValue::String(s) if !s.is_empty() => Some(s.to_string()), + AnyValue::StringOwned(s) if !s.is_empty() => Some(s.as_str().to_string()), + _ => None, + } + } + _ => {} + } + } + Some(MoneyValue::new(amount?, ccy)) +} + +pub(super) fn col_expr(path: &plasm_core::FieldPath) -> Expr { + col(path.dotted()) +} diff --git a/crates/plasm-runtime/src/row_compute/mod.rs b/crates/plasm-runtime/src/row_compute/mod.rs new file mode 100644 index 00000000..7d3f9914 --- /dev/null +++ b/crates/plasm-runtime/src/row_compute/mod.rs @@ -0,0 +1,14 @@ +//! Polars adapter for fused [`plasm_core::RowPlan`] execute. +//! +//! Public types here do not re-export `polars::*`. [`ComputeOp`] stays the hashed constructor; +//! this module is the only physical engine. + +mod adapter; +mod eval; +mod expressions; +mod json_frame; +mod money; +mod nodes; + +pub use adapter::PolarsAdapter; +pub use eval::{eval_compute_ops, ComputeEvalOutcome}; diff --git a/crates/plasm-runtime/src/row_compute/money.rs b/crates/plasm-runtime/src/row_compute/money.rs new file mode 100644 index 00000000..e1f4cccd --- /dev/null +++ b/crates/plasm-runtime/src/row_compute/money.rs @@ -0,0 +1,115 @@ +//! Money aggregate lowering and reconstruction. + +use super::json_frame::{col_expr, ColKind, FrameState, MONEY_AMOUNT, MONEY_CCY}; +use plasm_core::FieldPath; +use polars::prelude::*; +use rust_decimal::Decimal; + +pub(super) fn push_money_sum( + agg_exprs: &mut Vec, + visible: &mut Vec, + state: &mut FrameState, + name: &str, + field: &FieldPath, +) { + let amount = col_expr(field) + .struct_() + .field_by_name(MONEY_AMOUNT) + .cast(DataType::Decimal(Some(38), Some(8))); + let ccy = col_expr(field).struct_().field_by_name(MONEY_CCY); + agg_exprs.push(ccy.clone().n_unique().alias(format!("__ccy_n_{name}"))); + agg_exprs.push(ccy.first().alias(format!("__ccy_{name}"))); + agg_exprs.push(amount.sum().alias(name)); + visible.push(name.to_string()); + state.kinds.insert(name.to_string(), ColKind::Money); + state.money_sum_names.push(name.to_string()); +} + +pub(super) fn finalize_money_sums(state: &mut FrameState) -> PolarsResult<()> { + let names = std::mem::take(&mut state.money_sum_names); + for name in names { + let n_col = format!("__ccy_n_{name}"); + let c_col = format!("__ccy_{name}"); + let n_unique = state.df.column(&n_col)?; + let ccys = state.df.column(&c_col)?; + for i in 0..state.df.height() { + let n = match n_unique.get(i)? { + AnyValue::UInt32(n) => n as u64, + AnyValue::UInt64(n) => n, + AnyValue::Int64(n) if n >= 0 => n as u64, + AnyValue::Int32(n) if n >= 0 => n as u64, + AnyValue::Null => 0, + other => { + return Err(PolarsError::ComputeError( + format!("unexpected currency-count dtype {other:?}").into(), + )) + } + }; + if n > 1 { + let left = match ccys.get(i)? { + AnyValue::String(s) => s.to_string(), + AnyValue::StringOwned(s) => s.as_str().to_string(), + _ => "left".into(), + }; + return Err(PolarsError::ComputeError( + format!("cannot compare money in {left} to money in another currency").into(), + )); + } + } + + let amounts = state.df.column(&name)?; + let mut encoded: Vec> = Vec::with_capacity(state.df.height()); + for i in 0..state.df.height() { + let amount = any_amount_string(amounts.get(i)?)?; + let ccy = match ccys.get(i)? { + AnyValue::String(s) => s.to_string(), + AnyValue::StringOwned(s) => s.as_str().to_string(), + AnyValue::Null => String::new(), + other => other.to_string(), + }; + let mut map = serde_json::Map::new(); + map.insert("__plasm_money".into(), serde_json::Value::String(amount)); + if !ccy.is_empty() { + map.insert("currency".into(), serde_json::Value::String(ccy)); + } + encoded.push(Some(serde_json::Value::Object(map).to_string())); + } + let series = Series::new(PlSmallStr::from_str(&name), encoded); + state.df.with_column(series)?; + let _ = state.df.drop_in_place(&n_col); + let _ = state.df.drop_in_place(&c_col); + state.kinds.insert(name, ColKind::Json); + } + Ok(()) +} + +fn any_amount_string(v: AnyValue<'_>) -> PolarsResult { + Ok(match v { + AnyValue::Decimal(unscaled, scale) => format_decimal_i128(unscaled, scale), + AnyValue::Float64(f) => trim_float(f), + AnyValue::Float32(f) => trim_float(f as f64), + AnyValue::Int64(i) => i.to_string(), + AnyValue::Int32(i) => i.to_string(), + AnyValue::String(s) => s.to_string(), + AnyValue::StringOwned(s) => s.as_str().to_string(), + AnyValue::Null => "0".into(), + other => { + return Err(PolarsError::ComputeError( + format!("cannot encode money amount from {other:?}").into(), + )) + } + }) +} + +fn format_decimal_i128(unscaled: i128, scale: usize) -> String { + Decimal::from_i128_with_scale(unscaled, scale as u32) + .normalize() + .to_string() +} + +fn trim_float(f: f64) -> String { + let d = Decimal::from_f64_retain(f) + .unwrap_or(Decimal::ZERO) + .normalize(); + d.to_string() +} diff --git a/crates/plasm-runtime/src/row_compute/nodes.rs b/crates/plasm-runtime/src/row_compute/nodes.rs new file mode 100644 index 00000000..3c720921 --- /dev/null +++ b/crates/plasm-runtime/src/row_compute/nodes.rs @@ -0,0 +1,127 @@ +//! Apply fused row-plan nodes to a Polars lazy frame. + +use super::expressions::{infer_with_kind, pred_expr, with_expr}; +use super::json_frame::{col_expr, ColKind, FrameState, IDX_COL}; +use super::money::push_money_sum; +use chrono::{DateTime, Utc}; +use plasm_core::row_plan::NumericAgg; +use plasm_core::{FieldPath, PlanNode, TypedAggregate}; +use polars::prelude::*; + +pub(super) fn apply_node( + lf: LazyFrame, + node: &PlanNode, + state: &mut FrameState, + now: DateTime, +) -> PolarsResult { + match node { + PlanNode::Filter(filter) => { + let mut e = lit(true); + for p in filter.predicates() { + e = e.and(pred_expr(p)?); + } + Ok(lf.filter(e)) + } + PlanNode::Sort { key, descending } => Ok(lf.sort( + [key.dotted()], + SortMultipleOptions::default() + .with_order_descending(*descending) + .with_nulls_last(true) + .with_maintain_order(true), + )), + PlanNode::Limit { count } => Ok(lf.slice(0, count.get() as u32)), + PlanNode::Dedupe { keys } | PlanNode::Distinct { keys } => { + let subset: Option> = if keys.is_empty() { + None + } else { + Some( + keys.iter() + .map(|k| PlSmallStr::from_string(k.dotted())) + .collect(), + ) + }; + Ok(lf.unique_stable(subset, UniqueKeepStrategy::First)) + } + PlanNode::Project(spec) => { + let mut exprs = vec![col(IDX_COL)]; + let mut visible = Vec::new(); + for (name, path) in &spec.fields { + exprs.push(col_expr(path).alias(name.as_str())); + visible.push(name.as_str().to_string()); + if let Some(k) = state.kinds.get(&path.dotted()).copied() { + state.kinds.insert(name.as_str().to_string(), k); + } + } + state.visible = visible; + Ok(lf.select(exprs)) + } + PlanNode::With { columns } => { + let mut exprs = Vec::new(); + for col_def in columns { + let e = with_expr(&col_def.expr, state, now)?; + let name = col_def.name.as_str(); + state.visible.push(name.to_string()); + state + .kinds + .insert(name.to_string(), infer_with_kind(&col_def.expr, state)); + exprs.push(e.alias(name)); + } + Ok(lf.with_columns(exprs)) + } + PlanNode::GroupBy { keys, aggs } => group_by_lf(lf, keys, aggs, state, true), + PlanNode::Aggregate { aggs } => group_by_lf(lf, &[], aggs, state, false), + } +} + +fn group_by_lf( + lf: LazyFrame, + keys: &[FieldPath], + aggs: &[TypedAggregate], + state: &mut FrameState, + grouped: bool, +) -> PolarsResult { + let mut agg_exprs = Vec::new(); + let mut visible = keys.iter().map(FieldPath::dotted).collect::>(); + for agg in aggs { + match agg { + TypedAggregate::Count { name } => { + agg_exprs.push(len().alias(name.as_str())); + visible.push(name.as_str().to_string()); + state.kinds.insert(name.as_str().to_string(), ColKind::Int); + } + TypedAggregate::Numeric { name, fn_, field } => { + if *fn_ == NumericAgg::Sum + && state.kinds.get(&field.dotted()) == Some(&ColKind::Money) + { + push_money_sum(&mut agg_exprs, &mut visible, state, name.as_str(), field); + } else { + let c = col_expr(field).cast(DataType::Float64); + let e = match fn_ { + NumericAgg::Sum => c.sum(), + NumericAgg::Avg => c.mean(), + NumericAgg::Min => c.min(), + NumericAgg::Max => c.max(), + NumericAgg::First => c.first(), + NumericAgg::Last => c.last(), + }; + agg_exprs.push(e.alias(name.as_str())); + visible.push(name.as_str().to_string()); + state + .kinds + .insert(name.as_str().to_string(), ColKind::Float); + } + } + TypedAggregate::MoneySum { name, field, .. } => { + push_money_sum(&mut agg_exprs, &mut visible, state, name.as_str(), field); + } + } + } + state.visible = visible; + if grouped { + Ok(lf + .group_by(keys.iter().map(|k| col(k.dotted())).collect::>()) + .agg(agg_exprs)) + } else { + Ok(lf.select(agg_exprs)) + } +} diff --git a/crates/plasm-runtime/src/span_graph_tests.rs b/crates/plasm-runtime/src/span_graph_tests.rs new file mode 100644 index 00000000..87beec8e --- /dev/null +++ b/crates/plasm-runtime/src/span_graph_tests.rs @@ -0,0 +1,4 @@ +//! Span-graph contracts for plasm-runtime. +//! +//! Live call-site lock lives next to the HTTP execute fixtures: +//! `execution::tests::execute_operation_parents_http_compiled_request_on_live_get`. diff --git a/crates/plasm-runtime/src/spans.rs b/crates/plasm-runtime/src/spans.rs index 340b053c..55599522 100644 --- a/crates/plasm-runtime/src/spans.rs +++ b/crates/plasm-runtime/src/spans.rs @@ -10,7 +10,7 @@ use tracing::Span; pub(crate) fn http_compiled_request(method: &'static str, url_len: usize) -> Span { tracing::debug_span!( "plasm_runtime.http.compiled_request", - http_method = method, + http.method = method, url_len = url_len, ) } @@ -21,6 +21,30 @@ pub(crate) fn http_absolute_get(url_len: usize) -> Span { tracing::debug_span!("plasm_runtime.http.absolute_get", url_len = url_len) } +/// Resilient retry loop around outbound HTTP (record `attempt` as the loop advances). +#[inline] +pub(crate) fn http_retry() -> Span { + tracing::debug_span!("plasm_runtime.http.retry", attempt = tracing::field::Empty,) +} + +/// Execute a query expression (materialize stream). +#[inline] +pub(crate) fn execute_query() -> Span { + tracing::debug_span!("plasm_runtime.execute.query") +} + +/// Execute a get expression. +#[inline] +pub(crate) fn execute_get() -> Span { + tracing::debug_span!("plasm_runtime.execute.get") +} + +/// Execute a compiled operation (HTTP / GraphQL / EVM). +#[inline] +pub(crate) fn execute_operation() -> Span { + tracing::debug_span!("plasm_runtime.execute.operation") +} + /// Hydration pass that invokes provider capabilities to fill projected fields. #[inline] pub(crate) fn projection_hydrate(entity_type: &str, provider_count: usize) -> Span { diff --git a/doc-site/docs/concepts.md b/doc-site/docs/concepts.md index 510d025e..3138a918 100644 --- a/doc-site/docs/concepts.md +++ b/doc-site/docs/concepts.md @@ -54,7 +54,7 @@ Agents write **Plasm** programs against symbols exposed in **teaching table** in Legacy opaque `p#` tokens for fields/params are **rejected** at parse. -Expressions compose with pipes and postfix transforms. Multi-line payloads use tagged **heredocs** — see the [Language definition](reference/plasm-language-definition.md). +Expressions compose with pipes and postfix transforms (`.filter`, `.with`, `.sort`, `.group_by`, …). Multi-line payloads use tagged **heredocs** — see the [Language definition](reference/plasm-language-definition.md). With the **`plasm`** remote client, the **client owns the monotonic symbol table** locally; the server executes expanded programs over HTTP. See [Remote terminal](reference/plasm-cgs-remote-terminal.md). diff --git a/doc-site/docs/crates/index.md b/doc-site/docs/crates/index.md index 5689094b..028abeb9 100644 --- a/doc-site/docs/crates/index.md +++ b/doc-site/docs/crates/index.md @@ -7,7 +7,7 @@ Workspace layout for **[plasm-core](https://github.com/PlasmTools/plasm-core)**. | [**plasm-core**](https://docs.rs/plasm-core) | CGS, AST, typecheck, discovery, teaching table rendering — **catalog-agnostic**. | | [**plasm-cml**](https://docs.rs/plasm-cml) | CML AST and transport parsing (shared with compile). | | [**plasm-compile**](https://docs.rs/plasm-compile) | Predicates, decoding, template validation. | -| [**plasm-runtime**](https://docs.rs/plasm-runtime) | Execution engine, cache, replay, auth resolution. | +| [**plasm-runtime**](https://docs.rs/plasm-runtime) | Execution engine, cache, replay, auth resolution; Polars-backed [`row_compute`](https://github.com/PlasmTools/plasm-core/tree/main/crates/plasm-runtime/src/row_compute) for fused [`RowPlan`](https://github.com/PlasmTools/plasm-core/tree/main/crates/plasm-core/src/row_plan) chains. | | [**plasm-agent-core**](https://docs.rs/plasm-agent-core) | MCP host, sessions, traces, MCP sqlx metadata, HTTP execute. | | [**plasm-server**](https://github.com/PlasmTools/plasm-core/tree/main/crates/plasm-server) | **OSS appliance** binary — in-process kernel + TUI. | | [**plasm**](https://docs.rs/plasm) | Remote terminal **`plasm`**, **`plasm-cgs`**, **`plasm-pack-catalogs`**. | diff --git a/doc-site/docs/glossary.md b/doc-site/docs/glossary.md index 80c4c7c7..2082ed6c 100644 --- a/doc-site/docs/glossary.md +++ b/doc-site/docs/glossary.md @@ -3,6 +3,8 @@ | Term | Meaning | |------|---------| | **CGS** | Capability Graph Schema — `domain.yaml` semantic model (entities, relations, capabilities; split catalogs use **`values:`** + **`value_ref`**). | +| **`.with`** | Row-compute postfix that adds derived columns per row (`.with{col: expr}`) while preserving entity identity — see [Row compute](reference/plasm-row-compute.md#derived-columns-with). | +| **RowPlan** | Fused execute-time IR for row compute (`plasm_core::row_plan`); Polars-backed evaluation in `plasm_runtime::row_compute`. | | **CML** | Capability Mapping Language — `mappings.yaml` wire templates. | | **teaching table** | Symbol-tuned teaching text (`e#` / `m#` / `r#` plus **wire names** for fields/params; `v#` gloss only) for agents. | | **view** | CGS **`views:`** entry — composed read-only DAG over existing capabilities (not MCP tenant “registry views”). | diff --git a/doc-site/docs/index.md b/doc-site/docs/index.md index fe95dd58..c0b41454 100644 --- a/doc-site/docs/index.md +++ b/doc-site/docs/index.md @@ -40,6 +40,7 @@ Details and edge cases live in the [Reference](reference/cli-and-env.md) section | First commands from source | [Start here](getting-started.md) | | Mental model + vocabulary | [Concepts](concepts.md) | | Language + heredocs | [Language definition](reference/plasm-language-definition.md) | +| Row compute (`.filter`, `.with`, …) | [Row compute](reference/plasm-row-compute.md) | | MCP sessions and `intent` | [MCP session reuse](reference/mcp-session-reuse.md) | | Full CLI/env index | [CLI & environment](reference/cli-and-env.md) | diff --git a/doc-site/docs/reference/plasm-language-definition.md b/doc-site/docs/reference/plasm-language-definition.md index c7aa9bb5..9ba263ff 100644 --- a/doc-site/docs/reference/plasm-language-definition.md +++ b/doc-site/docs/reference/plasm-language-definition.md @@ -59,7 +59,7 @@ Each entry in `comp.steps` is a tagged serde object (`kind` discriminant). Wire |--------|------|------------| | `invoke` | Read / action / view surface | `plan_kind`, `qualified_entity`, `ir` **xor** `ir_template`, `projection`, `predicates`, `page_size`, `approval` | | `pure` | Literal / artifact data | `data` (`PlasmDataValue`) | -| `map` | Row compute (filter, sort, group, …) | `compute` (`ComputeTemplate`) | +| `map` | Row compute (filter, sort, group, derived columns, …) | `compute` (`ComputeTemplate`) | | `derive` | Per-row map over a source | `derive` (`DeriveTemplate`: `source`, `item_binding`, `inputs`, `value`) | | `flat_map_relation` | Relation fanout (`>>=`) | `relation` (`PlanRelationTraversal`: `source`, `relation`, `target`, `ir`, `binding_proofs`, `materialize`) | | `flat_map_effect` | `for_each` side effects | `source`, `item_binding`, `effect_template`, `projection`, `predicates`, `approval` | @@ -184,7 +184,7 @@ Implementation: unified entity constructor head resolution in [`entity_ref_parse | Fetch filter | `e#{field=…}` | wire is the query param/filter for that entity+capability | | Search filter | `e#~"…"{field=…}` | wire is the **Search**-capability param (homograph-safe vs Create/Update params) | | Relation hop | `receiver.r#` (or wire) | `r#` resolves to a declared relation wire; a filter wire after `.` yields `RelationSegmentWrongRole` except LHS-binding coercion (see [Binding RHS shapes](#binding-rhs-shapes-label--)) | -| Projection / postfix | `[field,…]`, `.sort(field)`, `.group_by(field)`, … | wire names resolve to `rows:` field symbols under the row entity | +| Projection / postfix | `[field,…]`, `.sort(field)`, `.group_by(field)`, `.with{col: expr}`, `.dedupe(…)`, `.distinct(…)`, … | wire names resolve to `rows:` field symbols under the row entity | --- @@ -218,7 +218,7 @@ Cross-binding references (`${stats.content}`, `body=report.content`) are also su ## Invariants -1. **Transforms are core postfix syntax** — `.limit(n)`, `.sort(field, desc)` / `.sort(field,dir)` (whitespace direction sugar accepted), `.filter{…}` / `.filter(…)`, `.aggregate(…)`, `.group_by(field).aggregate(specs)` (primary), `.group_by(field, …)` (comma sugar), `.singleton()`, `.page_size(n)`, bracket projections `[field,…]`, and row-to-text template blocks (`<` on bindings (two uses only):** `source => { k: _.field }` (derive map) or `source => e1(…).update(…)` (for_each). There is no `.derive(…)` surface. Row-to-text uses postfix `rows <`. - **Relation fanout:** `labels = issues.labels` **or** `labels = issues.r#` (opaque relation symbol from teaching TSV) — never `issues => e2.r#` or `source => binding.r#` (compile rejects relation hops on `=>`). A **filter wire after `.`** on a receiver is not a relation hop (use `.r#` or the relation wire). The RHS of `=>` is not `plasm_expr`; entity calls there stringify or fail compile. - **Homograph wires:** query filters and relation hops may share a wire name (e.g. `labels`). In-grammar resolution at the nav position disambiguates: `receiver.r#` / `receiver.labels` is a relation hop; the same wire in `{…}` is a filter/param. Teaching exemplars prefer `.r#` or wire names in relation position. @@ -247,7 +248,7 @@ A binding `label = E` names the plan node produced by `E`. **`label` and `E` are - `label.r#` ≡ `E.r#` (relation hop) - **`label.m#(…)` / `label.(…)` ≡ `E.m#(…)`** (method invoke on the bound row) -- Postfix on row lists (`label.filter{…}`, `[field,…]`) when row-preserving +- Postfix on row lists (`label.filter{…}`, `label.with{…}`, `[field,…]`) when row-preserving Side-effect invokes on **plural** bindings are rejected — use `rows => e#.m#(param=_.…)` or `.limit(1)` / `.singleton()` first. **`.content`** applies only to row-to-text **Render** bindings, not plain string/data bindings. @@ -256,6 +257,7 @@ Binding forms: | Form | Example | Lowers to | |------|---------|-----------| | Surface + postfix | `issues = e1{…}.page_size(100)` | Query / get + compute | +| Derived columns | `stale = issues.with{age_days: (now - updated_at)}` | Row compute [`ComputeOp::With`](https://github.com/PlasmTools/plasm-core/blob/main/crates/plasm-core/src/plasm_monad/payload/with_expr.rs) — preserves entity identity | | Relation hop | `labels = issues.labels` or `labels = issues.r#` | `RelationTraversal` (per-row fanout when parent is plural) | | Method invoke | `out = repo.m#(…)` when `repo = e#(…)` | Same invoke IR as `e#(…).m#(…)` | | Derive map | `cards = rows => { t: _.title }` | `Derive` (`value_or_template` only) | @@ -321,7 +323,8 @@ Surface scanning lives in **`plasm-oss/crates/plasm-core/src/expr_parser/`**: | [`program_surface.rs`](https://github.com/PlasmTools/plasm-core/blob/main/crates/plasm-core/src/expr_parser/program_surface.rs) | Physical-line merging across heredocs (`collect_program_statement_lines`), `;;` stripping, top-level comma/`=>` splitting (`split_top_level`, `split_token_top_level`), binding `=` splitting (`split_assignment_at_top_level` / `split_assignment_for_binding`), program label validation. | | [`predicate_surface.rs`](https://github.com/PlasmTools/plasm-core/blob/main/crates/plasm-core/src/expr_parser/predicate_surface.rs) | Query `{…}` predicate list: same comma splitting as `split_top_level`, plus quote/heredoc-aware comparison-operator scan for [`expr_correction`](https://github.com/PlasmTools/plasm-core/blob/main/crates/plasm-core/src/expr_correction/mod.rs) (no duplicate lexer). | | [`program.rs`](https://github.com/PlasmTools/plasm-core/blob/main/crates/plasm-core/src/expr_parser/program.rs) | Optional shape AST: bindings + postfix-peeled primaries (`parse_program_shape`). Does not attach CGS typing. | -| [`postfix.rs`](https://github.com/PlasmTools/plasm-core/blob/main/crates/plasm-core/src/expr_parser/postfix.rs) | Postfix peel (`.limit`, `.sort`, `[projection]`, row-to-text `<14}`. **Dry-live parity (invariants 6–10)** diff --git a/doc-site/docs/reference/plasm-row-compute.md b/doc-site/docs/reference/plasm-row-compute.md index 49e1e1c1..6cd1f58b 100644 --- a/doc-site/docs/reference/plasm-row-compute.md +++ b/doc-site/docs/reference/plasm-row-compute.md @@ -9,7 +9,7 @@ See also [plasm-language-definition.md](plasm-language-definition.md) for full g | Plane | Surface | When to use | |-------|---------|-------------| | **Catalog** | `e1{state="open"}` on a query/get | Reduce data at the API; predicates become query parameters or CML filters. | -| **Row** | `rows.filter{owner="alice"}` or `rows.filter(owner="alice")` | Filter, sort, group, or aggregate rows already fetched into the session artifact. | +| **Row** | `rows.filter{owner="alice"}` or `rows.filter(owner="alice")` or `rows.with{age: (now - updated_at)}` | Filter, derive columns, sort, group, or aggregate rows already fetched into the session artifact. | Use catalog filters when the API supports them and you want fewer round-trips. Use row filters when refining a binding, combining results from multiple steps, or when the field is not a query parameter. @@ -45,17 +45,65 @@ by_team = LangItem.group_by(owner, team, n=count, total=sum(score)) **aggregate** without a key applies functions over all rows: `all = items.aggregate(n=count)`. +## Derived columns (`.with`) + +Add computed columns to each row while keeping the upstream **entity identity** (relation-dot continuation still works on the binding): + +```text +stale = issues.with{age_days: (now - updated_at)} +boosted = items.with{boost: score * 2} +tagged = items.with{tag: owner + owner} +labeled = items.with{label: when(len(owner)>0, owner, title)} +``` + +**Surface:** `.with{col: expr, …}` or `.with(col: expr, …)` — comma-separated `name: expr` pairs inside the braces/parens. Column names are output labels (wire-style identifiers); expressions reference catalog **field wire names** on the current row. + +**Expression language (v1):** + +| Form | Meaning | +|------|---------| +| `field` / `parent.child` | Field path on the row | +| `null`, `true`, `false`, integer, float, `"text"` | Literals | +| `now` | Catalog-plane UTC clock (not a field lookup — a catalog field named `now` is shadowed) | +| `a + b`, `a - b`, `a * b`, `a / b` | Arithmetic (`+` also concatenates strings) | +| `len(field)` | String length | +| `when(lhs op rhs, then, else)` | Conditional; `op` is `=`, `!=`, `>`, `<`, `>=`, `<=` | + +**Temporal subtraction:** `(now - updated_at)` or `(updated_at - created_at)` yields a non-negative **integer day count** when operands are temporal fields or `now`. Use these in filters or further `.with` columns (e.g. `when(now - updated_at > 14, 1, 0)`). + +**Money:** `*` / `/` / `+` / `-` on money columns follow catalog money typing (same-currency rules; cross-currency arithmetic fails at runtime). + +**Disambiguation:** `.with{` / `.with(` is row compute. Identifiers such as `.join(…)` or `.open(…)` without a leading row-compute verb are **not** postfix operators — they remain path/relation surface and fail row-compute lowering. + +**Operator precedence (v1):** inside each expression, `*` and `/` bind tighter than `+` and `-` (e.g. `score * 2 + 1` is `(score * 2) + 1`). Parentheses override. + +## Dedupe and distinct + +Remove duplicate rows while preserving order (first occurrence wins): + +```text +unique = items.dedupe(owner) +unique_by_pair = items.dedupe(owner, team) +all_distinct = items.distinct() +``` + +- **`.dedupe(field, …)`** — unique on the listed key columns (catalog wire names). +- **`.distinct()`** — unique on the **full row** (all visible columns). +- **`.distinct(field, …)`** — same as `.dedupe(field, …)` (alias sugar). + +Both forms lower to the same Polars `unique_stable(…, First)` path. Like `.sort`, dedupe/distinct are **terminal** for relation-dot continuation on that label. + ## Chaining order Postfix applies left-to-right on the written expression (`a.limit(10).sort(x)` → sort after limit). Recommended SQL mental model: ```text -source → .filter{…} → .group_by(…) → .sort(…) → .limit(n) → [fields] → < { … }` | **Derive map** over rows — not a relation hop | See [plasm-language-definition.md](plasm-language-definition.md#binding-rhs-shapes-label). **`=>`** is only for derive maps and `for_each` on bindings; relation hops use `.r#`/wire, not `=>`. @@ -73,6 +122,11 @@ See [plasm-language-definition.md](plasm-language-definition.md#binding-rhs-shap - OR/NOT in row filters; `.having{…}`; `.derive()` postfix. - HTTP push-down of row filters (optimizer may add later without changing surface meaning). - `rows{…}` as a row-local filter shorthand. +- Surface `join` / equi-join between bindings (row pipeline rejects join-from-surface). + +## Execution engine + +Row compute lowers fused [`ComputeOp`](https://github.com/PlasmTools/plasm-core/blob/main/crates/plasm-core/src/plasm_monad/payload/compute.rs) chains to a [`RowPlan`](https://github.com/PlasmTools/plasm-core/blob/main/crates/plasm-core/src/row_plan/plan.rs) IR in `plasm_core::row_plan`, then executes through a Polars-backed adapter in `plasm_runtime::row_compute`. Collect barriers (program return, paging, invoke-arg holes, render) are the only legal materialization points — render and derive remain outside the fused pipeline. ## Federation