From 577db7906a748667bfc3a84ad4fcdcb76c5c2dfe Mon Sep 17 00:00:00 2001 From: Florian Kinder Date: Fri, 30 Jan 2026 18:58:48 +0100 Subject: [PATCH 1/8] feat(storage): add JSON format version constants Introduce JSONVersion type and constants to track input format versions. This prepares for versioned parsing when the JSON format changes. --- internal/storage/version.go | 24 ++++++++++++++++++++++++ internal/storage/version_test.go | 17 +++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 internal/storage/version.go create mode 100644 internal/storage/version_test.go diff --git a/internal/storage/version.go b/internal/storage/version.go new file mode 100644 index 00000000..72e4ed2e --- /dev/null +++ b/internal/storage/version.go @@ -0,0 +1,24 @@ +// internal/storage/version.go +package storage + +import "fmt" + +// JSONVersion represents the version of the input JSON format +type JSONVersion uint32 + +const ( + // JSONVersionUnknown indicates the version could not be determined + JSONVersionUnknown JSONVersion = 0 + // JSONVersionV1 is the original OCAP JSON format + JSONVersionV1 JSONVersion = 1 + // CurrentJSONVersion is the latest supported version + CurrentJSONVersion = JSONVersionV1 +) + +// String returns a human-readable version string +func (v JSONVersion) String() string { + if v == JSONVersionUnknown { + return "unknown" + } + return fmt.Sprintf("v%d", v) +} diff --git a/internal/storage/version_test.go b/internal/storage/version_test.go new file mode 100644 index 00000000..8eb4820f --- /dev/null +++ b/internal/storage/version_test.go @@ -0,0 +1,17 @@ +// internal/storage/version_test.go +package storage + +import "testing" + +func TestCurrentJSONVersion(t *testing.T) { + if CurrentJSONVersion < 1 { + t.Error("CurrentJSONVersion must be at least 1") + } +} + +func TestVersionString(t *testing.T) { + v := JSONVersion(1) + if v.String() != "v1" { + t.Errorf("expected v1, got %s", v.String()) + } +} From 66a9ece3c7741f0d95e6fb157e328341416fa1cf Mon Sep 17 00:00:00 2001 From: Florian Kinder Date: Fri, 30 Jan 2026 19:02:26 +0100 Subject: [PATCH 2/8] feat(storage): add JSON format version detection DetectJSONVersion analyzes JSON structure to determine which parser version should handle the data. Currently detects V1 by required fields. --- internal/storage/version.go | 19 +++++++++++++++++++ internal/storage/version_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/internal/storage/version.go b/internal/storage/version.go index 72e4ed2e..3b420f7d 100644 --- a/internal/storage/version.go +++ b/internal/storage/version.go @@ -22,3 +22,22 @@ func (v JSONVersion) String() string { } return fmt.Sprintf("v%d", v) } + +// DetectJSONVersion analyzes JSON data to determine its format version +func DetectJSONVersion(data map[string]interface{}) JSONVersion { + // V1 detection: requires worldName, missionName, endFrame, captureDelay, entities + requiredV1 := []string{"worldName", "missionName", "endFrame", "captureDelay", "entities"} + hasAllV1 := true + for _, key := range requiredV1 { + if _, ok := data[key]; !ok { + hasAllV1 = false + break + } + } + + if hasAllV1 { + return JSONVersionV1 + } + + return JSONVersionUnknown +} diff --git a/internal/storage/version_test.go b/internal/storage/version_test.go index 8eb4820f..b82a9f90 100644 --- a/internal/storage/version_test.go +++ b/internal/storage/version_test.go @@ -15,3 +15,32 @@ func TestVersionString(t *testing.T) { t.Errorf("expected v1, got %s", v.String()) } } + +func TestDetectJSONVersion_V1(t *testing.T) { + // V1 format has: worldName, missionName, endFrame, captureDelay, entities, events + data := map[string]interface{}{ + "worldName": "Altis", + "missionName": "Test", + "endFrame": 100.0, + "captureDelay": 1.0, + "entities": []interface{}{}, + "events": []interface{}{}, + } + + v := DetectJSONVersion(data) + if v != JSONVersionV1 { + t.Errorf("expected V1, got %s", v.String()) + } +} + +func TestDetectJSONVersion_Unknown(t *testing.T) { + // Missing required fields + data := map[string]interface{}{ + "foo": "bar", + } + + v := DetectJSONVersion(data) + if v != JSONVersionUnknown { + t.Errorf("expected Unknown, got %s", v.String()) + } +} From 78ac0032b3af64ebe71a75eb2d8c33b99cc54f70 Mon Sep 17 00:00:00 2001 From: Florian Kinder Date: Fri, 30 Jan 2026 19:05:53 +0100 Subject: [PATCH 3/8] feat(storage): add versioned parser interface and registry Introduce Parser interface for version-specific JSON parsing. Registry pattern allows adding new format versions without modifying existing code. --- internal/storage/parser.go | 37 +++++++++++++++++++++++++++++++++ internal/storage/parser_test.go | 13 ++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 internal/storage/parser.go create mode 100644 internal/storage/parser_test.go diff --git a/internal/storage/parser.go b/internal/storage/parser.go new file mode 100644 index 00000000..5e771444 --- /dev/null +++ b/internal/storage/parser.go @@ -0,0 +1,37 @@ +// internal/storage/parser.go +package storage + +import ( + "fmt" + + pb "github.com/OCAP2/web/pkg/schemas/protobuf" +) + +// ParseResult contains the parsed manifest and position data +type ParseResult struct { + Manifest *pb.Manifest + EntityPositions []entityPositionData +} + +// Parser converts JSON data to internal format +type Parser interface { + // Version returns which JSON version this parser handles + Version() JSONVersion + // Parse converts JSON data to ParseResult + Parse(data map[string]interface{}, chunkSize uint32) (*ParseResult, error) +} + +var parsers = make(map[JSONVersion]Parser) + +// RegisterParser adds a parser to the registry +func RegisterParser(p Parser) { + parsers[p.Version()] = p +} + +// GetParser returns a parser for the given version +func GetParser(v JSONVersion) (Parser, error) { + if p, ok := parsers[v]; ok { + return p, nil + } + return nil, fmt.Errorf("no parser for JSON version %s", v.String()) +} diff --git a/internal/storage/parser_test.go b/internal/storage/parser_test.go new file mode 100644 index 00000000..f3fff577 --- /dev/null +++ b/internal/storage/parser_test.go @@ -0,0 +1,13 @@ +// internal/storage/parser_test.go +package storage + +import ( + "testing" +) + +func TestParserRegistry_Unknown(t *testing.T) { + _, err := GetParser(JSONVersionUnknown) + if err == nil { + t.Error("expected error for unknown version") + } +} From 6e5d2228129c86ffbf6029f63bf61952db3cff84 Mon Sep 17 00:00:00 2001 From: Florian Kinder Date: Fri, 30 Jan 2026 19:09:15 +0100 Subject: [PATCH 4/8] feat(storage): extract V1 parser from converter Move JSON parsing logic to dedicated ParserV1 implementing Parser interface. This preserves existing behavior while enabling future format versions. --- internal/storage/parser_test.go | 58 +++++++ internal/storage/parser_v1.go | 266 ++++++++++++++++++++++++++++++++ 2 files changed, 324 insertions(+) create mode 100644 internal/storage/parser_v1.go diff --git a/internal/storage/parser_test.go b/internal/storage/parser_test.go index f3fff577..4e529823 100644 --- a/internal/storage/parser_test.go +++ b/internal/storage/parser_test.go @@ -11,3 +11,61 @@ func TestParserRegistry_Unknown(t *testing.T) { t.Error("expected error for unknown version") } } + +func TestParserRegistry(t *testing.T) { + parser, err := GetParser(JSONVersionV1) + if err != nil { + t.Fatalf("GetParser(V1) error: %v", err) + } + if parser == nil { + t.Fatal("parser is nil") + } +} + +func TestParserV1_Parse(t *testing.T) { + data := map[string]interface{}{ + "worldName": "Altis", + "missionName": "Test Mission", + "endFrame": 10.0, + "captureDelay": 1.0, + "entities": []interface{}{ + map[string]interface{}{ + "id": 0.0, + "type": "unit", + "name": "Player1", + "side": "WEST", + "group": "Alpha", + "role": "Rifleman", + "startFrameNum": 0.0, + "isPlayer": 1.0, + "positions": []interface{}{ + []interface{}{[]interface{}{100.0, 200.0, 0.0}, 90.0, 1.0, 0.0, "Player1", 1.0}, + }, + }, + }, + "events": []interface{}{}, + } + + parser, err := GetParser(JSONVersionV1) + if err != nil { + t.Fatalf("GetParser error: %v", err) + } + + result, err := parser.Parse(data, 300) + if err != nil { + t.Fatalf("Parse error: %v", err) + } + + if result.Manifest.WorldName != "Altis" { + t.Errorf("expected Altis, got %s", result.Manifest.WorldName) + } + if result.Manifest.Version != 1 { + t.Errorf("expected version 1, got %d", result.Manifest.Version) + } + if len(result.Manifest.Entities) != 1 { + t.Errorf("expected 1 entity, got %d", len(result.Manifest.Entities)) + } + if len(result.EntityPositions) != 1 { + t.Errorf("expected 1 position data, got %d", len(result.EntityPositions)) + } +} diff --git a/internal/storage/parser_v1.go b/internal/storage/parser_v1.go new file mode 100644 index 00000000..3a269bc7 --- /dev/null +++ b/internal/storage/parser_v1.go @@ -0,0 +1,266 @@ +// internal/storage/parser_v1.go +package storage + +import ( + pb "github.com/OCAP2/web/pkg/schemas/protobuf" +) + +func init() { + RegisterParser(&ParserV1{}) +} + +// ParserV1 parses the original OCAP JSON format (version 1) +type ParserV1 struct{} + +// Version returns JSONVersionV1 +func (p *ParserV1) Version() JSONVersion { + return JSONVersionV1 +} + +// Parse converts V1 JSON data to ParseResult +func (p *ParserV1) Parse(data map[string]interface{}, chunkSize uint32) (*ParseResult, error) { + manifest := &pb.Manifest{ + Version: 1, + WorldName: getString(data, "worldName"), + MissionName: getString(data, "missionName"), + FrameCount: getUint32(data, "endFrame"), + CaptureDelayMs: uint32(getFloat64(data, "captureDelay") * 1000), + } + + var entityPositions []entityPositionData + + // Parse entities + if entities, ok := data["entities"].([]interface{}); ok { + for _, ent := range entities { + em, ok := ent.(map[string]interface{}) + if !ok { + continue + } + + entityType := getString(em, "type") + startFrame := getUint32(em, "startFrameNum") + endFrame := p.calculateEndFrame(em, startFrame) + + def := &pb.EntityDef{ + Id: getUint32(em, "id"), + Type: stringToEntityType(entityType), + Name: getString(em, "name"), + Side: stringToSide(getString(em, "side")), + GroupName: getString(em, "group"), + Role: getString(em, "role"), + StartFrame: startFrame, + EndFrame: endFrame, + IsPlayer: getFloat64(em, "isPlayer") == 1, + VehicleClass: getString(em, "class"), + } + manifest.Entities = append(manifest.Entities, def) + + // Collect position data + if positions, ok := em["positions"].([]interface{}); ok { + entityPositions = append(entityPositions, entityPositionData{ + ID: def.Id, + Type: entityType, + StartFrame: startFrame, + Positions: positions, + }) + } + } + } + + // Parse events + if events, ok := data["events"].([]interface{}); ok { + for _, evt := range events { + evtArr, ok := evt.([]interface{}) + if !ok || len(evtArr) < 2 { + continue + } + + event := p.parseEvent(evtArr) + if event != nil { + manifest.Events = append(manifest.Events, event) + } + } + } + + // Parse markers + if markers, ok := data["Markers"].([]interface{}); ok { + for _, m := range markers { + markerArr, ok := m.([]interface{}) + if !ok { + continue + } + + marker := p.parseMarker(markerArr) + if marker != nil { + manifest.Markers = append(manifest.Markers, marker) + } + } + } + + // Parse times + if times, ok := data["times"].([]interface{}); ok { + for _, t := range times { + tm, ok := t.(map[string]interface{}) + if !ok { + continue + } + + timeSample := &pb.TimeSample{ + FrameNum: getUint32(tm, "frameNum"), + SystemTimeUtc: getString(tm, "systemTimeUTC"), + Date: getString(tm, "date"), + TimeMultiplier: float32(getFloat64(tm, "timeMultiplier")), + Time: float32(getFloat64(tm, "time")), + } + manifest.Times = append(manifest.Times, timeSample) + } + } + + return &ParseResult{ + Manifest: manifest, + EntityPositions: entityPositions, + }, nil +} + +// calculateEndFrame determines the end frame from positions array length +func (p *ParserV1) calculateEndFrame(em map[string]interface{}, startFrame uint32) uint32 { + if positions, ok := em["positions"].([]interface{}); ok { + return startFrame + uint32(len(positions)) - 1 + } + return startFrame +} + +// parseEvent converts a JSON event array to protobuf Event +func (p *ParserV1) parseEvent(evtArr []interface{}) *pb.Event { + if len(evtArr) < 2 { + return nil + } + + event := &pb.Event{ + FrameNum: uint32(toFloat64(evtArr[0])), + Type: toString(evtArr[1]), + } + + // Parse additional fields based on event type + // Common format: [frameNum, "type", sourceId, targetId, ...] + if len(evtArr) > 2 { + event.SourceId = uint32(toFloat64(evtArr[2])) + } + if len(evtArr) > 3 { + event.TargetId = uint32(toFloat64(evtArr[3])) + } + if len(evtArr) > 4 { + // Could be weapon name, message, or distance depending on event type + switch v := evtArr[4].(type) { + case string: + if event.Type == "hit" || event.Type == "killed" { + event.Weapon = v + } else { + event.Message = v + } + case float64: + event.Distance = float32(v) + } + } + if len(evtArr) > 5 { + if d, ok := evtArr[5].(float64); ok { + event.Distance = float32(d) + } + } + + return event +} + +// parseMarker converts a JSON marker array to protobuf MarkerDef +func (p *ParserV1) parseMarker(markerArr []interface{}) *pb.MarkerDef { + // Format: ["type", "text", startFrame, endFrame, playerId, "color", sideIndex, positions, size, "shape", "brush"] + if len(markerArr) < 7 { + return nil + } + + marker := &pb.MarkerDef{ + Type: toString(markerArr[0]), + Text: toString(markerArr[1]), + StartFrame: uint32(toFloat64(markerArr[2])), + EndFrame: uint32(toFloat64(markerArr[3])), + PlayerId: int32(toFloat64(markerArr[4])), + Color: toString(markerArr[5]), + Side: sideIndexToSide(int(toFloat64(markerArr[6]))), + } + + // Parse positions (index 7) + if len(markerArr) > 7 { + if positions, ok := markerArr[7].([]interface{}); ok { + for _, pos := range positions { + mp := p.parseMarkerPosition(pos) + if mp != nil { + marker.Positions = append(marker.Positions, mp) + } + } + } + } + + // Parse size (index 8) + if len(markerArr) > 8 { + if sizeArr, ok := markerArr[8].([]interface{}); ok { + for _, s := range sizeArr { + marker.Size = append(marker.Size, float32(toFloat64(s))) + } + } + } + + // Parse shape (index 9) + if len(markerArr) > 9 { + marker.Shape = toString(markerArr[9]) + } + + // Parse brush (index 10) + if len(markerArr) > 10 { + marker.Brush = toString(markerArr[10]) + } + + return marker +} + +// parseMarkerPosition converts position data to MarkerPosition +func (p *ParserV1) parseMarkerPosition(pos interface{}) *pb.MarkerPosition { + // Position format can be: [x, y, z] or [[x, y, z], frameNum, direction, alpha] + arr, ok := pos.([]interface{}) + if !ok || len(arr) == 0 { + return nil + } + + mp := &pb.MarkerPosition{} + + // Check if first element is a position array + if posArr, ok := arr[0].([]interface{}); ok { + // Format: [[x, y, z], frameNum, direction, alpha] + if len(posArr) >= 2 { + mp.PosX = float32(toFloat64(posArr[0])) + mp.PosY = float32(toFloat64(posArr[1])) + if len(posArr) > 2 { + mp.PosZ = float32(toFloat64(posArr[2])) + } + } + if len(arr) > 1 { + mp.FrameNum = uint32(toFloat64(arr[1])) + } + if len(arr) > 2 { + mp.Direction = float32(toFloat64(arr[2])) + } + if len(arr) > 3 { + mp.Alpha = float32(toFloat64(arr[3])) + } + } else { + // Simple format: [x, y, z] + if len(arr) >= 2 { + mp.PosX = float32(toFloat64(arr[0])) + mp.PosY = float32(toFloat64(arr[1])) + if len(arr) > 2 { + mp.PosZ = float32(toFloat64(arr[2])) + } + } + } + + return mp +} From 158aa603a43aa03ee63cad21af0440cc7426a0f6 Mon Sep 17 00:00:00 2001 From: Florian Kinder Date: Fri, 30 Jan 2026 19:15:58 +0100 Subject: [PATCH 5/8] refactor(storage): converter uses versioned parser registry Replace hardcoded parseJSONData with parseJSONDataVersioned that detects the input format version and delegates to the appropriate parser. Falls back to V1 for unrecognized formats. --- internal/storage/converter.go | 260 ++--------------------------- internal/storage/converter_test.go | 134 ++++++++++++--- internal/storage/flatbuffers.go | 2 +- internal/storage/parser.go | 8 + 4 files changed, 135 insertions(+), 269 deletions(-) diff --git a/internal/storage/converter.go b/internal/storage/converter.go index 11f22c2b..9963a7d6 100644 --- a/internal/storage/converter.go +++ b/internal/storage/converter.go @@ -40,7 +40,7 @@ func (c *Converter) Convert(ctx context.Context, jsonPath, outputPath string) er } // Parse into protobuf manifest and collect position data - manifest, entityPositions, err := c.parseJSONData(data) + manifest, entityPositions, err := c.parseJSONDataVersioned(data) if err != nil { return fmt.Errorf("parse JSON: %w", err) } @@ -101,257 +101,25 @@ func (c *Converter) loadJSON(path string) (map[string]interface{}, error) { return data, nil } -// EntityPositionData holds parsed position data for an entity -type entityPositionData struct { - ID uint32 - Type string - StartFrame uint32 - Positions []interface{} // Raw position arrays -} - -// parseJSONData converts JSON data to protobuf manifest and extracts position data -func (c *Converter) parseJSONData(data map[string]interface{}) (*pb.Manifest, []entityPositionData, error) { - manifest := &pb.Manifest{ - Version: 1, - WorldName: getString(data, "worldName"), - MissionName: getString(data, "missionName"), - FrameCount: getUint32(data, "endFrame"), - CaptureDelayMs: uint32(getFloat64(data, "captureDelay") * 1000), - } - - var entityPositions []entityPositionData - - // Parse entities - if entities, ok := data["entities"].([]interface{}); ok { - for _, ent := range entities { - em, ok := ent.(map[string]interface{}) - if !ok { - continue - } - - entityType := getString(em, "type") - startFrame := getUint32(em, "startFrameNum") - endFrame := c.calculateEndFrame(em, startFrame) - - def := &pb.EntityDef{ - Id: getUint32(em, "id"), - Type: stringToEntityType(entityType), - Name: getString(em, "name"), - Side: stringToSide(getString(em, "side")), - GroupName: getString(em, "group"), - Role: getString(em, "role"), - StartFrame: startFrame, - EndFrame: endFrame, - IsPlayer: getFloat64(em, "isPlayer") == 1, - VehicleClass: getString(em, "class"), - } - manifest.Entities = append(manifest.Entities, def) - - // Collect position data - if positions, ok := em["positions"].([]interface{}); ok { - entityPositions = append(entityPositions, entityPositionData{ - ID: def.Id, - Type: entityType, - StartFrame: startFrame, - Positions: positions, - }) - } - } - } - - // Parse events - if events, ok := data["events"].([]interface{}); ok { - for _, evt := range events { - evtArr, ok := evt.([]interface{}) - if !ok || len(evtArr) < 2 { - continue - } - - event := c.parseEvent(evtArr) - if event != nil { - manifest.Events = append(manifest.Events, event) - } - } - } - - // Parse markers - if markers, ok := data["Markers"].([]interface{}); ok { - for _, m := range markers { - markerArr, ok := m.([]interface{}) - if !ok { - continue - } - - marker := c.parseMarker(markerArr) - if marker != nil { - manifest.Markers = append(manifest.Markers, marker) - } - } - } - - // Parse times - if times, ok := data["times"].([]interface{}); ok { - for _, t := range times { - tm, ok := t.(map[string]interface{}) - if !ok { - continue - } - - timeSample := &pb.TimeSample{ - FrameNum: getUint32(tm, "frameNum"), - SystemTimeUtc: getString(tm, "systemTimeUTC"), - Date: getString(tm, "date"), - TimeMultiplier: float32(getFloat64(tm, "timeMultiplier")), - Time: float32(getFloat64(tm, "time")), - } - manifest.Times = append(manifest.Times, timeSample) - } - } - - return manifest, entityPositions, nil -} - -// calculateEndFrame determines the end frame from positions array length -func (c *Converter) calculateEndFrame(em map[string]interface{}, startFrame uint32) uint32 { - if positions, ok := em["positions"].([]interface{}); ok { - return startFrame + uint32(len(positions)) - 1 - } - return startFrame -} - -// parseEvent converts a JSON event array to protobuf Event -func (c *Converter) parseEvent(evtArr []interface{}) *pb.Event { - if len(evtArr) < 2 { - return nil - } - - event := &pb.Event{ - FrameNum: uint32(toFloat64(evtArr[0])), - Type: toString(evtArr[1]), +// parseJSONDataVersioned detects version and uses appropriate parser +func (c *Converter) parseJSONDataVersioned(data map[string]interface{}) (*pb.Manifest, []entityPositionData, error) { + version := DetectJSONVersion(data) + if version == JSONVersionUnknown { + // Fall back to V1 for backwards compatibility + version = JSONVersionV1 } - // Parse additional fields based on event type - // Common format: [frameNum, "type", sourceId, targetId, ...] - if len(evtArr) > 2 { - event.SourceId = uint32(toFloat64(evtArr[2])) - } - if len(evtArr) > 3 { - event.TargetId = uint32(toFloat64(evtArr[3])) - } - if len(evtArr) > 4 { - // Could be weapon name, message, or distance depending on event type - switch v := evtArr[4].(type) { - case string: - if event.Type == "hit" || event.Type == "killed" { - event.Weapon = v - } else { - event.Message = v - } - case float64: - event.Distance = float32(v) - } - } - if len(evtArr) > 5 { - if d, ok := evtArr[5].(float64); ok { - event.Distance = float32(d) - } - } - - return event -} - -// parseMarker converts a JSON marker array to protobuf MarkerDef -func (c *Converter) parseMarker(markerArr []interface{}) *pb.MarkerDef { - // Format: ["type", "text", startFrame, endFrame, playerId, "color", sideIndex, positions, size, "shape", "brush"] - if len(markerArr) < 7 { - return nil - } - - marker := &pb.MarkerDef{ - Type: toString(markerArr[0]), - Text: toString(markerArr[1]), - StartFrame: uint32(toFloat64(markerArr[2])), - EndFrame: uint32(toFloat64(markerArr[3])), - PlayerId: int32(toFloat64(markerArr[4])), - Color: toString(markerArr[5]), - Side: sideIndexToSide(int(toFloat64(markerArr[6]))), - } - - // Parse positions (index 7) - if len(markerArr) > 7 { - if positions, ok := markerArr[7].([]interface{}); ok { - for _, pos := range positions { - mp := c.parseMarkerPosition(pos) - if mp != nil { - marker.Positions = append(marker.Positions, mp) - } - } - } - } - - // Parse size (index 8) - if len(markerArr) > 8 { - if sizeArr, ok := markerArr[8].([]interface{}); ok { - for _, s := range sizeArr { - marker.Size = append(marker.Size, float32(toFloat64(s))) - } - } - } - - // Parse shape (index 9) - if len(markerArr) > 9 { - marker.Shape = toString(markerArr[9]) - } - - // Parse brush (index 10) - if len(markerArr) > 10 { - marker.Brush = toString(markerArr[10]) - } - - return marker -} - -// parseMarkerPosition converts position data to MarkerPosition -func (c *Converter) parseMarkerPosition(pos interface{}) *pb.MarkerPosition { - // Position format can be: [x, y, z] or [[x, y, z], frameNum, direction, alpha] - arr, ok := pos.([]interface{}) - if !ok || len(arr) == 0 { - return nil + parser, err := GetParser(version) + if err != nil { + return nil, nil, fmt.Errorf("get parser for %s: %w", version.String(), err) } - mp := &pb.MarkerPosition{} - - // Check if first element is a position array - if posArr, ok := arr[0].([]interface{}); ok { - // Format: [[x, y, z], frameNum, direction, alpha] - if len(posArr) >= 2 { - mp.PosX = float32(toFloat64(posArr[0])) - mp.PosY = float32(toFloat64(posArr[1])) - if len(posArr) > 2 { - mp.PosZ = float32(toFloat64(posArr[2])) - } - } - if len(arr) > 1 { - mp.FrameNum = uint32(toFloat64(arr[1])) - } - if len(arr) > 2 { - mp.Direction = float32(toFloat64(arr[2])) - } - if len(arr) > 3 { - mp.Alpha = float32(toFloat64(arr[3])) - } - } else { - // Simple format: [x, y, z] - if len(arr) >= 2 { - mp.PosX = float32(toFloat64(arr[0])) - mp.PosY = float32(toFloat64(arr[1])) - if len(arr) > 2 { - mp.PosZ = float32(toFloat64(arr[2])) - } - } + result, err := parser.Parse(data, c.ChunkSize) + if err != nil { + return nil, nil, fmt.Errorf("parse with %s: %w", version.String(), err) } - return mp + return result.Manifest, result.EntityPositions, nil } // writeManifest writes the manifest protobuf file diff --git a/internal/storage/converter_test.go b/internal/storage/converter_test.go index 484b2610..feb60b27 100644 --- a/internal/storage/converter_test.go +++ b/internal/storage/converter_test.go @@ -14,6 +14,96 @@ import ( pb "github.com/OCAP2/web/pkg/schemas/protobuf" ) +func TestConverter_UsesVersionedParser(t *testing.T) { + // Create test JSON data + data := map[string]interface{}{ + "worldName": "Altis", + "missionName": "Test", + "endFrame": 10.0, + "captureDelay": 1.0, + "entities": []interface{}{}, + "events": []interface{}{}, + } + + c := NewConverter(300) + manifest, positions, err := c.parseJSONDataVersioned(data) + if err != nil { + t.Fatalf("parseJSONDataVersioned error: %v", err) + } + + if manifest.Version != 1 { + t.Errorf("expected version 1, got %d", manifest.Version) + } + _ = positions // positions can be empty for this test +} + +func TestConverter_UsesVersionedParser_WithEntities(t *testing.T) { + // Create test JSON data with entities + data := map[string]interface{}{ + "worldName": "Stratis", + "missionName": "Test Mission", + "endFrame": 5.0, + "captureDelay": 0.5, + "entities": []interface{}{ + map[string]interface{}{ + "id": 1.0, + "type": "unit", + "name": "Player1", + "side": "WEST", + "group": "Alpha", + "role": "Rifleman", + "startFrameNum": 0.0, + "isPlayer": 1.0, + "positions": []interface{}{ + []interface{}{[]interface{}{100.0, 200.0, 0.0}, 90.0, 1.0, 0.0, "Player1", 1.0}, + }, + }, + }, + "events": []interface{}{}, + "Markers": []interface{}{}, + "times": []interface{}{}, + } + + c := NewConverter(300) + manifest, positions, err := c.parseJSONDataVersioned(data) + if err != nil { + t.Fatalf("parseJSONDataVersioned error: %v", err) + } + + if manifest.WorldName != "Stratis" { + t.Errorf("expected WorldName 'Stratis', got %q", manifest.WorldName) + } + if len(manifest.Entities) != 1 { + t.Errorf("expected 1 entity, got %d", len(manifest.Entities)) + } + if len(positions) != 1 { + t.Errorf("expected 1 position entry, got %d", len(positions)) + } +} + +func TestConverter_UsesVersionedParser_UnknownVersion(t *testing.T) { + // Create minimal JSON data that doesn't match V1 requirements + // Missing required fields should still work (fallback to V1) + data := map[string]interface{}{ + "worldName": "Test", + "missionName": "Test", + "endFrame": 5.0, + "captureDelay": 1.0, + "entities": []interface{}{}, + } + + c := NewConverter(300) + manifest, _, err := c.parseJSONDataVersioned(data) + if err != nil { + t.Fatalf("parseJSONDataVersioned error: %v", err) + } + + // Should fall back to V1 and parse successfully + if manifest.Version != 1 { + t.Errorf("expected version 1 (fallback), got %d", manifest.Version) + } +} + func TestConverter_Convert(t *testing.T) { // Create temp directories tmpDir := t.TempDir() @@ -472,8 +562,8 @@ func TestNewConverter_DefaultChunkSize(t *testing.T) { } } -func TestConverter_ParseEvent(t *testing.T) { - converter := NewConverter(DefaultChunkSize) +func TestParserV1_ParseEvent(t *testing.T) { + parser := &ParserV1{} tests := []struct { name string @@ -503,7 +593,7 @@ func TestConverter_ParseEvent(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - event := converter.parseEvent(tt.input) + event := parser.parseEvent(tt.input) if tt.wantOK { if event == nil { t.Error("expected non-nil event") @@ -643,11 +733,11 @@ func TestToString(t *testing.T) { } } -func TestConverter_ParseMarkerPosition(t *testing.T) { - converter := NewConverter(DefaultChunkSize) +func TestParserV1_ParseMarkerPosition(t *testing.T) { + parser := &ParserV1{} t.Run("simple format [x, y, z]", func(t *testing.T) { - pos := converter.parseMarkerPosition([]interface{}{100.0, 200.0, 10.0}) + pos := parser.parseMarkerPosition([]interface{}{100.0, 200.0, 10.0}) if pos == nil { t.Fatal("expected non-nil position") } @@ -663,7 +753,7 @@ func TestConverter_ParseMarkerPosition(t *testing.T) { }) t.Run("simple format [x, y] without z", func(t *testing.T) { - pos := converter.parseMarkerPosition([]interface{}{100.0, 200.0}) + pos := parser.parseMarkerPosition([]interface{}{100.0, 200.0}) if pos == nil { t.Fatal("expected non-nil position") } @@ -679,7 +769,7 @@ func TestConverter_ParseMarkerPosition(t *testing.T) { }) t.Run("complex format [[x, y, z], frameNum, direction, alpha]", func(t *testing.T) { - pos := converter.parseMarkerPosition([]interface{}{ + pos := parser.parseMarkerPosition([]interface{}{ []interface{}{100.0, 200.0, 10.0}, 50.0, 90.0, @@ -709,7 +799,7 @@ func TestConverter_ParseMarkerPosition(t *testing.T) { }) t.Run("complex format [[x, y], frameNum] without z", func(t *testing.T) { - pos := converter.parseMarkerPosition([]interface{}{ + pos := parser.parseMarkerPosition([]interface{}{ []interface{}{100.0, 200.0}, 50.0, }) @@ -728,29 +818,29 @@ func TestConverter_ParseMarkerPosition(t *testing.T) { }) t.Run("nil input", func(t *testing.T) { - pos := converter.parseMarkerPosition(nil) + pos := parser.parseMarkerPosition(nil) if pos != nil { t.Error("expected nil position for nil input") } }) t.Run("non-array input", func(t *testing.T) { - pos := converter.parseMarkerPosition("not an array") + pos := parser.parseMarkerPosition("not an array") if pos != nil { t.Error("expected nil position for non-array input") } }) t.Run("empty array", func(t *testing.T) { - pos := converter.parseMarkerPosition([]interface{}{}) + pos := parser.parseMarkerPosition([]interface{}{}) if pos != nil { t.Error("expected nil position for empty array") } }) } -func TestConverter_CalculateEndFrame(t *testing.T) { - converter := NewConverter(DefaultChunkSize) +func TestParserV1_CalculateEndFrame(t *testing.T) { + parser := &ParserV1{} t.Run("with positions", func(t *testing.T) { em := map[string]interface{}{ @@ -762,7 +852,7 @@ func TestConverter_CalculateEndFrame(t *testing.T) { []interface{}{}, }, } - endFrame := converter.calculateEndFrame(em, 10) + endFrame := parser.calculateEndFrame(em, 10) // startFrame + len(positions) - 1 = 10 + 5 - 1 = 14 if endFrame != 14 { t.Errorf("endFrame = %d, want 14", endFrame) @@ -771,7 +861,7 @@ func TestConverter_CalculateEndFrame(t *testing.T) { t.Run("without positions", func(t *testing.T) { em := map[string]interface{}{} - endFrame := converter.calculateEndFrame(em, 10) + endFrame := parser.calculateEndFrame(em, 10) // Should return startFrame when no positions if endFrame != 10 { t.Errorf("endFrame = %d, want 10", endFrame) @@ -782,7 +872,7 @@ func TestConverter_CalculateEndFrame(t *testing.T) { em := map[string]interface{}{ "positions": "not an array", } - endFrame := converter.calculateEndFrame(em, 10) + endFrame := parser.calculateEndFrame(em, 10) // Should return startFrame when positions is wrong type if endFrame != 10 { t.Errorf("endFrame = %d, want 10", endFrame) @@ -790,12 +880,12 @@ func TestConverter_CalculateEndFrame(t *testing.T) { }) } -func TestConverter_ParseEvent_Distance(t *testing.T) { - converter := NewConverter(DefaultChunkSize) +func TestParserV1_ParseEvent_Distance(t *testing.T) { + parser := &ParserV1{} t.Run("event with numeric distance at index 4", func(t *testing.T) { // When index 4 is a number (not a weapon string), it's treated as distance - event := converter.parseEvent([]interface{}{100.0, "move", 1.0, 2.0, 50.5}) + event := parser.parseEvent([]interface{}{100.0, "move", 1.0, 2.0, 50.5}) if event == nil { t.Fatal("expected non-nil event") } @@ -806,7 +896,7 @@ func TestConverter_ParseEvent_Distance(t *testing.T) { t.Run("event with message at index 4", func(t *testing.T) { // For non-hit/killed events, string at index 4 is message - event := converter.parseEvent([]interface{}{100.0, "chat", 1.0, 2.0, "Hello world"}) + event := parser.parseEvent([]interface{}{100.0, "chat", 1.0, 2.0, "Hello world"}) if event == nil { t.Fatal("expected non-nil event") } @@ -817,7 +907,7 @@ func TestConverter_ParseEvent_Distance(t *testing.T) { t.Run("event with weapon and distance", func(t *testing.T) { // killed/hit events have weapon at index 4 and distance at index 5 - event := converter.parseEvent([]interface{}{100.0, "killed", 1.0, 2.0, "arifle_MX", 150.5}) + event := parser.parseEvent([]interface{}{100.0, "killed", 1.0, 2.0, "arifle_MX", 150.5}) if event == nil { t.Fatal("expected non-nil event") } diff --git a/internal/storage/flatbuffers.go b/internal/storage/flatbuffers.go index bfddeb89..9c5ce6ed 100644 --- a/internal/storage/flatbuffers.go +++ b/internal/storage/flatbuffers.go @@ -89,7 +89,7 @@ func (e *FlatBuffersEngine) Convert(ctx context.Context, jsonPath, outputPath st return fmt.Errorf("load JSON: %w", err) } - pbManifest, entityPositions, err := converter.parseJSONData(data) + pbManifest, entityPositions, err := converter.parseJSONDataVersioned(data) if err != nil { return fmt.Errorf("parse JSON: %w", err) } diff --git a/internal/storage/parser.go b/internal/storage/parser.go index 5e771444..2bf002f7 100644 --- a/internal/storage/parser.go +++ b/internal/storage/parser.go @@ -7,6 +7,14 @@ import ( pb "github.com/OCAP2/web/pkg/schemas/protobuf" ) +// entityPositionData holds parsed position data for an entity +type entityPositionData struct { + ID uint32 + Type string + StartFrame uint32 + Positions []interface{} // Raw position arrays +} + // ParseResult contains the parsed manifest and position data type ParseResult struct { Manifest *pb.Manifest From 1b241308a0613b649d5fa71b31dcbbbca35f6463 Mon Sep 17 00:00:00 2001 From: Florian Kinder Date: Fri, 30 Jan 2026 19:21:40 +0100 Subject: [PATCH 6/8] feat(db): add json_format_version column to operations Track which JSON format version was used for each recording. Enables future re-conversion when format changes. --- internal/server/operation.go | 51 +++++++++++++++++++++---------- internal/server/operation_test.go | 2 +- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/internal/server/operation.go b/internal/server/operation.go index dcdc3c19..dd0a4c1e 100644 --- a/internal/server/operation.go +++ b/internal/server/operation.go @@ -10,15 +10,16 @@ import ( ) type Operation struct { - ID int64 `json:"id"` - WorldName string `json:"world_name"` - MissionName string `json:"mission_name"` - MissionDuration float64 `json:"mission_duration"` - Filename string `json:"filename"` - Date string `json:"date"` - Tag string `json:"tag"` - StorageFormat string `json:"storageFormat"` - ConversionStatus string `json:"conversionStatus"` + ID int64 `json:"id"` + WorldName string `json:"world_name"` + MissionName string `json:"mission_name"` + MissionDuration float64 `json:"mission_duration"` + Filename string `json:"filename"` + Date string `json:"date"` + Tag string `json:"tag"` + StorageFormat string `json:"storageFormat"` + ConversionStatus string `json:"conversionStatus"` + JSONFormatVersion int `json:"jsonFormatVersion,omitempty"` } type Filter struct { @@ -124,6 +125,18 @@ func (r *RepoOperation) migration() (err error) { } } + if version < 4 { + _, err = r.db.Exec(`ALTER TABLE operations ADD COLUMN json_format_version INTEGER DEFAULT 1`) + if err != nil { + return fmt.Errorf("merge db to v4 failed (json_format_version): %w", err) + } + + _, err = r.db.Exec(`INSERT INTO version (db) VALUES (4)`) + if err != nil { + return fmt.Errorf("failed to increase version 4: %w", err) + } + } + return nil } @@ -160,12 +173,16 @@ func (r *RepoOperation) Store(ctx context.Context, operation *Operation) error { if conversionStatus == "" { conversionStatus = "pending" } + jsonFormatVersion := operation.JSONFormatVersion + if jsonFormatVersion == 0 { + jsonFormatVersion = 1 + } query := ` INSERT INTO operations - (world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status) + (world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status, json_format_version) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8) + ($1, $2, $3, $4, $5, $6, $7, $8, $9) ` result, err := r.db.ExecContext( ctx, @@ -178,6 +195,7 @@ func (r *RepoOperation) Store(ctx context.Context, operation *Operation) error { operation.Tag, storageFormat, conversionStatus, + jsonFormatVersion, ) if err != nil { return err @@ -206,7 +224,7 @@ func (r *RepoOperation) Select(ctx context.Context, filter Filter) ([]Operation, query := ` SELECT - id, world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status + id, world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status, json_format_version FROM operations WHERE @@ -247,6 +265,7 @@ func (*RepoOperation) scan(ctx context.Context, rows *sql.Rows) ([]Operation, er &o.Tag, &o.StorageFormat, &o.ConversionStatus, + &o.JSONFormatVersion, ) if err != nil { return nil, err @@ -259,12 +278,12 @@ func (*RepoOperation) scan(ctx context.Context, rows *sql.Rows) ([]Operation, er // GetByID retrieves a single operation by its ID func (r *RepoOperation) GetByID(ctx context.Context, id string) (*Operation, error) { row := r.db.QueryRowContext(ctx, - `SELECT id, world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status + `SELECT id, world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status, json_format_version FROM operations WHERE id = ?`, id) var op Operation err := row.Scan(&op.ID, &op.WorldName, &op.MissionName, &op.MissionDuration, - &op.Filename, &op.Date, &op.Tag, &op.StorageFormat, &op.ConversionStatus) + &op.Filename, &op.Date, &op.Tag, &op.StorageFormat, &op.ConversionStatus, &op.JSONFormatVersion) if err != nil { return nil, err } @@ -274,7 +293,7 @@ func (r *RepoOperation) GetByID(ctx context.Context, id string) (*Operation, err // SelectPending returns operations with pending conversion status func (r *RepoOperation) SelectPending(ctx context.Context, limit int) ([]Operation, error) { rows, err := r.db.QueryContext(ctx, - `SELECT id, world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status + `SELECT id, world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status, json_format_version FROM operations WHERE conversion_status = 'pending' ORDER BY id ASC @@ -290,7 +309,7 @@ func (r *RepoOperation) SelectPending(ctx context.Context, limit int) ([]Operati // SelectAll returns all operations for conversion func (r *RepoOperation) SelectAll(ctx context.Context) ([]Operation, error) { rows, err := r.db.QueryContext(ctx, - `SELECT id, world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status + `SELECT id, world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status, json_format_version FROM operations ORDER BY id ASC`) if err != nil { diff --git a/internal/server/operation_test.go b/internal/server/operation_test.go index abe03206..d5b6a2f0 100644 --- a/internal/server/operation_test.go +++ b/internal/server/operation_test.go @@ -387,7 +387,7 @@ func TestMigrationRerun(t *testing.T) { var version int err = repo2.db.QueryRow("SELECT db FROM version ORDER BY db DESC LIMIT 1").Scan(&version) assert.NoError(t, err) - assert.Equal(t, 3, version) + assert.Equal(t, 4, version) } func TestGetTypesEmpty(t *testing.T) { From 1629bddb843042fd4f73c12165c40b7f50e992e9 Mon Sep 17 00:00:00 2001 From: Florian Kinder Date: Fri, 30 Jan 2026 19:26:34 +0100 Subject: [PATCH 7/8] feat(api): include JSON format version in format info endpoint Clients can now check which JSON format version was used to create each recording. --- internal/server/handler.go | 2 ++ internal/server/handler_test.go | 38 +++++++++++++++++++-------------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/internal/server/handler.go b/internal/server/handler.go index b79ae680..b3def7e7 100644 --- a/internal/server/handler.go +++ b/internal/server/handler.go @@ -42,6 +42,7 @@ type FormatInfo struct { Format string `json:"format"` ChunkCount int `json:"chunkCount"` SupportsStreaming bool `json:"supportsStreaming"` + JSONFormatVersion int `json:"jsonFormatVersion,omitempty"` } // HandlerOption configures the Handler @@ -241,6 +242,7 @@ func (h *Handler) GetOperationFormat(c echo.Context) error { Format: format, ChunkCount: chunkCount, SupportsStreaming: engine.SupportsStreaming(), + JSONFormatVersion: op.JSONFormatVersion, }) } diff --git a/internal/server/handler_test.go b/internal/server/handler_test.go index 063da80a..4102f7dc 100644 --- a/internal/server/handler_test.go +++ b/internal/server/handler_test.go @@ -75,14 +75,15 @@ func TestGetOperationFormat(t *testing.T) { ctx := context.Background() op := &Operation{ - WorldName: "altis", - MissionName: "Test Mission", - MissionDuration: 3600, - Filename: "test_mission", - Date: "2026-01-30", - Tag: "coop", - StorageFormat: "json", - ConversionStatus: "completed", + WorldName: "altis", + MissionName: "Test Mission", + MissionDuration: 3600, + Filename: "test_mission", + Date: "2026-01-30", + Tag: "coop", + StorageFormat: "json", + ConversionStatus: "completed", + JSONFormatVersion: 1, } err = repo.Store(ctx, op) assert.NoError(t, err) @@ -114,6 +115,7 @@ func TestGetOperationFormat(t *testing.T) { assert.Equal(t, "json", formatInfo.Format) assert.Equal(t, 1, formatInfo.ChunkCount) assert.False(t, formatInfo.SupportsStreaming) + assert.Equal(t, 1, formatInfo.JSONFormatVersion) // Test: Get format for non-existing operation req = httptest.NewRequest(http.MethodGet, "/api/v1/operations/999/format", nil) @@ -141,14 +143,15 @@ func TestGetOperationFormatProtobuf(t *testing.T) { ctx := context.Background() op := &Operation{ - WorldName: "altis", - MissionName: "Test Mission Protobuf", - MissionDuration: 3600, - Filename: "test_mission_pb", - Date: "2026-01-30", - Tag: "coop", - StorageFormat: "protobuf", - ConversionStatus: "completed", + WorldName: "altis", + MissionName: "Test Mission Protobuf", + MissionDuration: 3600, + Filename: "test_mission_pb", + Date: "2026-01-30", + Tag: "coop", + StorageFormat: "protobuf", + ConversionStatus: "completed", + JSONFormatVersion: 1, } err = repo.Store(ctx, op) assert.NoError(t, err) @@ -179,6 +182,7 @@ func TestGetOperationFormatProtobuf(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "protobuf", formatInfo.Format) assert.True(t, formatInfo.SupportsStreaming) + assert.Equal(t, 1, formatInfo.JSONFormatVersion) } func TestGetOperationManifest(t *testing.T) { @@ -1264,6 +1268,7 @@ func TestGetOperationFormat_EmptyStorageFormat(t *testing.T) { err = json.Unmarshal(rec.Body.Bytes(), &result) assert.NoError(t, err) assert.Equal(t, "json", result.Format) + assert.Equal(t, 1, result.JSONFormatVersion) // Default version is 1 } func TestGetOperationFormat_UnknownFormat(t *testing.T) { @@ -1315,6 +1320,7 @@ func TestGetOperationFormat_UnknownFormat(t *testing.T) { err = json.Unmarshal(rec.Body.Bytes(), &result) assert.NoError(t, err) assert.Equal(t, "json", result.Format) // Should fallback to json + assert.Equal(t, 1, result.JSONFormatVersion) // Default version is 1 } func TestGetOperationManifest_FlatBuffers(t *testing.T) { From ccd890282c96b0a76d59549dc98510049d27f72a Mon Sep 17 00:00:00 2001 From: Florian Kinder Date: Fri, 30 Jan 2026 19:27:51 +0100 Subject: [PATCH 8/8] docs(storage): add JSON format versioning guide Document how to add new format versions when the OCAP JSON schema changes. --- internal/storage/VERSIONING.md | 68 ++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 internal/storage/VERSIONING.md diff --git a/internal/storage/VERSIONING.md b/internal/storage/VERSIONING.md new file mode 100644 index 00000000..ecb93354 --- /dev/null +++ b/internal/storage/VERSIONING.md @@ -0,0 +1,68 @@ +# JSON Format Versioning + +## Adding a New Version + +When the OCAP JSON format changes, follow these steps: + +### 1. Increment the Version Constant + +In `version.go`: + +```go +const ( + JSONVersionV1 JSONVersion = 1 + JSONVersionV2 JSONVersion = 2 // Add new version + CurrentJSONVersion = JSONVersionV2 // Update current +) +``` + +### 2. Update Version Detection + +In `version.go`, add detection logic for the new format: + +```go +func DetectJSONVersion(data map[string]interface{}) JSONVersion { + // Check for V2-specific fields first (newest versions first) + if _, ok := data["newV2Field"]; ok { + return JSONVersionV2 + } + // ... existing V1 detection ... +} +``` + +### 3. Create Parser for New Version + +Create `parser_v2.go`: + +```go +func init() { + RegisterParser(&ParserV2{}) +} + +type ParserV2 struct{} + +func (p *ParserV2) Version() JSONVersion { return JSONVersionV2 } + +func (p *ParserV2) Parse(data map[string]interface{}, chunkSize uint32) (*ParseResult, error) { + // Handle new format, output same ParseResult structure +} +``` + +### 4. Handle Field Renames/Migrations + +If a field was renamed, handle both old and new names: + +```go +func getFieldWithFallback(data map[string]interface{}, newName, oldName string) interface{} { + if v, ok := data[newName]; ok { + return v + } + return data[oldName] +} +``` + +## Version History + +| Version | Date | Changes | +|---------|------|---------| +| V1 | Original | Initial OCAP format |