Skip to content

feat: add JSON format versioning support for converter #88

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
fank wants to merge 8 commits into from
Closed

feat: add JSON format versioning support for converter #88

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
577db79
feat(storage): add JSON format version constants
fank Jan 30, 2026
66a9ece
feat(storage): add JSON format version detection
fank Jan 30, 2026
78ac003
feat(storage): add versioned parser interface and registry
fank Jan 30, 2026
6e5d222
feat(storage): extract V1 parser from converter
fank Jan 30, 2026
158aa60
refactor(storage): converter uses versioned parser registry
fank Jan 30, 2026
1b24130
feat(db): add json_format_version column to operations
fank Jan 30, 2026
1629bdd
feat(api): include JSON format version in format info endpoint
fank Jan 30, 2026
ccd8902
docs(storage): add JSON format versioning guide
fank Jan 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions internal/server/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ type FormatInfo struct {
Format string `json:"format"`
ChunkCount int `json:"chunkCount"`
SupportsStreaming bool `json:"supportsStreaming"`
JSONFormatVersion int `json:"jsonFormatVersion,omitempty"`
}

// HandlerOption configures the Handler
Expand Down Expand Up @@ -241,6 +242,7 @@ func (h *Handler) GetOperationFormat(c echo.Context) error {
Format: format,
ChunkCount: chunkCount,
SupportsStreaming: engine.SupportsStreaming(),
JSONFormatVersion: op.JSONFormatVersion,
})
}

Expand Down
38 changes: 22 additions & 16 deletions internal/server/handler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,14 +75,15 @@ func TestGetOperationFormat(t *testing.T) {

ctx := context.Background()
op := &Operation{
WorldName: "altis",
MissionName: "Test Mission",
MissionDuration: 3600,
Filename: "test_mission",
Date: "2026-01-30",
Tag: "coop",
StorageFormat: "json",
ConversionStatus: "completed",
WorldName: "altis",
MissionName: "Test Mission",
MissionDuration: 3600,
Filename: "test_mission",
Date: "2026-01-30",
Tag: "coop",
StorageFormat: "json",
ConversionStatus: "completed",
JSONFormatVersion: 1,
}
err = repo.Store(ctx, op)
assert.NoError(t, err)
Expand Down Expand Up @@ -114,6 +115,7 @@ func TestGetOperationFormat(t *testing.T) {
assert.Equal(t, "json", formatInfo.Format)
assert.Equal(t, 1, formatInfo.ChunkCount)
assert.False(t, formatInfo.SupportsStreaming)
assert.Equal(t, 1, formatInfo.JSONFormatVersion)

// Test: Get format for non-existing operation
req = httptest.NewRequest(http.MethodGet, "/api/v1/operations/999/format", nil)
Expand Down Expand Up @@ -141,14 +143,15 @@ func TestGetOperationFormatProtobuf(t *testing.T) {

ctx := context.Background()
op := &Operation{
WorldName: "altis",
MissionName: "Test Mission Protobuf",
MissionDuration: 3600,
Filename: "test_mission_pb",
Date: "2026-01-30",
Tag: "coop",
StorageFormat: "protobuf",
ConversionStatus: "completed",
WorldName: "altis",
MissionName: "Test Mission Protobuf",
MissionDuration: 3600,
Filename: "test_mission_pb",
Date: "2026-01-30",
Tag: "coop",
StorageFormat: "protobuf",
ConversionStatus: "completed",
JSONFormatVersion: 1,
}
err = repo.Store(ctx, op)
assert.NoError(t, err)
Expand Down Expand Up @@ -179,6 +182,7 @@ func TestGetOperationFormatProtobuf(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, "protobuf", formatInfo.Format)
assert.True(t, formatInfo.SupportsStreaming)
assert.Equal(t, 1, formatInfo.JSONFormatVersion)
}

func TestGetOperationManifest(t *testing.T) {
Expand Down Expand Up @@ -1264,6 +1268,7 @@ func TestGetOperationFormat_EmptyStorageFormat(t *testing.T) {
err = json.Unmarshal(rec.Body.Bytes(), &result)
assert.NoError(t, err)
assert.Equal(t, "json", result.Format)
assert.Equal(t, 1, result.JSONFormatVersion) // Default version is 1
}

func TestGetOperationFormat_UnknownFormat(t *testing.T) {
Expand Down Expand Up @@ -1315,6 +1320,7 @@ func TestGetOperationFormat_UnknownFormat(t *testing.T) {
err = json.Unmarshal(rec.Body.Bytes(), &result)
assert.NoError(t, err)
assert.Equal(t, "json", result.Format) // Should fallback to json
assert.Equal(t, 1, result.JSONFormatVersion) // Default version is 1
}

func TestGetOperationManifest_FlatBuffers(t *testing.T) {
Expand Down
51 changes: 35 additions & 16 deletions internal/server/operation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,16 @@ import (
)

type Operation struct {
ID int64 `json:"id"`
WorldName string `json:"world_name"`
MissionName string `json:"mission_name"`
MissionDuration float64 `json:"mission_duration"`
Filename string `json:"filename"`
Date string `json:"date"`
Tag string `json:"tag"`
StorageFormat string `json:"storageFormat"`
ConversionStatus string `json:"conversionStatus"`
ID int64 `json:"id"`
WorldName string `json:"world_name"`
MissionName string `json:"mission_name"`
MissionDuration float64 `json:"mission_duration"`
Filename string `json:"filename"`
Date string `json:"date"`
Tag string `json:"tag"`
StorageFormat string `json:"storageFormat"`
ConversionStatus string `json:"conversionStatus"`
JSONFormatVersion int `json:"jsonFormatVersion,omitempty"`
}

type Filter struct {
Expand Down Expand Up @@ -124,6 +125,18 @@ func (r *RepoOperation) migration() (err error) {
}
}

if version < 4 {
_, err = r.db.Exec(`ALTER TABLE operations ADD COLUMN json_format_version INTEGER DEFAULT 1`)
if err != nil {
return fmt.Errorf("merge db to v4 failed (json_format_version): %w", err)
}

_, err = r.db.Exec(`INSERT INTO version (db) VALUES (4)`)
if err != nil {
return fmt.Errorf("failed to increase version 4: %w", err)
}
}

return nil
}

Expand Down Expand Up @@ -160,12 +173,16 @@ func (r *RepoOperation) Store(ctx context.Context, operation *Operation) error {
if conversionStatus == "" {
conversionStatus = "pending"
}
jsonFormatVersion := operation.JSONFormatVersion
if jsonFormatVersion == 0 {
jsonFormatVersion = 1
}

query := `
INSERT INTO operations
(world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status)
(world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status, json_format_version)
VALUES
($1, $2, $3, $4, $5, $6, $7, $8)
($1, $2, $3, $4, $5, $6, $7, $8, $9)
`
result, err := r.db.ExecContext(
ctx,
Expand All @@ -178,6 +195,7 @@ func (r *RepoOperation) Store(ctx context.Context, operation *Operation) error {
operation.Tag,
storageFormat,
conversionStatus,
jsonFormatVersion,
)
if err != nil {
return err
Expand Down Expand Up @@ -206,7 +224,7 @@ func (r *RepoOperation) Select(ctx context.Context, filter Filter) ([]Operation,

query := `
SELECT
id, world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status
id, world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status, json_format_version
FROM
operations
WHERE
Expand Down Expand Up @@ -247,6 +265,7 @@ func (*RepoOperation) scan(ctx context.Context, rows *sql.Rows) ([]Operation, er
&o.Tag,
&o.StorageFormat,
&o.ConversionStatus,
&o.JSONFormatVersion,
)
if err != nil {
return nil, err
Expand All @@ -259,12 +278,12 @@ func (*RepoOperation) scan(ctx context.Context, rows *sql.Rows) ([]Operation, er
// GetByID retrieves a single operation by its ID
func (r *RepoOperation) GetByID(ctx context.Context, id string) (*Operation, error) {
row := r.db.QueryRowContext(ctx,
`SELECT id, world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status
`SELECT id, world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status, json_format_version
FROM operations WHERE id = ?`, id)

var op Operation
err := row.Scan(&op.ID, &op.WorldName, &op.MissionName, &op.MissionDuration,
&op.Filename, &op.Date, &op.Tag, &op.StorageFormat, &op.ConversionStatus)
&op.Filename, &op.Date, &op.Tag, &op.StorageFormat, &op.ConversionStatus, &op.JSONFormatVersion)
if err != nil {
return nil, err
}
Expand All @@ -274,7 +293,7 @@ func (r *RepoOperation) GetByID(ctx context.Context, id string) (*Operation, err
// SelectPending returns operations with pending conversion status
func (r *RepoOperation) SelectPending(ctx context.Context, limit int) ([]Operation, error) {
rows, err := r.db.QueryContext(ctx,
`SELECT id, world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status
`SELECT id, world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status, json_format_version
FROM operations
WHERE conversion_status = 'pending'
ORDER BY id ASC
Expand All @@ -290,7 +309,7 @@ func (r *RepoOperation) SelectPending(ctx context.Context, limit int) ([]Operati
// SelectAll returns all operations for conversion
func (r *RepoOperation) SelectAll(ctx context.Context) ([]Operation, error) {
rows, err := r.db.QueryContext(ctx,
`SELECT id, world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status
`SELECT id, world_name, mission_name, mission_duration, filename, date, tag, storage_format, conversion_status, json_format_version
FROM operations
ORDER BY id ASC`)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion internal/server/operation_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -387,7 +387,7 @@ func TestMigrationRerun(t *testing.T) {
var version int
err = repo2.db.QueryRow("SELECT db FROM version ORDER BY db DESC LIMIT 1").Scan(&version)
assert.NoError(t, err)
assert.Equal(t, 3, version)
assert.Equal(t, 4, version)
}

func TestGetTypesEmpty(t *testing.T) {
Expand Down
68 changes: 68 additions & 0 deletions internal/storage/VERSIONING.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
# JSON Format Versioning

## Adding a New Version

When the OCAP JSON format changes, follow these steps:

### 1. Increment the Version Constant

In `version.go`:

```go
const (
JSONVersionV1 JSONVersion = 1
JSONVersionV2 JSONVersion = 2 // Add new version
CurrentJSONVersion = JSONVersionV2 // Update current
)
```

### 2. Update Version Detection

In `version.go`, add detection logic for the new format:

```go
func DetectJSONVersion(data map[string]interface{}) JSONVersion {
// Check for V2-specific fields first (newest versions first)
if _, ok := data["newV2Field"]; ok {
return JSONVersionV2
}
// ... existing V1 detection ...
}
```

### 3. Create Parser for New Version

Create `parser_v2.go`:

```go
func init() {
RegisterParser(&ParserV2{})
}

type ParserV2 struct{}

func (p *ParserV2) Version() JSONVersion { return JSONVersionV2 }

func (p *ParserV2) Parse(data map[string]interface{}, chunkSize uint32) (*ParseResult, error) {
// Handle new format, output same ParseResult structure
}
```

### 4. Handle Field Renames/Migrations

If a field was renamed, handle both old and new names:

```go
func getFieldWithFallback(data map[string]interface{}, newName, oldName string) interface{} {
if v, ok := data[newName]; ok {
return v
}
return data[oldName]
}
```

## Version History

| Version | Date | Changes |
|---------|------|---------|
| V1 | Original | Initial OCAP format |
Loading
Loading