Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/cli-go/docs/supabase/db/reset.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@ Resets the local database to a clean state.

Requires the local development stack to be started by running `supabase start`.

Recreates the local Postgres container and applies all local migrations found in `supabase/migrations` directory. If test data is defined in `supabase/seed.sql`, it will be seeded after the migrations are run. Any other data or schema changes made during local development will be discarded.
Recreates the local Postgres container and applies all local migrations found in `supabase/migrations` directory. If test data is defined in `supabase/seed.sql` or configured in `[db.seed] sql_paths`, it will be seeded after the migrations are run. Any other data or schema changes made during local development will be discarded.

When running db reset with `--linked` or `--db-url` flag, a SQL script is executed to identify and drop all user created entities in the remote database. Since Postgres roles are cluster level entities, any custom roles created through the dashboard or `supabase/roles.sql` will not be deleted by remote reset.
3 changes: 2 additions & 1 deletion apps/cli-go/pkg/config/templates/config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,8 @@ schema_paths = []
# If enabled, seeds the database after migrations during a db reset.
enabled = true
# Specifies an ordered list of seed files to load during db reset.
# Supports glob patterns relative to supabase directory: "./seeds/*.sql"
# Supports glob patterns relative to supabase directory: ["./seeds/*.sql", "./seeds/*.sql.gz"]
# Supports gzipped SQL files with ".sql.gz" extension.
sql_paths = ["./seed.sql"]

[db.network_restrictions]
Expand Down
3 changes: 2 additions & 1 deletion apps/cli-go/pkg/config/testdata/config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ test_key = "test_value"
# If enabled, seeds the database after migrations during a db reset.
enabled = true
# Specifies an ordered list of seed files to load during db reset.
# Supports glob patterns relative to supabase directory: "./seeds/*.sql"
# Supports glob patterns relative to supabase directory: ["./seeds/*.sql", "./seeds/*.sql.gz"]
# Supports gzipped SQL files with ".sql.gz" extension.
sql_paths = ["./seed.sql"]

[db.network_restrictions]
Expand Down
127 changes: 117 additions & 10 deletions apps/cli-go/pkg/migration/file.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@ package migration

import (
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
stderrors "errors"
"fmt"
"io"
"io/fs"
Expand All @@ -31,7 +33,12 @@ var (
typeNamePattern = regexp.MustCompile(`type "([^"]+)" does not exist`)
)

const defaultSeedFileSizeLimit = 10 << 30

func NewMigrationFromFile(path string, fsys fs.FS) (*MigrationFile, error) {
if isCompressedSQL(path) {
return nil, errors.Errorf("compressed SQL files are only supported for seed files: %s", path)
}
lines, err := parseFile(path, fsys)
if err != nil {
return nil, err
Expand All @@ -53,14 +60,17 @@ func parseFile(path string, fsys fs.FS) ([]string, error) {
return nil, errors.Errorf("failed to open migration file: %w", err)
}
defer sql.Close()
// Unless explicitly specified, Use file length as max buffer size
if !viper.IsSet("SCANNER_BUFFER_SIZE") {
if fi, err := sql.Stat(); err == nil {
if size := int(fi.Size()); size > parser.MaxScannerCapacity {
parser.MaxScannerCapacity = size
}
}
setScannerCapacity(fileSize(sql))
return parser.SplitAndTrim(sql)
}

func parseSeedFile(path string, fsys fs.FS) ([]string, error) {
sql, scannerBufferSize, err := openSeedSQL(path, fsys)
if err != nil {
return nil, err
}
defer sql.Close()
setScannerCapacity(scannerBufferSize)
return parser.SplitAndTrim(sql)
}

Expand Down Expand Up @@ -182,11 +192,13 @@ type SeedFile struct {
}

func NewSeedFile(path string, fsys fs.FS) (*SeedFile, error) {
sql, err := fsys.Open(path)
sql, _, err := openSeedSQL(path, fsys)
if err != nil {
return nil, errors.Errorf("failed to open seed file: %w", err)
return nil, err
}
defer sql.Close()
// Seed history hashes the decompressed SQL so gzip metadata changes and
// equivalent recompressions do not mark the seed dirty.
hash := sha256.New()
if _, err := io.Copy(hash, sql); err != nil {
return nil, errors.Errorf("failed to hash file: %w", err)
Expand All @@ -195,9 +207,104 @@ func NewSeedFile(path string, fsys fs.FS) (*SeedFile, error) {
return &SeedFile{Path: path, Hash: digest}, nil
}

func openSeedSQL(path string, fsys fs.FS) (io.ReadCloser, int, error) {
sql, err := fsys.Open(path)
if err != nil {
return nil, 0, errors.Errorf("failed to open seed file: %w", err)
}
if !isCompressedSQL(path) {
return sql, fileSize(sql), nil
}
gz, err := gzip.NewReader(sql)
if err != nil {
_ = sql.Close()
return nil, 0, errors.Errorf("failed to decompress seed file: %w", err)
}
limit := seedFileSizeLimit()
return &compressedSQLReader{
gz: gz,
file: sql,
maxBytes: limit,
}, safeInt(limit), nil
}

func setScannerCapacity(scannerBufferSize int) {
// Unless explicitly specified, use file length as max buffer size.
if !viper.IsSet("SCANNER_BUFFER_SIZE") {
if scannerBufferSize > parser.MaxScannerCapacity {
parser.MaxScannerCapacity = scannerBufferSize
}
}
}

func fileSize(sql fs.File) int {
info, err := sql.Stat()
if err != nil {
return 0
}
return safeInt(info.Size())
}

func seedFileSizeLimit() int64 {
limit := int64(viper.GetSizeInBytes("SEED_FILE_SIZE_LIMIT"))
if limit <= 0 {
return defaultSeedFileSizeLimit
}
return limit
}

func safeInt(size int64) int {
if size <= 0 {
return 0
}
maxInt := int64(^uint(0) >> 1)
if size > maxInt {
return int(maxInt)
}
return int(size)
}

func isCompressedSQL(path string) bool {
return strings.HasSuffix(strings.ToLower(path), ".sql.gz")
}

type compressedSQLReader struct {
gz *gzip.Reader
file fs.File
maxBytes int64
bytesRead int64
}

func (r *compressedSQLReader) Read(p []byte) (int, error) {
if r.maxBytes > 0 {
remaining := r.maxBytes - r.bytesRead
if remaining <= 0 {
var probe [1]byte
n, err := r.gz.Read(probe[:])
if n > 0 {
return 0, errors.Errorf("decompressed seed file exceeds %d bytes", r.maxBytes)
}
return 0, err
}
if int64(len(p)) > remaining {
p = p[:remaining]
}
}
n, err := r.gz.Read(p)
r.bytesRead += int64(n)
if n > 0 && err == io.EOF {
return n, nil
}
return n, err
}

func (r *compressedSQLReader) Close() error {
return stderrors.Join(r.gz.Close(), r.file.Close())
}

func (m *SeedFile) ExecBatchWithCache(ctx context.Context, conn *pgx.Conn, fsys fs.FS) error {
// Parse each file individually to reduce memory usage
lines, err := parseFile(m.Path, fsys)
lines, err := parseSeedFile(m.Path, fsys)
if err != nil {
return err
}
Expand Down
8 changes: 8 additions & 0 deletions apps/cli-go/pkg/migration/file_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,14 @@ func TestMigrationFile(t *testing.T) {
assert.Equal(t, "20220727064247", migration.Version)
})

t.Run("new from gzipped file returns clear error", func(t *testing.T) {
// Run test
migration, err := NewMigrationFromFile("20220727064247_create_table.sql.gz", fs.MapFS{})
// Check error
assert.ErrorContains(t, err, "compressed SQL files are only supported for seed files")
assert.Nil(t, migration)
})

t.Run("new from reader errors on max token", func(t *testing.T) {
viper.Reset()
sql := "\tBEGIN; " + strings.Repeat("a", parser.MaxScannerCapacity)
Expand Down
Loading
Loading