feat(migrate-db): SQLite <-> .dump conversion and Download Migration in Overview

Binary: extend the migrate-db subcommand with --dump and --restore so a
SQLite database can be exported to a portable SQL text dump and rebuilt from
one, alongside the existing --dsn PostgreSQL copy. Implemented in Go via the
bundled sqlite driver (new database/dump_sqlite.go); no external sqlite3 client
is required. Add ExportPostgresToSQLite (reverse of MigrateData) to build a
SQLite .db from live PostgreSQL data, reusing the shared copyAllModels helper.

Overview: add a "Download Migration" item to Backup & Restore plus a
getMigration endpoint/service that returns a .dump on SQLite or a .db on
PostgreSQL, so the data can seed a panel on the other backend. Document the
endpoint in api-docs and translate the three new strings across all locales.

Tests: cover the destination-side copy (AutoMigrate + copyTable into SQLite)
and the dump/restore round-trip including quoted values. Ignore *.dump.

The x-ui.sh helper that drives this from the CLI is in PR #4910.
This commit is contained in:
MHSanaei
2026-06-04 15:32:22 +02:00
parent 5c1d64b841
commit a07c7b7f4e
24 changed files with 599 additions and 22 deletions

View File

@@ -86,6 +86,15 @@ func MigrateData(srcPath, dstDSN string) error {
}
}
// Empty the destination tables so the migration is idempotent: a fresh
// PostgreSQL DB already holds an auto-seeded admin (id=1) from any prior
// panel start, and a partially-failed earlier run leaves rows behind. Either
// way a plain INSERT with explicit ids would collide on users_pkey, so clear
// our tables (only) before copying.
if err := truncatePostgresTables(dst, migrationModels()); err != nil {
return fmt.Errorf("clear destination tables: %w", err)
}
totalRows := 0
for _, m := range migrationModels() {
n, err := copyTable(src, dst, m)
@@ -105,6 +114,62 @@ func MigrateData(srcPath, dstDSN string) error {
return nil
}
// ExportPostgresToSQLite copies every row from the PostgreSQL database described
// by srcDSN into a fresh SQLite file at dstPath. It is the reverse of
// MigrateData and is used to hand a PostgreSQL-backed panel a portable .db file.
// dstPath is created/overwritten; the PostgreSQL source is left untouched.
func ExportPostgresToSQLite(srcDSN, dstPath string) error {
if srcDSN == "" {
return errors.New("source DSN is required")
}
if err := os.MkdirAll(path.Dir(dstPath), 0755); err != nil {
return err
}
// Start from an empty file so AutoMigrate creates the canonical schema.
if err := os.Remove(dstPath); err != nil && !os.IsNotExist(err) {
return err
}
src, err := gorm.Open(postgres.Open(srcDSN), &gorm.Config{Logger: logger.Discard})
if err != nil {
return fmt.Errorf("open postgres source: %w", err)
}
srcSQL, err := src.DB()
if err != nil {
return err
}
defer srcSQL.Close()
// No WAL: keep all data in the main file so it is complete once closed.
dst, err := gorm.Open(sqlite.Open(dstPath+"?_busy_timeout=10000"), &gorm.Config{Logger: logger.Discard})
if err != nil {
return fmt.Errorf("open sqlite destination: %w", err)
}
dstSQL, err := dst.DB()
if err != nil {
return err
}
defer dstSQL.Close()
return copyAllModels(src, dst)
}
// copyAllModels (re)creates the schema on dst and copies every migrated table
// from src to dst in FK-safe order. src/dst may be any gorm backend.
func copyAllModels(src, dst *gorm.DB) error {
for _, m := range migrationModels() {
if err := dst.AutoMigrate(m); err != nil {
return fmt.Errorf("AutoMigrate %T: %w", m, err)
}
}
for _, m := range migrationModels() {
if _, err := copyTable(src, dst, m); err != nil {
return fmt.Errorf("copy %T: %w", m, err)
}
}
return nil
}
func copyTable(src, dst *gorm.DB, mdl any) (int, error) {
const batchSize = 500
@@ -157,6 +222,26 @@ func copyTable(src, dst *gorm.DB, mdl any) (int, error) {
return total, nil
}
// truncatePostgresTables empties every migrated table on dst in a single
// statement, resetting identity sequences. CASCADE covers the inbound/client
// foreign keys regardless of insertion order. Only the panel's own tables are
// touched, never the rest of the schema.
func truncatePostgresTables(dst *gorm.DB, models []any) error {
tables := make([]string, 0, len(models))
for _, m := range models {
stmt := &gorm.Statement{DB: dst}
if err := stmt.Parse(m); err != nil {
return err
}
tables = append(tables, `"`+stmt.Schema.Table+`"`)
}
if len(tables) == 0 {
return nil
}
log.Println("Clearing destination tables...")
return dst.Exec("TRUNCATE TABLE " + strings.Join(tables, ", ") + " RESTART IDENTITY CASCADE").Error
}
// resetPostgresSequences advances each migrated table's id sequence past MAX(id),
// otherwise the next INSERT-without-id would clash with copied rows.
func resetPostgresSequences(dst *gorm.DB) error {