From 5606f0b9573d1e38fc38d4e38682b78e17625e65 Mon Sep 17 00:00:00 2001 From: Tit Petric Date: Mon, 8 Oct 2018 15:15:44 +0200 Subject: [PATCH] upd(sam): update migrations --- sam/db/migrate.go | 75 +++++++++++++++++++++--------- sam/db/migrate_test.go | 4 +- sam/db/mysql/statik.go | 3 +- sam/db/schema/mysql/migrations.sql | 7 +++ 4 files changed, 65 insertions(+), 24 deletions(-) create mode 100644 sam/db/schema/mysql/migrations.sql diff --git a/sam/db/migrate.go b/sam/db/migrate.go index 350107cf9..33ce3bc0e 100644 --- a/sam/db/migrate.go +++ b/sam/db/migrate.go @@ -4,8 +4,10 @@ import ( "fmt" "log" "os" + "path/filepath" "regexp" "sort" + "strings" _ "github.com/crusttech/crust/sam/db/mysql" "github.com/pkg/errors" @@ -22,6 +24,13 @@ func statements(contents []byte, err error) ([]string, error) { return regexp.MustCompilePOSIX(";$").Split(string(contents), -1), nil } +type migration struct { + Project string + Filename string + StatementIndex int `db:"statement_index"` + Status string +} + func Migrate(db *factory.DB) error { statikFS, err := fs.New() if err != nil { @@ -30,12 +39,15 @@ func Migrate(db *factory.DB) error { var files []string - fs.Walk(statikFS, "/", func(filename string, info os.FileInfo, err error) error { - if len(filename) > 4 && filename[len(filename)-4:] == ".sql" { + if err := fs.Walk(statikFS, "/", func(filename string, info os.FileInfo, err error) error { + matched, err := filepath.Match("/*.up.sql", filename) + if matched { files = append(files, filename) } - return nil - }) + return err + }); err != nil { + return errors.Wrap(err, "Error when listing files for migrations") + } sort.Strings(files) @@ -43,37 +55,56 @@ func Migrate(db *factory.DB) error { return errors.New("No files encoded for migration, need at least one SQL file") } - // @todo: create table migrations by hand, log each filename status/date - /* - create table if not exists migrations ( - filename string, status [ok, fail], message string (may be error text?), stamp datetime - ) - */ - - for _, filename := range files { - // @todo: select * from migrations to check if (filename) was processed, skip if yes - - stmts, err := statements(fs.ReadFile(statikFS, filename)) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Error applying migration %s", filename)) + migrate := func(filename string, useLog bool) error { + status := migration{ + Project: "sam", + Filename: filename, + } + if useLog { + if err := db.Get(&status, "select * from migrations where project=? and filename=?", status.Project, status.Filename); err != nil { + return err + } + if status.Status == "ok" { + return nil + } } up := func() error { + stmts, err := statements(fs.ReadFile(statikFS, filename)) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Error reading migration %s", filename)) + } + log.Println("Running migration for", filename) - for _, query := range stmts { - if _, err := db.Exec(query); err != nil { - if fmt.Sprintf("%s", err) != "exec query failed: Error 1065: Query was empty" { + for idx, query := range stmts { + if strings.TrimSpace(query) != "" && idx >= status.StatementIndex { + status.StatementIndex = idx + if _, err := db.Exec(query); err != nil { return err } } } log.Println("Migration done OK") - - // @todo: insert/update into migrations with (filename) + status.Status = "ok" return nil } if err := db.Transaction(up); err != nil { + status.Status = err.Error() + if useLog { + db.Replace("migrations", status) + } + return err + } + return nil + } + + if err := migrate("/migrations.sql", false); err != nil { + return err + } + + for _, filename := range files { + if err := migrate(filename, true); err != nil { return err } } diff --git a/sam/db/migrate_test.go b/sam/db/migrate_test.go index 45eaa779a..e07143cad 100644 --- a/sam/db/migrate_test.go +++ b/sam/db/migrate_test.go @@ -2,6 +2,7 @@ package db import ( "context" + "log" "testing" "time" @@ -11,7 +12,7 @@ import ( func TestMigrations(t *testing.T) { args := []string{ - "--rm", + // "--rm", "-e", "MYSQL_ROOT_PASSWORD=root", "-e", "MYSQL_DATABASE=test", } @@ -23,6 +24,7 @@ func TestMigrations(t *testing.T) { mysql, err := dockertest.RunContainerContext(ctx, "titpetric/percona-xtrabackup", "3306", func(addr string) error { factory.Database.Add("default", "root:root@tcp("+addr+")/test?collation=utf8mb4_general_ci") + log.Println(addr) _, err := factory.Database.Get() count++ time.Sleep(time.Second) diff --git a/sam/db/mysql/statik.go b/sam/db/mysql/statik.go index 98788f44f..fc342fa72 100644 --- a/sam/db/mysql/statik.go +++ b/sam/db/mysql/statik.go @@ -1,5 +1,6 @@ // Code generated by statik. DO NOT EDIT. +// Package statik contains static assets. package mysql import ( @@ -7,6 +8,6 @@ import ( ) func init() { - data := "PK\x03\x04\x14\x00\x08\x00\x00\x00WBAM\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1a\x00 \x0020180704080000.base.up.sqlUT\x05\x00\x01g\xd8\xb1[-- all known organisations (crust instances) and our relation towards them\nCREATE TABLE organisations (\n id BIGINT UNSIGNED NOT NULL,\n fqn TEXT NOT NULL, -- fully qualified name of the organisation\n name TEXT NOT NULL, -- display name of the organisation\n\n created_at DATETIME NOT NULL DEFAULT NOW(),\n updated_at DATETIME NULL,\n archived_at DATETIME NULL,\n deleted_at DATETIME NULL, -- organisation soft delete\n\n PRIMARY KEY (id)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\n-- Keeps all known teams\nCREATE TABLE teams (\n id BIGINT UNSIGNED NOT NULL,\n name TEXT NOT NULL, -- display name of the team\n handle TEXT NOT NULL, -- team handle string\n\n created_at DATETIME NOT NULL DEFAULT NOW(),\n updated_at DATETIME NULL,\n archived_at DATETIME NULL,\n deleted_at DATETIME NULL, -- team soft delete\n\n PRIMARY KEY (id)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\n-- Keeps all known channels\nCREATE TABLE channels (\n id BIGINT UNSIGNED NOT NULL,\n name TEXT NOT NULL, -- display name of the channel\n topic TEXT NOT NULL,\n meta JSON NOT NULL,\n\n type ENUM ('private', 'public', 'group') NOT NULL DEFAULT 'public',\n\n rel_organisation BIGINT UNSIGNED NOT NULL REFERENCES organisation(id),\n rel_creator BIGINT UNSIGNED NOT NULL,\n\n created_at DATETIME NOT NULL DEFAULT NOW(),\n updated_at DATETIME NULL,\n archived_at DATETIME NULL,\n deleted_at DATETIME NULL, -- channel soft delete\n\n rel_last_message BIGINT UNSIGNED NOT NULL DEFAULT 0,\n\n PRIMARY KEY (id)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\n-- Keeps team memberships\nCREATE TABLE team_members (\n rel_team BIGINT UNSIGNED NOT NULL REFERENCES organisation(id),\n rel_user BIGINT UNSIGNED NOT NULL,\n\n PRIMARY KEY (rel_team, rel_user)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\n-- handles channel membership\nCREATE TABLE channel_members (\n rel_channel BIGINT UNSIGNED NOT NULL REFERENCES channels(id),\n rel_user BIGINT UNSIGNED NOT NULL,\n\n type ENUM ('owner', 'member') NOT NULL DEFAULT 'member',\n\n created_at DATETIME NOT NULL DEFAULT NOW(),\n updated_at DATETIME NULL,\n\n PRIMARY KEY (rel_channel, rel_user)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\nCREATE TABLE channel_views (\n rel_channel BIGINT UNSIGNED NOT NULL REFERENCES channels(id),\n rel_user BIGINT UNSIGNED NOT NULL,\n\n -- timestamp of last view, should be enough to find out which messaghr\n viewed_at DATETIME NOT NULL DEFAULT NOW(),\n\n -- new messages count since last view\n new_since INT UNSIGNED NOT NULL DEFAULT 0,\n\n PRIMARY KEY (rel_user, rel_channel)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\nCREATE TABLE channel_pins (\n rel_channel BIGINT UNSIGNED NOT NULL REFERENCES channels(id),\n rel_message BIGINT UNSIGNED NOT NULL REFERENCES messages(id),\n rel_user BIGINT UNSIGNED NOT NULL,\n\n created_at DATETIME NOT NULL DEFAULT NOW(),\n\n PRIMARY KEY (rel_channel, rel_message)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\nCREATE TABLE messages (\n id BIGINT UNSIGNED NOT NULL,\n type TEXT,\n message TEXT NOT NULL,\n meta JSON,\n rel_user BIGINT UNSIGNED NOT NULL,\n rel_channel BIGINT UNSIGNED NOT NULL REFERENCES channels(id),\n reply_to BIGINT UNSIGNED NULL REFERENCES messages(id),\n\n created_at DATETIME NOT NULL DEFAULT NOW(),\n updated_at DATETIME NULL,\n deleted_at DATETIME NULL,\n\n PRIMARY KEY (id)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\nCREATE TABLE reactions (\n id BIGINT UNSIGNED NOT NULL,\n rel_user BIGINT UNSIGNED NOT NULL,\n rel_message BIGINT UNSIGNED NOT NULL REFERENCES messages(id),\n rel_channel BIGINT UNSIGNED NOT NULL REFERENCES channels(id),\n reaction TEXT NOT NULL,\n\n created_at DATETIME NOT NULL DEFAULT NOW(),\n\n PRIMARY KEY (id)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\nCREATE TABLE attachments (\n id BIGINT UNSIGNED NOT NULL,\n rel_user BIGINT UNSIGNED NOT NULL,\n\n url VARCHAR(512),\n preview_url VARCHAR(512),\n\n size INT UNSIGNED,\n mimetype VARCHAR(255),\n name TEXT,\n\n meta JSON,\n\n created_at DATETIME NOT NULL DEFAULT NOW(),\n updated_at DATETIME NULL,\n deleted_at DATETIME NULL,\n\n PRIMARY KEY (id)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\nCREATE TABLE message_attachment (\n rel_message BIGINT UNSIGNED NOT NULL REFERENCES messages(id),\n rel_attachment BIGINT UNSIGNED NOT NULL REFERENCES attachment(id),\n\n PRIMARY KEY (rel_message)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\nCREATE TABLE event_queue (\n id BIGINT UNSIGNED NOT NULL,\n origin BIGINT UNSIGNED NOT NULL,\n subscriber TEXT,\n payload JSON,\n\n PRIMARY KEY (id)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\nCREATE TABLE event_queue_synced (\n origin BIGINT UNSIGNED NOT NULL,\n rel_last BIGINT UNSIGNED NOT NULL,\n\n PRIMARY KEY (origin)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\nPK\x07\x08x\x16\x11\x1f\x94\x15\x00\x00\x94\x15\x00\x00PK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00WBAMx\x16\x11\x1f\x94\x15\x00\x00\x94\x15\x00\x00\x1a\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x00\x00\x00\x0020180704080000.base.up.sqlUT\x05\x00\x01g\xd8\xb1[PK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00Q\x00\x00\x00\xe5\x15\x00\x00\x00\x00" + data := "PK\x03\x04\x14\x00\x08\x00\x00\x00\xdaeHM\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1a\x00 \x0020180704080000.base.up.sqlUT\x05\x00\x01\xbcQ\xbb[-- all known organisations (crust instances) and our relation towards them\nCREATE TABLE organisations (\n id BIGINT UNSIGNED NOT NULL,\n fqn TEXT NOT NULL, -- fully qualified name of the organisation\n name TEXT NOT NULL, -- display name of the organisation\n\n created_at DATETIME NOT NULL DEFAULT NOW(),\n updated_at DATETIME NULL,\n archived_at DATETIME NULL,\n deleted_at DATETIME NULL, -- organisation soft delete\n\n PRIMARY KEY (id)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\n-- Keeps all known teams\nCREATE TABLE teams (\n id BIGINT UNSIGNED NOT NULL,\n name TEXT NOT NULL, -- display name of the team\n handle TEXT NOT NULL, -- team handle string\n\n created_at DATETIME NOT NULL DEFAULT NOW(),\n updated_at DATETIME NULL,\n archived_at DATETIME NULL,\n deleted_at DATETIME NULL, -- team soft delete\n\n PRIMARY KEY (id)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\n-- Keeps all known channels\nCREATE TABLE channels (\n id BIGINT UNSIGNED NOT NULL,\n name TEXT NOT NULL, -- display name of the channel\n topic TEXT NOT NULL,\n meta JSON NOT NULL,\n\n type ENUM ('private', 'public', 'group') NOT NULL DEFAULT 'public',\n\n rel_organisation BIGINT UNSIGNED NOT NULL REFERENCES organisation(id),\n rel_creator BIGINT UNSIGNED NOT NULL,\n\n created_at DATETIME NOT NULL DEFAULT NOW(),\n updated_at DATETIME NULL,\n archived_at DATETIME NULL,\n deleted_at DATETIME NULL, -- channel soft delete\n\n rel_last_message BIGINT UNSIGNED NOT NULL DEFAULT 0,\n\n PRIMARY KEY (id)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\n-- Keeps team memberships\nCREATE TABLE team_members (\n rel_team BIGINT UNSIGNED NOT NULL REFERENCES organisation(id),\n rel_user BIGINT UNSIGNED NOT NULL,\n\n PRIMARY KEY (rel_team, rel_user)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\n-- handles channel membership\nCREATE TABLE channel_members (\n rel_channel BIGINT UNSIGNED NOT NULL REFERENCES channels(id),\n rel_user BIGINT UNSIGNED NOT NULL,\n\n type ENUM ('owner', 'member', 'invitee') NOT NULL DEFAULT 'member',\n\n created_at DATETIME NOT NULL DEFAULT NOW(),\n updated_at DATETIME NULL,\n\n PRIMARY KEY (rel_channel, rel_user)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\nCREATE TABLE channel_views (\n rel_channel BIGINT UNSIGNED NOT NULL REFERENCES channels(id),\n rel_user BIGINT UNSIGNED NOT NULL,\n\n -- timestamp of last view, should be enough to find out which messaghr\n viewed_at DATETIME NOT NULL DEFAULT NOW(),\n\n -- new messages count since last view\n new_since INT UNSIGNED NOT NULL DEFAULT 0,\n\n PRIMARY KEY (rel_user, rel_channel)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\nCREATE TABLE channel_pins (\n rel_channel BIGINT UNSIGNED NOT NULL REFERENCES channels(id),\n rel_message BIGINT UNSIGNED NOT NULL REFERENCES messages(id),\n rel_user BIGINT UNSIGNED NOT NULL,\n\n created_at DATETIME NOT NULL DEFAULT NOW(),\n\n PRIMARY KEY (rel_channel, rel_message)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\nCREATE TABLE messages (\n id BIGINT UNSIGNED NOT NULL,\n type TEXT,\n message TEXT NOT NULL,\n meta JSON,\n rel_user BIGINT UNSIGNED NOT NULL,\n rel_channel BIGINT UNSIGNED NOT NULL REFERENCES channels(id),\n reply_to BIGINT UNSIGNED NULL REFERENCES messages(id),\n\n created_at DATETIME NOT NULL DEFAULT NOW(),\n updated_at DATETIME NULL,\n deleted_at DATETIME NULL,\n\n PRIMARY KEY (id)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\nCREATE TABLE reactions (\n id BIGINT UNSIGNED NOT NULL,\n rel_user BIGINT UNSIGNED NOT NULL,\n rel_message BIGINT UNSIGNED NOT NULL REFERENCES messages(id),\n rel_channel BIGINT UNSIGNED NOT NULL REFERENCES channels(id),\n reaction TEXT NOT NULL,\n\n created_at DATETIME NOT NULL DEFAULT NOW(),\n\n PRIMARY KEY (id)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\nCREATE TABLE attachments (\n id BIGINT UNSIGNED NOT NULL,\n rel_user BIGINT UNSIGNED NOT NULL,\n\n url VARCHAR(512),\n preview_url VARCHAR(512),\n\n size INT UNSIGNED,\n mimetype VARCHAR(255),\n name TEXT,\n\n meta JSON,\n\n created_at DATETIME NOT NULL DEFAULT NOW(),\n updated_at DATETIME NULL,\n deleted_at DATETIME NULL,\n\n PRIMARY KEY (id)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\nCREATE TABLE message_attachment (\n rel_message BIGINT UNSIGNED NOT NULL REFERENCES messages(id),\n rel_attachment BIGINT UNSIGNED NOT NULL REFERENCES attachment(id),\n\n PRIMARY KEY (rel_message)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\nCREATE TABLE event_queue (\n id BIGINT UNSIGNED NOT NULL,\n origin BIGINT UNSIGNED NOT NULL,\n subscriber TEXT,\n payload JSON,\n\n PRIMARY KEY (id)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\nCREATE TABLE event_queue_synced (\n origin BIGINT UNSIGNED NOT NULL,\n rel_last BIGINT UNSIGNED NOT NULL,\n\n PRIMARY KEY (origin)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\nPK\x07\x08\xd2g\xcd\xce\x9f\x15\x00\x00\x9f\x15\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xdaeHM\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0e\x00 \x00migrations.sqlUT\x05\x00\x01\xbcQ\xbb[CREATE TABLE IF NOT EXISTS `migrations` (\n `project` varchar(16) NOT NULL COMMENT 'sam, crm, ...',\n `filename` varchar(255) NOT NULL COMMENT 'yyyymmddHHMMSS.sql',\n `statement_index` int(11) NOT NULL COMMENT 'Statement number from SQL file',\n `status` varchar(16) NOT NULL COMMENT 'ok or full error message'\n) ENGINE=InnoDB DEFAULT CHARSET=utf8;\n\nPK\x07\x08~ 9=Z\x01\x00\x00Z\x01\x00\x00PK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xdaeHM\xd2g\xcd\xce\x9f\x15\x00\x00\x9f\x15\x00\x00\x1a\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x00\x00\x00\x0020180704080000.base.up.sqlUT\x05\x00\x01\xbcQ\xbb[PK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xdaeHM~ 9=Z\x01\x00\x00Z\x01\x00\x00\x0e\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xf0\x15\x00\x00migrations.sqlUT\x05\x00\x01\xbcQ\xbb[PK\x05\x06\x00\x00\x00\x00\x02\x00\x02\x00\x96\x00\x00\x00\x8f\x17\x00\x00\x00\x00" fs.Register(data) } diff --git a/sam/db/schema/mysql/migrations.sql b/sam/db/schema/mysql/migrations.sql new file mode 100644 index 000000000..1f2b57ee4 --- /dev/null +++ b/sam/db/schema/mysql/migrations.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS `migrations` ( + `project` varchar(16) NOT NULL COMMENT 'sam, crm, ...', + `filename` varchar(255) NOT NULL COMMENT 'yyyymmddHHMMSS.sql', + `statement_index` int(11) NOT NULL COMMENT 'Statement number from SQL file', + `status` varchar(16) NOT NULL COMMENT 'ok or full error message' +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +