Improve RDBMS DDL manipulation

store/mysql/upgrade.go

@@ -27,7 +27,6 @@ func NewUpgrader(log *zap.Logger, store *Store) *upgrader {
 
 	// All modifications we need for the DDL generator
 	// to properly support MySQL dialect:
-
 	u.ddl.AddTemplate("create-table-suffix", "ENGINE=InnoDB DEFAULT CHARSET=utf8")
 
 	// Sadly, MySQL does not support partial indexes
@@ -166,63 +165,99 @@ func (u upgrader) TableSchema(ctx context.Context, table string) (ddl.Columns, e
 }
 
 // AddColumn adds column to table
-// @todo extract column lookup
 func (u upgrader) AddColumn(ctx context.Context, table string, col *ddl.Column) (added bool, err error) {
-	var (
-		lookup = `SELECT IS_NULLABLE = 'YES' AS IS_NULLABLE,
-                         DATA_TYPE
-                    FROM INFORMATION_SCHEMA.COLUMNS 
-                   WHERE TABLE_SCHEMA = ? 
-                     AND TABLE_NAME = ?
-                     AND COLUMN_NAME = ?`
-
-		tmp struct {
-			IsNullable bool   `db:"IS_NULLABLE"`
-			DataType   string `db:"DATA_TYPE"`
+	err = func() error {
+		var columns ddl.Columns
+		if columns, err = u.getColumns(ctx, table); err != nil {
+			return err
+		}
+
+		if columns.Get(col.Name) != nil {
+			return nil
 		}
-	)
 
-	if err = u.s.DB().GetContext(ctx, &tmp, lookup, u.s.Config().DBName, table, col.Name); err == sql.ErrNoRows {
 		if err = u.Exec(ctx, u.ddl.AddColumn(table, col)); err != nil {
-			return false, fmt.Errorf("could not add column %s to table %s: %w", table, col.Name, err)
+			return err
 		}
 
-		return true, nil
-	} else if err != nil {
-		return false, fmt.Errorf("could not check if column exists: %w", err)
+		added = true
+		return nil
+	}()
+
+	if err != nil {
+		return false, fmt.Errorf("could not add column %q to %q: %w", col.Name, table, err)
 	}
 
-	return false, nil
+	return
 }
 
 // DropColumn drops column from table
-// @todo extract column lookup
 func (u upgrader) DropColumn(ctx context.Context, table, column string) (dropped bool, err error) {
-	var (
-		lookup = `SELECT IS_NULLABLE = 'YES' AS IS_NULLABLE,
-                         DATA_TYPE
-                    FROM INFORMATION_SCHEMA.COLUMNS 
-                   WHERE TABLE_SCHEMA = ? 
-                     AND TABLE_NAME = ?
-                     AND COLUMN_NAME = ?`
-
-		tmp struct {
-			IsNullable bool   `db:"IS_NULLABLE"`
-			DataType   string `db:"DATA_TYPE"`
+	err = func() error {
+		var columns ddl.Columns
+		if columns, err = u.getColumns(ctx, table); err != nil {
+			return err
+		}
+
+		if columns.Get(column) == nil {
+			return nil
 		}
-	)
 
-	if err = u.s.DB().GetContext(ctx, &tmp, lookup, u.s.Config().DBName, table, column); err == nil {
 		if err = u.Exec(ctx, u.ddl.DropColumn(table, column)); err != nil {
-			return false, fmt.Errorf("could not add column %s to table %s: %w", table, column, err)
+			return err
 		}
 
-		return true, nil
-	} else if err != sql.ErrNoRows {
-		return false, fmt.Errorf("could not check if column exists: %w", err)
+		dropped = true
+		return nil
+	}()
+
+	if err != nil {
+		return false, fmt.Errorf("could not drop column %q from %q: %w", column, table, err)
 	}
 
-	return false, nil
+	return
+}
+
+// RenameColumn renames column on a table
+func (u upgrader) RenameColumn(ctx context.Context, table, oldName, newName string) (changed bool, err error) {
+	err = func() error {
+		if oldName == newName {
+			return nil
+		}
+
+		var columns ddl.Columns
+		if columns, err = u.getColumns(ctx, table); err != nil {
+			return err
+		}
+
+		if columns.Get(oldName) == nil {
+			// Old column does not exist anymore
+
+			if columns.Get(newName) == nil {
+				return fmt.Errorf("old and new columns are missing")
+			}
+
+			return nil
+		}
+
+		if columns.Get(newName) != nil {
+			return fmt.Errorf("new column already exists")
+
+		}
+
+		if err = u.Exec(ctx, u.ddl.RenameColumn(table, oldName, newName)); err != nil {
+			return err
+		}
+
+		changed = true
+		return nil
+	}()
+
+	if err != nil {
+		return false, fmt.Errorf("could not rename column %q on table %q to %q: %w", oldName, table, newName, err)
+	}
+
+	return
 }
 
 func (u upgrader) AddPrimaryKey(ctx context.Context, table string, ind *ddl.Index) (added bool, err error) {
@@ -232,3 +267,41 @@ func (u upgrader) AddPrimaryKey(ctx context.Context, table string, ind *ddl.Inde
 
 	return true, nil
 }
+
+// loads and returns all tables columns
+func (u upgrader) getColumns(ctx context.Context, table string) (out ddl.Columns, err error) {
+	type (
+		col struct {
+			Name       string `db:"COLUMN_NAME"`
+			IsNullable bool   `db:"IS_NULLABLE"`
+			DataType   string `db:"DATA_TYPE"`
+		}
+	)
+
+	var (
+		lookup = `SELECT COLUMN_NAME,
+                         IS_NULLABLE = 'YES' AS IS_NULLABLE,
+                         DATA_TYPE
+                    FROM INFORMATION_SCHEMA.COLUMNS 
+                   WHERE TABLE_SCHEMA = ? 
+                     AND TABLE_NAME = ?`
+
+		cols []*col
+	)
+
+	if err = u.s.DB().SelectContext(ctx, &cols, lookup, u.s.Config().DBName, table); err != nil {
+		return nil, err
+	}
+
+	out = make([]*ddl.Column, len(cols))
+	for i := range cols {
+		out[i] = &ddl.Column{
+			Name: cols[i].Name,
+			//Type:         ddl.ColumnType{},
+			IsNull: cols[i].IsNullable,
+			//DefaultValue: "",
+		}
+	}
+
+	return out, nil
+}

store/pgsql/upgrade.go

@@ -5,7 +5,6 @@ package pgsql
 
 import (
 	"context"
-	"database/sql"
 	"fmt"
 	"github.com/cortezaproject/corteza-server/store/rdbms"
 	"github.com/cortezaproject/corteza-server/store/rdbms/ddl"
@@ -114,63 +113,99 @@ func (u upgrader) TableSchema(ctx context.Context, table string) (ddl.Columns, e
 }
 
 // AddColumn adds column to table
-// @todo extract column lookup
 func (u upgrader) AddColumn(ctx context.Context, table string, col *ddl.Column) (added bool, err error) {
-	var (
-		lookup = `SELECT is_nullable = 'YES' AS is_nullable,
-                         data_type
-                    FROM information_schema.columns 
-                   WHERE table_catalog = $1 
-                     AND table_name = $2
-                     AND column_name = $3`
-
-		tmp struct {
-			IsNullable bool   `db:"is_nullable"`
-			DataType   string `db:"data_type"`
+	err = func() error {
+		var columns ddl.Columns
+		if columns, err = u.getColumns(ctx, table); err != nil {
+			return err
+		}
+
+		if columns.Get(col.Name) != nil {
+			return nil
 		}
-	)
 
-	if err = u.s.DB().GetContext(ctx, &tmp, lookup, u.s.Config().DBName, table, col.Name); err == sql.ErrNoRows {
 		if err = u.Exec(ctx, u.ddl.AddColumn(table, col)); err != nil {
-			return false, fmt.Errorf("could not add column %s to table %s: %w", table, col.Name, err)
+			return err
 		}
 
-		return true, nil
-	} else if err != nil {
-		return false, fmt.Errorf("could not check if column exists: %w", err)
+		added = true
+		return nil
+	}()
+
+	if err != nil {
+		return false, fmt.Errorf("could not add column %q to %q: %w", col.Name, table, err)
 	}
 
-	return false, nil
+	return
 }
 
 // DropColumn drops column from table
-// @todo extract column lookup
 func (u upgrader) DropColumn(ctx context.Context, table, column string) (dropped bool, err error) {
-	var (
-		lookup = `SELECT is_nullable = 'YES' AS is_nullable,
-                         data_type
-                    FROM information_schema.columns 
-                   WHERE table_catalog = $1 
-                     AND table_name = $2
-                     AND column_name = $3`
-
-		tmp struct {
-			IsNullable bool   `db:"is_nullable"`
-			DataType   string `db:"data_type"`
+	err = func() error {
+		var columns ddl.Columns
+		if columns, err = u.getColumns(ctx, table); err != nil {
+			return err
+		}
+
+		if columns.Get(column) == nil {
+			return nil
 		}
-	)
 
-	if err = u.s.DB().GetContext(ctx, &tmp, lookup, u.s.Config().DBName, table, column); err == nil {
 		if err = u.Exec(ctx, u.ddl.DropColumn(table, column)); err != nil {
-			return false, fmt.Errorf("could not add column %s to table %s: %w", table, column, err)
+			return err
 		}
 
-		return true, nil
-	} else if err != sql.ErrNoRows {
-		return false, fmt.Errorf("could not check if column exists: %w", err)
+		dropped = true
+		return nil
+	}()
+
+	if err != nil {
+		return false, fmt.Errorf("could not drop column %q from %q: %w", column, table, err)
 	}
 
-	return false, nil
+	return
+}
+
+// RenameColumn renames column on a table
+func (u upgrader) RenameColumn(ctx context.Context, table, oldName, newName string) (changed bool, err error) {
+	err = func() error {
+		if oldName == newName {
+			return nil
+		}
+
+		var columns ddl.Columns
+		if columns, err = u.getColumns(ctx, table); err != nil {
+			return err
+		}
+
+		if columns.Get(oldName) == nil {
+			// Old column does not exist anymore
+
+			if columns.Get(newName) == nil {
+				return fmt.Errorf("old and new columns are missing")
+			}
+
+			return nil
+		}
+
+		if columns.Get(newName) != nil {
+			return fmt.Errorf("new column already exists")
+
+		}
+
+		if err = u.Exec(ctx, u.ddl.RenameColumn(table, oldName, newName)); err != nil {
+			return err
+		}
+
+		changed = true
+		return nil
+	}()
+
+	if err != nil {
+		return false, fmt.Errorf("could not rename column %q on table %q to %q: %w", oldName, table, newName, err)
+	}
+
+	return
 }
 
 func (u upgrader) AddPrimaryKey(ctx context.Context, table string, ind *ddl.Index) (added bool, err error) {
@@ -180,3 +215,41 @@ func (u upgrader) AddPrimaryKey(ctx context.Context, table string, ind *ddl.Inde
 
 	return true, nil
 }
+
+// loads and returns all tables columns
+func (u upgrader) getColumns(ctx context.Context, table string) (out ddl.Columns, err error) {
+	type (
+		col struct {
+			Name       string `db:"column_name"`
+			IsNullable bool   `db:"is_nullable"`
+			DataType   string `db:"data_type"`
+		}
+	)
+
+	var (
+		lookup = `SELECT column_name,
+                         is_nullable = 'YES' AS is_nullable,
+                         data_type
+                    FROM information_schema.columns 
+                   WHERE table_catalog = $1 
+                     AND table_name = $2`
+
+		cols []*col
+	)
+
+	if err = u.s.DB().SelectContext(ctx, &cols, lookup, u.s.Config().DBName, table); err != nil {
+		return nil, err
+	}
+
+	out = make([]*ddl.Column, len(cols))
+	for i := range cols {
+		out[i] = &ddl.Column{
+			Name: cols[i].Name,
+			//Type:         ddl.ColumnType{},
+			IsNull: cols[i].IsNullable,
+			//DefaultValue: "",
+		}
+	}
+
+	return out, nil
+}

store/rdbms/ddl/def.go

@@ -219,3 +219,13 @@ func IWhere(cnd string) indexManipulator {
 		i.Condition = cnd
 	}
 }
+
+func (cc Columns) Get(name string) *Column {
+	for c := range cc {
+		if cc[c].Name == name {
+			return cc[c]
+		}
+	}
+
+	return nil
+}

store/rdbms/ddl/gen.go

@@ -15,6 +15,9 @@ type (
 	}
 )
 
+// @todo all DDL operations (altering, droping, adding...) from all *sql* implementations should be moved here
+//       logic is more or less the same with a differet approach on how to read schema specs (tables, columns)
+
 const (
 	// table creation
 	genericCreateTable = `
@@ -38,6 +41,7 @@ CREATE TABLE {{ .Name }} (
 	genericAddColumn     = `ALTER TABLE {{ .Table }} ADD {{ template "create-table-column" .Column }}`
 	genericAddPrimaryKey = `ALTER TABLE {{ .Table }} ADD CONSTRAINT PRIMARY KEY {{ template "index-fields" .PrimaryKey.Fields }}`
 	genericDropColumn    = `ALTER TABLE {{ .Table }} DROP {{ .Column }}`
+	genericRenameColumn  = `ALTER TABLE {{ .Table }} RENAME COLUMN {{ .OldName }} TO {{ .NewName }}`
 
 	// index creation
 	genericCreateIndex = `CREATE {{ if .Unique }}UNIQUE {{ end }}INDEX {{ template "index-name" . }} ON {{ .Table }} {{ template "index-fields" .Fields }}{{ template "index-condition" . }}`
@@ -66,6 +70,7 @@ func NewGenerator(log *zap.Logger) *Generator {
 	g.AddTemplate("add-column", genericAddColumn)
 	g.AddTemplate("add-primary-key", genericAddPrimaryKey)
 	g.AddTemplate("drop-column", genericDropColumn)
+	g.AddTemplate("rename-column", genericRenameColumn)
 	g.AddTemplate("create-index", genericCreateIndex)
 	g.AddTemplate("index-condition", genericIndexCondition)
 	g.AddTemplate("index-name", genericIndexName)
@@ -113,6 +118,14 @@ func (g *Generator) DropColumn(table, column string) string {
 	})
 }
 
+func (g *Generator) RenameColumn(table, oldName, newName string) string {
+	return g.executeTemplate("rename-column", map[string]interface{}{
+		"Table":   table,
+		"OldName": oldName,
+		"NewName": newName,
+	})
+}
+
 func (g *Generator) AddPrimaryKey(table string, pk *Index) string {
 	return g.executeTemplate("add-primary-key", map[string]interface{}{
 		"Table":      table,

store/rdbms/generic_upgrades.go

@@ -18,6 +18,7 @@ type (
 		AddColumn(context.Context, string, *ddl.Column) (bool, error)
 		DropTable(context.Context, string) (bool, error)
 		DropColumn(context.Context, string, string) (bool, error)
+		RenameColumn(context.Context, string, string, string) (bool, error)
 		AddPrimaryKey(context.Context, string, *ddl.Index) (bool, error)
 		Exec(context.Context, string, ...interface{}) error
 	}

store/sqlite/upgrade.go

@@ -102,10 +102,6 @@ func (u upgrader) TableExists(ctx context.Context, table string) (bool, error) {
 	return exists, nil
 }
 
-func (u upgrader) TableSchema(ctx context.Context, table string) (ddl.Columns, error) {
-	return nil, fmt.Errorf("pending implementation")
-}
-
 func (u upgrader) DropTable(ctx context.Context, table string) (dropped bool, err error) {
 	var exists bool
 	exists, err = u.TableExists(ctx, table)
@@ -121,64 +117,141 @@ func (u upgrader) DropTable(ctx context.Context, table string) (dropped bool, er
 	return true, nil
 }
 
+func (u upgrader) TableSchema(ctx context.Context, table string) (ddl.Columns, error) {
+	return nil, fmt.Errorf("pending implementation")
+}
+
 // AddColumn adds column to table
-// @todo extract column lookup
 func (u upgrader) AddColumn(ctx context.Context, table string, col *ddl.Column) (added bool, err error) {
-	var (
-		lookup = fmt.Sprintf(`PRAGMA TABLE_INFO(%q)`, table)
-
-		tmp []struct {
-			CID          int            `db:"cid"`
-			Name         string         `db:"name"`
-			NotNull      bool           `db:"notnull"`
-			PrimaryKey   bool           `db:"pk"`
-			DefaultValue sql.NullString `db:"dflt_value"`
-			Type         string         `db:"type"`
+	err = func() error {
+		var columns ddl.Columns
+		if columns, err = u.getColumns(ctx, table); err != nil {
+			return err
+		}
+
+		if columns.Get(col.Name) != nil {
+			return nil
 		}
-	)
 
-	if err = u.s.DB().SelectContext(ctx, &tmp, lookup); err == sql.ErrNoRows {
 		if err = u.Exec(ctx, u.ddl.AddColumn(table, col)); err != nil {
-			return false, fmt.Errorf("could not add column %s to table %s: %w", table, col.Name, err)
+			return err
 		}
 
-		return true, nil
-	} else if err != nil {
-		return false, fmt.Errorf("could not check if column exists: %w", err)
+		added = true
+		return nil
+	}()
+
+	if err != nil {
+		return false, fmt.Errorf("could not add column %q to %q: %w", col.Name, table, err)
 	}
 
-	return false, nil
+	return
 }
 
 // DropColumn drops column from table
-// @todo extract column lookup
 func (u upgrader) DropColumn(ctx context.Context, table, column string) (dropped bool, err error) {
-	var (
-		lookup = fmt.Sprintf(`PRAGMA TABLE_INFO(%q)`, table)
-
-		tmp []struct {
-			CID          int            `db:"cid"`
-			Name         string         `db:"name"`
-			NotNull      bool           `db:"notnull"`
-			PrimaryKey   bool           `db:"pk"`
-			DefaultValue sql.NullString `db:"dflt_value"`
-			Type         string         `db:"type"`
+	err = func() error {
+		var columns ddl.Columns
+		if columns, err = u.getColumns(ctx, table); err != nil {
+			return err
+		}
+
+		if columns.Get(column) == nil {
+			return nil
 		}
-	)
 
-	if err = u.s.DB().SelectContext(ctx, &tmp, lookup); err == sql.ErrNoRows {
 		if err = u.Exec(ctx, u.ddl.DropColumn(table, column)); err != nil {
-			return false, fmt.Errorf("could not add column %s to table %s: %w", table, column, err)
+			return err
 		}
 
-		return true, nil
-	} else if err != nil {
-		return false, fmt.Errorf("could not check if column exists: %w", err)
+		dropped = true
+		return nil
+	}()
+
+	if err != nil {
+		return false, fmt.Errorf("could not drop column %q from %q: %w", column, table, err)
 	}
 
-	return false, nil
+	return
+}
+
+// RenameColumn renames column on a table
+func (u upgrader) RenameColumn(ctx context.Context, table, oldName, newName string) (changed bool, err error) {
+	err = func() error {
+		if oldName == newName {
+			return nil
+		}
+
+		var columns ddl.Columns
+		if columns, err = u.getColumns(ctx, table); err != nil {
+			return err
+		}
+
+		if columns.Get(oldName) == nil {
+			// Old column does not exist anymore
+
+			if columns.Get(newName) == nil {
+				return fmt.Errorf("old and new columns are missing")
+			}
+
+			return nil
+		}
+
+		if columns.Get(newName) != nil {
+			return fmt.Errorf("new column already exists")
+
+		}
+
+		if err = u.Exec(ctx, u.ddl.RenameColumn(table, oldName, newName)); err != nil {
+			return err
+		}
+
+		changed = true
+		return nil
+	}()
+
+	if err != nil {
+		return false, fmt.Errorf("could not rename column %q on table %q to %q: %w", oldName, table, newName, err)
+	}
+
+	return
 }
 
 func (u upgrader) AddPrimaryKey(ctx context.Context, table string, ind *ddl.Index) (added bool, err error) {
 	return false, fmt.Errorf("adding primary keys on sqlite tables is not implemented")
 }
+
+// loads and returns all tables columns
+func (u upgrader) getColumns(ctx context.Context, table string) (out ddl.Columns, err error) {
+	type (
+		col struct {
+			CID          int            `db:"cid"`
+			Name         string         `db:"name"`
+			NotNull      bool           `db:"notnull"`
+			PrimaryKey   bool           `db:"pk"`
+			DefaultValue sql.NullString `db:"dflt_value"`
+			Type         string         `db:"type"`
+		}
+	)
+
+	var (
+		lookup = fmt.Sprintf(`PRAGMA TABLE_INFO(%q)`, table)
+		cols   []*col
+	)
+
+	if err = u.s.DB().SelectContext(ctx, &cols, lookup, u.s.Config().DBName, table); err != nil {
+		return nil, err
+	}
+
+	out = make([]*ddl.Column, len(cols))
+	for i := range cols {
+		out[i] = &ddl.Column{
+			Name: cols[i].Name,
+			//Type:         ddl.ColumnType{},
+			IsNull: !cols[i].NotNull,
+			//DefaultValue: "",
+		}
+	}
+
+	return out, nil
+}