From 8cbb2026e089165d49ec737891b45efb0a31f101 Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Wed, 29 Jun 2022 15:46:37 +0200 Subject: [PATCH] Fix and extend RDBMS upgrade procdures --- app/boot_levels.go | 9 +- store/adapters/rdbms/connect.go | 2 + .../rdbms/ddl/{generators.go => commands.go} | 119 ++++++++++++++---- store/adapters/rdbms/drivers/dialect.go | 4 + store/adapters/rdbms/drivers/mysql/connect.go | 20 ++- store/adapters/rdbms/drivers/mysql/dialect.go | 26 ++-- store/adapters/rdbms/drivers/mysql/schema.go | 65 +++++++--- .../rdbms/drivers/postgres/dialect.go | 26 ++-- .../adapters/rdbms/drivers/postgres/schema.go | 42 ++++++- .../adapters/rdbms/drivers/sqlite/connect.go | 2 +- .../adapters/rdbms/drivers/sqlite/dialect.go | 26 ++-- store/adapters/rdbms/drivers/sqlite/schema.go | 52 ++++++-- store/adapters/rdbms/log.go | 2 +- store/adapters/rdbms/rdbms_store.go | 5 +- store/adapters/rdbms/upgrade.go | 57 ++++----- store/adapters/rdbms/upgrade_fixes.go | 26 ++++ store/adapters/rdbms/upgrade_tables.go | 15 --- 17 files changed, 354 insertions(+), 144 deletions(-) rename store/adapters/rdbms/ddl/{generators.go => commands.go} (63%) create mode 100644 store/adapters/rdbms/upgrade_fixes.go diff --git a/app/boot_levels.go b/app/boot_levels.go index 4e55f3619..d8978cc87 100644 --- a/app/boot_levels.go +++ b/app/boot_levels.go @@ -202,20 +202,19 @@ func (app *CortezaApp) InitStore(ctx context.Context) (err error) { } } - app.Log.Info("running store update") - if !app.Opt.Upgrade.Always { app.Log.Info("store upgrade skipped (UPGRADE_ALWAYS=false)") } else { + log := app.Log.Named("store") + log.Info("running schema upgrade") ctx = actionlog.RequestOriginToContext(ctx, actionlog.RequestOrigin_APP_Upgrade) // If not explicitly set (UPGRADE_DEBUG=true) suppress logging in upgrader - log := zap.NewNop() if app.Opt.Upgrade.Debug { - log = app.Log.Named("store.upgrade") log.Info("store upgrade running in debug mode (UPGRADE_DEBUG=true)") } else { - app.Log.Info("store upgrade running (to enable upgrade debug logging set UPGRADE_DEBUG=true)") + log.Info("store upgrade running (to enable upgrade debug logging set UPGRADE_DEBUG=true)") + log = zap.NewNop() } if err = store.Upgrade(ctx, log, app.Store); err != nil { diff --git a/store/adapters/rdbms/connect.go b/store/adapters/rdbms/connect.go index dfe1403f0..69fe12527 100644 --- a/store/adapters/rdbms/connect.go +++ b/store/adapters/rdbms/connect.go @@ -34,6 +34,8 @@ func Connect(ctx context.Context, log *zap.Logger, cfg *ConnConfig) (db *sqlx.DB base *sql.DB ) + log = log.Named("store") + if base, err = sql.Open(cfg.DriverName, cfg.DataSourceName); err != nil { return } diff --git a/store/adapters/rdbms/ddl/generators.go b/store/adapters/rdbms/ddl/commands.go similarity index 63% rename from store/adapters/rdbms/ddl/generators.go rename to store/adapters/rdbms/ddl/commands.go index f5385d633..1263036cf 100644 --- a/store/adapters/rdbms/ddl/generators.go +++ b/store/adapters/rdbms/ddl/commands.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - "github.com/cortezaproject/corteza-server/store/adapters/rdbms/drivers" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" "github.com/jmoiron/sqlx" @@ -13,27 +12,43 @@ import ( type ( trColTypeFn func(ColumnType) string - CreateTableTemplate struct { - *Table - OmitIfNotExistsClause bool - SuffixClause string - TrColumnTypes trColTypeFn + dialect interface { + // GOQU returns goqu's dialect wrapper struct + GOQU() goqu.DialectWrapper + + NativeColumnType(columnType ColumnType) string } - CreateIndexTemplate struct { - *Index + CreateTable struct { + Dialect dialect + + Table *Table + + OmitIfNotExistsClause bool + SuffixClause string + } + + CreateIndex struct { + Dialect dialect + Index *Index OmitIfNotExistsClause bool OmitFieldLength bool } + + AddColumn struct { + Dialect dialect + Table *Table + Column *Column + } ) -func CreateIndexTemplates(base *CreateIndexTemplate, ii ...*Index) []any { +func CreateIndexTemplates(base *CreateIndex, ii ...*Index) []any { var ( tt = make([]any, len(ii)) ) for i := range ii { - tt[i] = &CreateIndexTemplate{ + tt[i] = &CreateIndex{ Index: ii[i], OmitIfNotExistsClause: base.OmitIfNotExistsClause, OmitFieldLength: base.OmitFieldLength, @@ -43,8 +58,12 @@ func CreateIndexTemplates(base *CreateIndexTemplate, ii ...*Index) []any { return tt } -// utility for executing series fo commands -func Exec(ctx context.Context, db sqlx.ExecerContext, ss ...any) (err error) { +// Exec is an utility for executing series of commands +// +// Parameters can be string, Stringer interface or goqu's exp.SQLExpression +// +// Any other type will result in panic +func Exec(ctx context.Context, db sqlx.ExtContext, ss ...any) (err error) { for _, s := range ss { var ( sql string @@ -58,6 +77,9 @@ func Exec(ctx context.Context, db sqlx.ExecerContext, ss ...any) (err error) { sql = c.String() case exp.SQLExpression: sql, args, err = c.ToSQL() + if err != nil { + return + } default: panic(fmt.Sprintf("unexecutable input (%T)", s)) } @@ -70,11 +92,15 @@ func Exec(ctx context.Context, db sqlx.ExecerContext, ss ...any) (err error) { return } -func TableExists(ctx context.Context, db sqlx.QueryerContext, d drivers.Dialect, table, schema string) (bool, error) { +func TableExists(ctx context.Context, db sqlx.QueryerContext, d dialect, table, schema string) (bool, error) { return GetBool(ctx, db, GenTableCheck(d, table, schema)) } -func GenTableCheck(d drivers.Dialect, table, schema string) *goqu.SelectDataset { +func ColumnExists(ctx context.Context, db sqlx.QueryerContext, d dialect, column, table, schema string) (bool, error) { + return GetBool(ctx, db, GenColumnCheck(d, column, table, schema)) +} + +func GenTableCheck(d dialect, table, schema string) *goqu.SelectDataset { return d.GOQU().Select(goqu.COUNT(goqu.Star()).Gt(0)). From("information_schema.tables"). Where( @@ -83,11 +109,47 @@ func GenTableCheck(d drivers.Dialect, table, schema string) *goqu.SelectDataset ) } -func IndexExists(ctx context.Context, db sqlx.QueryerContext, d drivers.Dialect, index, table, schema string) (bool, error) { +func GenColumnCheck(d dialect, column, table, schema string) *goqu.SelectDataset { + return d.GOQU().Select(goqu.COUNT(goqu.Star()).Gt(0)). + From("information_schema.columns"). + Where( + exp.ParseIdentifier("table_name").Eq(table), + exp.ParseIdentifier("table_schema").Eq(schema), + exp.ParseIdentifier("column_name").Eq(column), + ) +} + +// +//func GenAddColumn(ctx context.Context, dialect dialect, db sqlx.ExtContext, t *Table, cc ...*Column) (err error) { +// var ( +// aux []any +// exists bool +// ) +// +// for _, c := range cc { +// // check column existence +// if exists, err = ColumnExists(ctx, db, dialect, c.Name, t.Name, "public"); err != nil { +// return +// } else if exists { +// // column exists +// continue +// } +// +// aux = append(aux, &AddColumn{ +// Dialect: dialect, +// Table: t, +// Column: c, +// }) +// } +// +// return Exec(ctx, db, aux...) +//} + +func IndexExists(ctx context.Context, db sqlx.QueryerContext, d dialect, index, table, schema string) (bool, error) { return GetBool(ctx, db, GenIndexCheck(d, index, table, schema)) } -func GenIndexCheck(d drivers.Dialect, index, table, schema string) *goqu.SelectDataset { +func GenIndexCheck(d dialect, index, table, schema string) *goqu.SelectDataset { return d.GOQU().Select(goqu.COUNT(goqu.Star()).Gt(0)). From("information_schema.statistics"). Where( @@ -97,19 +159,15 @@ func GenIndexCheck(d drivers.Dialect, index, table, schema string) *goqu.SelectD ) } -func (t *CreateTableTemplate) String() string { - if t.TrColumnTypes == nil { - t.TrColumnTypes = ColumnTypeTranslator - } - +func (t *CreateTable) String() string { sql := "CREATE TABLE " if !t.OmitIfNotExistsClause { sql += "IF NOT EXISTS " } - sql += "\"" + t.Name + "\" (\n" - sql += GenCreateTableBody(t.Table, t.TrColumnTypes) + sql += "\"" + t.Table.Name + "\" (\n" + sql += GenCreateTableBody(t.Table, t.Dialect.NativeColumnType) sql += "\n)" sql += t.SuffixClause @@ -165,7 +223,7 @@ func GenPrimaryKey(pk *Index) string { return sql } -func (t *CreateIndexTemplate) String() string { +func (t *CreateIndex) String() string { sql := "CREATE " if t.Index.Unique { @@ -212,6 +270,19 @@ func (t *CreateIndexTemplate) String() string { return sql } +func (c *AddColumn) String() string { + sql := "ALTER TABLE" + " " + c.Table.Name + " ADD COLUMN " + c.Column.Name + " " + c.Dialect.NativeColumnType(c.Column.Type) + if !c.Column.IsNull { + sql += " NOT NULL" + } + + if len(c.Column.DefaultValue) > 0 { + sql += " DEFAULT " + c.Column.DefaultValue + } + + return sql +} + func GetBool(ctx context.Context, db sqlx.QueryerContext, query exp.SQLExpression) (bool, error) { var ( exists bool diff --git a/store/adapters/rdbms/drivers/dialect.go b/store/adapters/rdbms/drivers/dialect.go index 661dc2b03..eb73a78db 100644 --- a/store/adapters/rdbms/drivers/dialect.go +++ b/store/adapters/rdbms/drivers/dialect.go @@ -2,6 +2,7 @@ package drivers import ( "github.com/cortezaproject/corteza-server/pkg/dal" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" ) @@ -30,5 +31,8 @@ type ( // TypeWrap returns driver's type implementation for a particular attribute type TypeWrap(dal.Type) Type + + // NativeColumnType converts column type to type that can be used in the underlying rdbms + NativeColumnType(columnType ddl.ColumnType) string } ) diff --git a/store/adapters/rdbms/drivers/mysql/connect.go b/store/adapters/rdbms/drivers/mysql/connect.go index d30c78d6d..73ba64a66 100644 --- a/store/adapters/rdbms/drivers/mysql/connect.go +++ b/store/adapters/rdbms/drivers/mysql/connect.go @@ -47,7 +47,7 @@ func Connect(ctx context.Context, dsn string) (_ store.Storer, err error) { TxRetryErrHandler: txRetryErrHandler, ErrorHandler: errorHandler, - SchemaAPI: &schema{dbName: cfg.DBName, dialect: Dialect()}, + SchemaAPI: &schema{dbName: cfg.DBName}, } s.SetDefaults() @@ -133,13 +133,23 @@ func connSetup(ctx context.Context, db sqlx.ExecerContext) (err error) { } func txRetryErrHandler(try int, err error) bool { + var ( + mysqlErr *mysql.MySQLError + ok bool + ) + + // unwrap errors until we find one comming from MySQL for errors.Unwrap(err) != nil { err = errors.Unwrap(err) - } - var mysqlErr, ok = err.(*mysql.MySQLError) - if !ok || mysqlErr == nil { - return false + mysqlErr, ok = err.(*mysql.MySQLError) + if !ok || mysqlErr == nil { + return false + } + + if mysqlErr != nil { + break + } } switch mysqlErr.Number { diff --git a/store/adapters/rdbms/drivers/mysql/dialect.go b/store/adapters/rdbms/drivers/mysql/dialect.go index 5e62b3dcc..a050bf5fa 100644 --- a/store/adapters/rdbms/drivers/mysql/dialect.go +++ b/store/adapters/rdbms/drivers/mysql/dialect.go @@ -2,6 +2,7 @@ package mysql import ( "fmt" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" "strconv" "strings" @@ -12,32 +13,33 @@ import ( ) type ( - dialect struct{} + mysqlDialect struct{} ) var ( - goquDialectWrapper = goqu.Dialect("mysql") + _ drivers.Dialect = &mysqlDialect{} - _ drivers.Dialect = &dialect{} + dialect = &mysqlDialect{} + goquDialectWrapper = goqu.Dialect("mysql") ) -func Dialect() *dialect { - return &dialect{} +func Dialect() *mysqlDialect { + return dialect } -func (dialect) GOQU() goqu.DialectWrapper { +func (mysqlDialect) GOQU() goqu.DialectWrapper { return goquDialectWrapper } -func (dialect) DeepIdentJSON(ident exp.IdentifierExpression, pp ...any) (exp.LiteralExpression, error) { +func (mysqlDialect) DeepIdentJSON(ident exp.IdentifierExpression, pp ...any) (exp.LiteralExpression, error) { return JSONPath(ident, pp...) } -func (d dialect) TableCodec(m *dal.Model) drivers.TableCodec { +func (d mysqlDialect) TableCodec(m *dal.Model) drivers.TableCodec { return drivers.NewTableCodec(m, d) } -func (d dialect) TypeWrap(dt dal.Type) drivers.Type { +func (d mysqlDialect) TypeWrap(dt dal.Type) drivers.Type { // Any exception to general type-wrap implementation in the drivers package // should be placed here switch c := dt.(type) { @@ -59,7 +61,7 @@ func (d dialect) TypeWrap(dt dal.Type) drivers.Type { // AttributeCast for mySQL // // https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#function_cast -func (dialect) AttributeCast(attr *dal.Attribute, val exp.LiteralExpression) (exp.LiteralExpression, error) { +func (mysqlDialect) AttributeCast(attr *dal.Attribute, val exp.LiteralExpression) (exp.LiteralExpression, error) { var ( c exp.CastExpression ) @@ -96,6 +98,10 @@ func (dialect) AttributeCast(attr *dal.Attribute, val exp.LiteralExpression) (ex return exp.NewLiteralExpression("?", c), nil } +func (mysqlDialect) NativeColumnType(ct ddl.ColumnType) string { + return columnTypeTranslator(ct) +} + func JSONPath(ident exp.IdentifierExpression, pp ...any) (exp.LiteralExpression, error) { var ( sql strings.Builder diff --git a/store/adapters/rdbms/drivers/mysql/schema.go b/store/adapters/rdbms/drivers/mysql/schema.go index 74f7472b9..8586f0b0a 100644 --- a/store/adapters/rdbms/drivers/mysql/schema.go +++ b/store/adapters/rdbms/drivers/mysql/schema.go @@ -9,23 +9,28 @@ import ( type ( schema struct { - dbName string - dialect *dialect + dbName string } ) +// TableExists checks if table exists in the MySQL database func (s *schema) TableExists(ctx context.Context, db sqlx.QueryerContext, table string) (bool, error) { - return ddl.TableExists(ctx, db, s.dialect, table, "public") + return ddl.TableExists(ctx, db, Dialect(), table, s.dbName) +} + +// ColumnExists checks if column exists in the MySQL table +func (s *schema) ColumnExists(ctx context.Context, db sqlx.QueryerContext, column, table string) (bool, error) { + return ddl.ColumnExists(ctx, db, Dialect(), column, table, s.dbName) } // CreateTable // -// MySQL does not hav CREATE-INDEX-IF-NOT-EXISTS; we need to check index existance manually +// MySQL does not hav CREATE-INDEX-IF-NOT-EXISTS; we need to check index existence manually func (s *schema) CreateTable(ctx context.Context, db sqlx.ExtContext, t *ddl.Table) (err error) { - tc := &ddl.CreateTableTemplate{ - Table: t, - TrColumnTypes: columnTypTranslator, - SuffixClause: "ENGINE=InnoDB DEFAULT CHARSET=utf8", + tc := &ddl.CreateTable{ + Dialect: Dialect(), + Table: t, + SuffixClause: "ENGINE=InnoDB DEFAULT CHARSET=utf8", } if err = ddl.Exec(ctx, db, tc); err != nil { @@ -42,13 +47,14 @@ func (s *schema) CreateTable(ctx context.Context, db sqlx.ExtContext, t *ddl.Tab } var doesIt bool - if doesIt, err = ddl.IndexExists(ctx, db, s.dialect, index.Name, index.Table, s.dbName); err != nil { + if doesIt, err = ddl.IndexExists(ctx, db, Dialect(), index.Name, index.Table, s.dbName); err != nil { return } else if doesIt { continue } - ic := &ddl.CreateIndexTemplate{ + ic := &ddl.CreateIndex{ + Dialect: Dialect(), Index: index, OmitIfNotExistsClause: true, } @@ -62,17 +68,40 @@ func (s *schema) CreateTable(ctx context.Context, db sqlx.ExtContext, t *ddl.Tab return } -func columnTypTranslator(ct ddl.ColumnType) string { +func (s *schema) AddColumn(ctx context.Context, db sqlx.ExtContext, t *ddl.Table, cc ...*ddl.Column) (err error) { + var ( + aux []any + exists bool + ) + + for _, c := range cc { + // check column existence + if exists, err = s.ColumnExists(ctx, db, c.Name, t.Name); err != nil { + return + } else if exists { + // column exists + continue + } + + // Sadly, some column types in MySQL can not have default values + if c.Type.Type == ddl.ColumnTypeJson || c.Type.Type == ddl.ColumnTypeBinary || c.Type.Type == ddl.ColumnTypeText { + c.DefaultValue = "" + } + + aux = append(aux, &ddl.AddColumn{ + Dialect: dialect, + Table: t, + Column: c, + }) + } + + return ddl.Exec(ctx, db, aux...) +} + +func columnTypeTranslator(ct ddl.ColumnType) string { switch ct.Type { case ddl.ColumnTypeIdentifier: return "BIGINT UNSIGNED" - case ddl.ColumnTypeText: - // @todo when compose_record_value is removed, this will no longer be needed - if y, has := ct.Flags["mysqlLongText"].(bool); has && y { - return "LONGTEXT" - } - - return "TEXT" case ddl.ColumnTypeBinary: return "BLOB" case ddl.ColumnTypeTimestamp: diff --git a/store/adapters/rdbms/drivers/postgres/dialect.go b/store/adapters/rdbms/drivers/postgres/dialect.go index 54625fdc5..2e24ca9a7 100644 --- a/store/adapters/rdbms/drivers/postgres/dialect.go +++ b/store/adapters/rdbms/drivers/postgres/dialect.go @@ -2,38 +2,40 @@ package postgres import ( "github.com/cortezaproject/corteza-server/pkg/dal" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" "github.com/cortezaproject/corteza-server/store/adapters/rdbms/drivers" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" ) type ( - dialect struct{} + mysqlDialect struct{} ) var ( - goquDialectWrapper = goqu.Dialect("postgres") + _ drivers.Dialect = &mysqlDialect{} - _ drivers.Dialect = &dialect{} + dialect = &mysqlDialect{} + goquDialectWrapper = goqu.Dialect("postgres") ) -func Dialect() *dialect { - return &dialect{} +func Dialect() *mysqlDialect { + return dialect } -func (dialect) GOQU() goqu.DialectWrapper { +func (mysqlDialect) GOQU() goqu.DialectWrapper { return goquDialectWrapper } -func (dialect) DeepIdentJSON(ident exp.IdentifierExpression, pp ...any) (exp.LiteralExpression, error) { +func (mysqlDialect) DeepIdentJSON(ident exp.IdentifierExpression, pp ...any) (exp.LiteralExpression, error) { return drivers.DeepIdentJSON(ident, pp...), nil } -func (d dialect) TableCodec(m *dal.Model) drivers.TableCodec { +func (d mysqlDialect) TableCodec(m *dal.Model) drivers.TableCodec { return drivers.NewTableCodec(m, d) } -func (d dialect) TypeWrap(dt dal.Type) drivers.Type { +func (d mysqlDialect) TypeWrap(dt dal.Type) drivers.Type { // Any exception to general type-wrap implementation in the drivers package // should be placed here switch c := dt.(type) { @@ -44,7 +46,7 @@ func (d dialect) TypeWrap(dt dal.Type) drivers.Type { return drivers.TypeWrap(dt) } -func (dialect) AttributeCast(attr *dal.Attribute, val exp.LiteralExpression) (exp.LiteralExpression, error) { +func (mysqlDialect) AttributeCast(attr *dal.Attribute, val exp.LiteralExpression) (exp.LiteralExpression, error) { var ( c exp.CastExpression ) @@ -69,3 +71,7 @@ func (dialect) AttributeCast(attr *dal.Attribute, val exp.LiteralExpression) (ex return exp.NewLiteralExpression("?", c), nil } + +func (mysqlDialect) NativeColumnType(ct ddl.ColumnType) string { + return ddl.ColumnTypeTranslator(ct) +} diff --git a/store/adapters/rdbms/drivers/postgres/schema.go b/store/adapters/rdbms/drivers/postgres/schema.go index 324dfd5bb..57dad0d74 100644 --- a/store/adapters/rdbms/drivers/postgres/schema.go +++ b/store/adapters/rdbms/drivers/postgres/schema.go @@ -13,22 +13,56 @@ import ( type ( schema struct { schemaName string - dialect *dialect } ) +var ( + _ = &schema{} +) + func (s *schema) TableExists(ctx context.Context, db sqlx.QueryerContext, table string) (bool, error) { - return ddl.TableExists(ctx, db, s.dialect, table, "public") + return ddl.TableExists(ctx, db, Dialect(), table, "public") +} + +// ColumnExists checks if column exists in the MySQL table +func (s *schema) ColumnExists(ctx context.Context, db sqlx.QueryerContext, column, table string) (bool, error) { + return ddl.ColumnExists(ctx, db, Dialect(), column, table, "public") } func (s *schema) CreateTable(ctx context.Context, db sqlx.ExtContext, t *ddl.Table) (err error) { tt := append([]any{ - &ddl.CreateTableTemplate{ + &ddl.CreateTable{ + Dialect: Dialect(), Table: t, SuffixClause: " WITHOUT OIDS", }}, - ddl.CreateIndexTemplates(&ddl.CreateIndexTemplate{OmitFieldLength: true}, t.Indexes...)..., + ddl.CreateIndexTemplates(&ddl.CreateIndex{OmitFieldLength: true}, t.Indexes...)..., ) return ddl.Exec(ctx, db, tt...) } + +func (s *schema) AddColumn(ctx context.Context, db sqlx.ExtContext, t *ddl.Table, cc ...*ddl.Column) (err error) { + var ( + aux []any + exists bool + ) + + for _, c := range cc { + // check column existence + if exists, err = s.ColumnExists(ctx, db, c.Name, t.Name); err != nil { + return + } else if exists { + // column exists + continue + } + + aux = append(aux, &ddl.AddColumn{ + Dialect: dialect, + Table: t, + Column: c, + }) + } + + return ddl.Exec(ctx, db, aux...) +} diff --git a/store/adapters/rdbms/drivers/sqlite/connect.go b/store/adapters/rdbms/drivers/sqlite/connect.go index 29adc49e9..709e555bf 100644 --- a/store/adapters/rdbms/drivers/sqlite/connect.go +++ b/store/adapters/rdbms/drivers/sqlite/connect.go @@ -73,7 +73,7 @@ func Connect(ctx context.Context, dsn string) (_ store.Storer, err error) { TxRetryLimit: -1, //TxRetryErrHandler: txRetryErrHandler, - SchemaAPI: &schema{dialect: Dialect()}, + SchemaAPI: &schema{}, } s.SetDefaults() diff --git a/store/adapters/rdbms/drivers/sqlite/dialect.go b/store/adapters/rdbms/drivers/sqlite/dialect.go index 62cda83ee..c0f353444 100644 --- a/store/adapters/rdbms/drivers/sqlite/dialect.go +++ b/store/adapters/rdbms/drivers/sqlite/dialect.go @@ -2,6 +2,7 @@ package sqlite import ( "github.com/cortezaproject/corteza-server/pkg/dal" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" "github.com/cortezaproject/corteza-server/store/adapters/rdbms/drivers" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/dialect/sqlite3" @@ -9,13 +10,14 @@ import ( ) type ( - dialect struct{} + sqliteDialect struct{} ) var ( - goquDialectWrapper = goqu.Dialect("sqlite3") + _ drivers.Dialect = &sqliteDialect{} - _ drivers.Dialect = &dialect{} + dialect = &sqliteDialect{} + goquDialectWrapper = goqu.Dialect("sqlite3") ) func init() { @@ -28,28 +30,32 @@ func init() { goqu.RegisterDialect("sqlite3", d) } -func Dialect() *dialect { - return &dialect{} +func Dialect() *sqliteDialect { + return dialect } -func (dialect) GOQU() goqu.DialectWrapper { +func (sqliteDialect) GOQU() goqu.DialectWrapper { return goquDialectWrapper } -func (dialect) DeepIdentJSON(ident exp.IdentifierExpression, pp ...any) (exp.LiteralExpression, error) { +func (sqliteDialect) DeepIdentJSON(ident exp.IdentifierExpression, pp ...any) (exp.LiteralExpression, error) { return drivers.DeepIdentJSON(ident, pp...), nil } -func (d dialect) TableCodec(m *dal.Model) drivers.TableCodec { +func (d sqliteDialect) TableCodec(m *dal.Model) drivers.TableCodec { return drivers.NewTableCodec(m, d) } -func (d dialect) TypeWrap(t dal.Type) drivers.Type { +func (d sqliteDialect) TypeWrap(t dal.Type) drivers.Type { // Any exception to general type-wrap implementation in the drivers package // should be placed here return drivers.TypeWrap(t) } -func (dialect) AttributeCast(attr *dal.Attribute, val exp.LiteralExpression) (exp.LiteralExpression, error) { +func (sqliteDialect) AttributeCast(attr *dal.Attribute, val exp.LiteralExpression) (exp.LiteralExpression, error) { return drivers.AttributeCast(attr, val) } + +func (sqliteDialect) NativeColumnType(ct ddl.ColumnType) string { + return columnTypeTranslator(ct) +} diff --git a/store/adapters/rdbms/drivers/sqlite/schema.go b/store/adapters/rdbms/drivers/sqlite/schema.go index 2d00d7b90..b84172dbe 100644 --- a/store/adapters/rdbms/drivers/sqlite/schema.go +++ b/store/adapters/rdbms/drivers/sqlite/schema.go @@ -9,9 +9,7 @@ import ( ) type ( - schema struct { - dialect *dialect - } + schema struct{} ) func (s *schema) TableExists(ctx context.Context, db sqlx.QueryerContext, table string) (bool, error) { @@ -27,19 +25,57 @@ func (s *schema) TableExists(ctx context.Context, db sqlx.QueryerContext, table return exists, nil } +func (s *schema) ColumnExists(ctx context.Context, db sqlx.QueryerContext, column, table string) (bool, error) { + var ( + exists bool + sql = `SELECT COUNT(*) > 0 FROM pragma_table_info(?) WHERE name = ?;` + ) + + if err := sqlx.GetContext(ctx, db, &exists, sql, table, column); err != nil { + return false, fmt.Errorf("could not check if column exists: %w", err) + } + + return exists, nil +} + func (s *schema) CreateTable(ctx context.Context, db sqlx.ExtContext, t *ddl.Table) (err error) { tt := append( - []any{&ddl.CreateTableTemplate{ - Table: t, - TrColumnTypes: columnTypTranslator, + []any{&ddl.CreateTable{ + Dialect: Dialect(), + Table: t, }}, - ddl.CreateIndexTemplates(&ddl.CreateIndexTemplate{OmitFieldLength: true}, t.Indexes...)..., + ddl.CreateIndexTemplates(&ddl.CreateIndex{OmitFieldLength: true}, t.Indexes...)..., ) return ddl.Exec(ctx, db, tt...) } -func columnTypTranslator(ct ddl.ColumnType) string { +func (s *schema) AddColumn(ctx context.Context, db sqlx.ExtContext, t *ddl.Table, cc ...*ddl.Column) (err error) { + var ( + aux []any + exists bool + ) + + for _, c := range cc { + // check column existence + if exists, err = s.ColumnExists(ctx, db, c.Name, t.Name); err != nil { + return + } else if exists { + // column exists + continue + } + + aux = append(aux, &ddl.AddColumn{ + Dialect: dialect, + Table: t, + Column: c, + }) + } + + return ddl.Exec(ctx, db, aux...) +} + +func columnTypeTranslator(ct ddl.ColumnType) string { switch ct.Type { case ddl.ColumnTypeTimestamp: return "TIMESTAMP" diff --git a/store/adapters/rdbms/log.go b/store/adapters/rdbms/log.go index 334ca269e..efdaf639e 100644 --- a/store/adapters/rdbms/log.go +++ b/store/adapters/rdbms/log.go @@ -12,7 +12,7 @@ import ( // It checks the given context for logger before falling back to one set on the store func (s Store) log(ctx context.Context) *zap.Logger { return logger.ContextValue(ctx, s.Logger, logger.Default(), zap.NewNop()). - Named("store.rdbms"). + Named("store"). WithOptions(zap.AddCallerSkip(2)) } diff --git a/store/adapters/rdbms/rdbms_store.go b/store/adapters/rdbms/rdbms_store.go index 07503988e..3154bb4e1 100644 --- a/store/adapters/rdbms/rdbms_store.go +++ b/store/adapters/rdbms/rdbms_store.go @@ -34,8 +34,9 @@ type ( } schemaAPI interface { - TableExists(ctx context.Context, db sqlx.QueryerContext, table string) (bool, error) - CreateTable(ctx context.Context, db sqlx.ExtContext, t *ddl.Table) error + TableExists(context.Context, sqlx.QueryerContext, string) (bool, error) + CreateTable(context.Context, sqlx.ExtContext, *ddl.Table) error + AddColumn(context.Context, sqlx.ExtContext, *ddl.Table, ...*ddl.Column) error } Store struct { diff --git a/store/adapters/rdbms/upgrade.go b/store/adapters/rdbms/upgrade.go index 674cde928..d21b29240 100644 --- a/store/adapters/rdbms/upgrade.go +++ b/store/adapters/rdbms/upgrade.go @@ -3,44 +3,39 @@ package rdbms import ( "context" "fmt" + "go.uber.org/zap" ) func (s *Store) Upgrade(ctx context.Context) (err error) { - if err = UpgradeBeforeTableCreation(ctx, s); err != nil { - return - } + var ( + tableExists bool + ) - if err = UpgradeCreateTables(ctx, s); err != nil { - return - } - - if err = UpgradeAfterTableCreation(ctx, s); err != nil { - return - } - - return -} - -// UpgradeBeforeTableCreation all actions that need to happen before tables are created -// -// Important note! -// Corteza needs to be updated to the latest patch release under 2022.3.x before upgrading to 2022.9! -func UpgradeBeforeTableCreation(ctx context.Context, s *Store) (err error) { - // @todo figure out how we will detect if database is on the latest version - return -} - -// UpgradeCreateTables creates all tables needed by RDBMS store for Corteza to function properly -func UpgradeCreateTables(ctx context.Context, s *Store) (err error) { for _, t := range Tables() { - if err = s.SchemaAPI.CreateTable(ctx, s.DB, t); err != nil { - return fmt.Errorf("could not create table %s: %w", t.Name, err) + + tableExists, err = s.SchemaAPI.TableExists(ctx, s.DB, t.Name) + if err != nil { + return fmt.Errorf("could not check table %q existance: %w", t.Name, err) + } + + if !tableExists { + s.log(ctx).Debug("creating table", zap.String("table", t.Name)) + if err = s.SchemaAPI.CreateTable(ctx, s.DB, t); err != nil { + return fmt.Errorf("could not create table %q: %w", t.Name, err) + } + } + } + + fixes := []func(context.Context, *Store) error{ + fix202209_extendComposeModuleForPrivacyAndDAL, + fix202209_extendComposeModuleFieldsForPrivacyAndDAL, + } + + for _, fix := range fixes { + if err = fix(ctx, s); err != nil { + return } } return } - -func UpgradeAfterTableCreation(ctx context.Context, s *Store) (err error) { - return -} diff --git a/store/adapters/rdbms/upgrade_fixes.go b/store/adapters/rdbms/upgrade_fixes.go new file mode 100644 index 000000000..7470dbdea --- /dev/null +++ b/store/adapters/rdbms/upgrade_fixes.go @@ -0,0 +1,26 @@ +package rdbms + +import ( + "context" + . "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" +) + +func fix202209_extendComposeModuleForPrivacyAndDAL(ctx context.Context, s *Store) (err error) { + s.log(ctx).Info("extending compose_module table with privacy and model_config columns") + return s.SchemaAPI.AddColumn( + ctx, s.DB, + &Table{Name: "compose_module"}, + &Column{Type: ColumnType{Type: ColumnTypeJson}, DefaultValue: "'{}'", Name: "privacy"}, + &Column{Type: ColumnType{Type: ColumnTypeJson}, DefaultValue: "'{}'", Name: "model_config"}, + ) +} + +func fix202209_extendComposeModuleFieldsForPrivacyAndDAL(ctx context.Context, s *Store) (err error) { + s.log(ctx).Info("extending compose_module_field table with privacy and encoding_strategy columns") + return s.SchemaAPI.AddColumn( + ctx, s.DB, + &Table{Name: "compose_module_field"}, + &Column{Type: ColumnType{Type: ColumnTypeJson}, DefaultValue: "'{}'", Name: "privacy"}, + &Column{Type: ColumnType{Type: ColumnTypeJson}, DefaultValue: "'{}'", Name: "encoding_strategy"}, + ) +} diff --git a/store/adapters/rdbms/upgrade_tables.go b/store/adapters/rdbms/upgrade_tables.go index d24a58a42..efdc758f5 100644 --- a/store/adapters/rdbms/upgrade_tables.go +++ b/store/adapters/rdbms/upgrade_tables.go @@ -59,7 +59,6 @@ func Tables() []*Table { tableComposeNamespace(), tableComposePage(), tableComposeRecord(), - tableComposeRecordValue(), tableFederationModuleShared(), tableFederationModuleExposed(), tableFederationModuleMapping(), @@ -535,20 +534,6 @@ func tableComposeRecord() *Table { ) } -func tableComposeRecordValue() *Table { - return TableDef("compose_record_value", - ColumnDef("record_id", ColumnTypeIdentifier), - ColumnDef("name", ColumnTypeVarchar, ColumnTypeLength(64)), - ColumnDef("value", ColumnTypeText, ColumnTypeFlag("mysqlLongText", true)), - ColumnDef("ref", ColumnTypeIdentifier), - ColumnDef("place", ColumnTypeInteger), - ColumnDef("deleted_at", ColumnTypeTimestamp, Null), - - PrimaryKey(IColumn("record_id", "name", "place")), - AddIndex("ref", IColumn("ref"), IWhere("ref > 0")), - ) -} - func tableFederationModuleShared() *Table { return TableDef("federation_module_shared", ID,