diff --git a/app/boot_levels.go b/app/boot_levels.go index 7c2e6be9e..4c6ec1195 100644 --- a/app/boot_levels.go +++ b/app/boot_levels.go @@ -571,12 +571,12 @@ func (app *CortezaApp) initSystemEntities(ctx context.Context) (err error) { // Basic provision for system resources that we need before anything else if rr, err = provision.SystemRoles(ctx, app.Log, app.Store); err != nil { - return fmt.Errorf("could not provision system roles") + return fmt.Errorf("could not provision system roles: %w", err) } // Basic provision for system users that we need before anything else if uu, err = provision.SystemUsers(ctx, app.Log, app.Store); err != nil { - return fmt.Errorf("could not provision system users") + return fmt.Errorf("could not provision system users: %w", err) } // set system users & roles with so that the whole app knows what to use diff --git a/automation/model/models.gen.go b/automation/model/models.gen.go index 53138990e..807f2b7b9 100644 --- a/automation/model/models.gen.go +++ b/automation/model/models.gen.go @@ -32,7 +32,7 @@ var ( &dal.Attribute{ Ident: "Handle", - Type: &dal.TypeText{Length: 255}, + Type: &dal.TypeText{Length: 64}, Store: &dal.CodecAlias{Ident: "handle"}, }, @@ -155,6 +155,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_by"}, }, }, + + Indexes: dal.IndexSet{}, } Session = &dal.Model{ @@ -249,6 +251,8 @@ var ( Store: &dal.CodecAlias{Ident: "error"}, }, }, + + Indexes: dal.IndexSet{}, } Trigger = &dal.Model{ @@ -375,6 +379,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_by"}, }, }, + + Indexes: dal.IndexSet{}, } ) diff --git a/codegen/assets/templates/gocode/dal/$component_model.go.tpl b/codegen/assets/templates/gocode/dal/$component_model.go.tpl index 6a225d3b7..d03abfbfe 100644 --- a/codegen/assets/templates/gocode/dal/$component_model.go.tpl +++ b/codegen/assets/templates/gocode/dal/$component_model.go.tpl @@ -55,6 +55,31 @@ var ( }, {{ end -}} }, + + Indexes: dal.IndexSet{ + {{- range .indexes }} + &dal.Index{ + Ident: {{ printf "%q" .ident }}, + Type: {{ printf "%q" .type }}, + {{ if .unique }}Unique: true,{{ end }} + {{ if .predicate }}Predicate: {{ printf "%q" .predicate }},{{ end }} + Fields: []*dal.IndexField{ + {{- range .fields }} + { + AttributeIdent: {{ printf "%q" .attribute }}, + {{- if .modifiers }} + Modifiers: []dal.IndexFieldModifier{ {{- range .modifiers }}{{ printf "%q" . }},{{- end }} }, + {{- end }} + {{- if eq .sort "ASC" }}Sort: dal.IndexFieldSortAsc, {{ end }} + {{- if eq .sort "DESC" }}Sort: dal.IndexFieldSortDesc, {{ end }} + {{- if eq .nulls "FIRST" }}Nulls: dal.IndexFieldNullsFirst,{{ end }} + {{- if eq .nulls "LAST" }}Nulls: dal.IndexFieldNullsLast, {{ end }} + }, + {{ end -}} + }, + }, + {{ end -}} + }, } {{ end }} ) diff --git a/codegen/schema/model.cue b/codegen/schema/model.cue index 16a8de7b0..aed18cdfc 100644 --- a/codegen/schema/model.cue +++ b/codegen/schema/model.cue @@ -9,6 +9,10 @@ import ( attributes: { [name=_]: {"name": name} & #ModelAttribute } + + indexes?: { + [name=_]: {"name": name} & #ModelIndex + } } #ModelAttributeDalType: @@ -206,3 +210,49 @@ SortableTimestampNilField: { jsonTag: "json:\"\(json.field)\(_omitEmpty)\(_string)\"" } } + +#ModelIndex: { + name: #ident + + _attributes: { [_]: #ModelAttribute } + _words: strings.Replace(strings.Replace(name, "_", " ", -1), ".", " ", -1) + + _ident: strings.ToCamel(strings.Replace(strings.ToTitle(_words), " ", "", -1)) + + // lowercase (unexported, golang) identifier + ident: #ident | *_ident + + primary: bool | *(strings.ToLower(name) == "primary") + unique: bool | *(strings.Contains(name, "unique") || primary) + + type: "BTREE" | *"BTREE" + + attribute?: string + attributes?: [string, ...] + + if attribute != _|_ { + attributes: [attribute] + } + + fields: [#ModelIndexField, ...] | *([ + if attributes != _|_ { + for a in attributes { + "attribute": a + + #ModelIndexField + } + } + ]) + + predicate?: string +} + +#IndexFieldModifier: "LOWERCASE" + +#ModelIndexField: { + attribute: string + modifiers?: [#IndexFieldModifier, ...] + length?: number + sort?: "DESC" | "ASC" + nulls?: "FIRST" | "LAST" +} diff --git a/codegen/server.dal_models.cue b/codegen/server.dal_models.cue index 1ea392a80..bdf3b52ef 100644 --- a/codegen/server.dal_models.cue +++ b/codegen/server.dal_models.cue @@ -21,11 +21,12 @@ import ( cmpIdent: cmp.ident // Operation/resource validators, grouped by resource models: [ - for res in cmp.resources if res.model.attributes != _|_ { + for res in cmp.resources if (res.model.attributes != _|_) { var: "\(res.expIdent)" resType: "types.\(res.expIdent)ResourceType" - ident: res.model.ident + ident: res.model.ident + attributes: [ for attr in res.model.attributes { attr @@ -39,6 +40,39 @@ import ( } } ] + + if res.model.indexes != _|_ { + indexes: [ + for index in res.model.indexes { + if index.primary { + ident: "PRIMARY", + } + + if !index.primary { + ident: index.ident, + unique: index.unique, + } + + type: index.type, + + predicate?: index.predicate, + + fields: [ + for field in index.fields { + // craft a handy string that will yield a descriptive error + // when referencing an unexisting attribute + "model (\(res.model.ident)) index (\(index.ident)) field attribute reference (\(field.attribute)) validation": + res.model.attributes[field.attribute].ident + + "attribute": res.model.attributes[field.attribute].expIdent + "modifiers"?: field.modifiers + "sort"?: field.sort + "nulls"?: field.nulls + }, + ] + } + ] + } }, ] } diff --git a/compose/attachment.cue b/compose/attachment.cue index dd28f9517..32f5a13e8 100644 --- a/compose/attachment.cue +++ b/compose/attachment.cue @@ -35,6 +35,11 @@ attachment: { updated_at: schema.SortableTimestampNilField deleted_at: schema.SortableTimestampNilField } + + indexes: { + "primary": { attribute: "id" } + "namespace": { attribute: "namespace_id" }, + } } filter: { diff --git a/compose/chart.cue b/compose/chart.cue index 233d69698..be9ee9a65 100644 --- a/compose/chart.cue +++ b/compose/chart.cue @@ -16,7 +16,7 @@ chart: { handle: schema.HandleField name: {sortable: true} config: { goType: "types.ChartConfig" } - namespace_id: { + namespace_id: { ident: "namespaceID", goType: "uint64", storeIdent: "rel_namespace" @@ -26,6 +26,15 @@ chart: { updated_at: schema.SortableTimestampNilField deleted_at: schema.SortableTimestampNilField } + + indexes: { + "primary": { attribute: "id" } + "namespace": { attribute: "namespace_id" }, + "unique_handle": { + fields: [{ attribute: "handle", modifiers: ["LOWERCASE"] }] + predicate: "handle != '' AND deleted_at IS NULL" + } + } } filter: { diff --git a/compose/model/models.gen.go b/compose/model/models.gen.go index 787c2359d..25d400a3e 100644 --- a/compose/model/models.gen.go +++ b/compose/model/models.gen.go @@ -103,6 +103,30 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_at"}, }, }, + + Indexes: dal.IndexSet{ + &dal.Index{ + Ident: "PRIMARY", + Type: "BTREE", + + Fields: []*dal.IndexField{ + { + AttributeIdent: "ID", + }, + }, + }, + + &dal.Index{ + Ident: "namespace", + Type: "BTREE", + + Fields: []*dal.IndexField{ + { + AttributeIdent: "NamespaceID", + }, + }, + }, + }, } Chart = &dal.Model{ @@ -118,7 +142,7 @@ var ( &dal.Attribute{ Ident: "Handle", - Type: &dal.TypeText{Length: 255}, + Type: &dal.TypeText{Length: 64}, Store: &dal.CodecAlias{Ident: "handle"}, }, @@ -166,6 +190,42 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_at"}, }, }, + + Indexes: dal.IndexSet{ + &dal.Index{ + Ident: "PRIMARY", + Type: "BTREE", + + Fields: []*dal.IndexField{ + { + AttributeIdent: "ID", + }, + }, + }, + + &dal.Index{ + Ident: "namespace", + Type: "BTREE", + + Fields: []*dal.IndexField{ + { + AttributeIdent: "NamespaceID", + }, + }, + }, + + &dal.Index{ + Ident: "uniqueHandle", + Type: "BTREE", + Unique: true, + + Fields: []*dal.IndexField{ + { + AttributeIdent: "Handle", + }, + }, + }, + }, } Module = &dal.Model{ @@ -181,7 +241,7 @@ var ( &dal.Attribute{ Ident: "Handle", - Type: &dal.TypeText{Length: 255}, + Type: &dal.TypeText{Length: 64}, Store: &dal.CodecAlias{Ident: "handle"}, }, @@ -253,6 +313,46 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_at"}, }, }, + + Indexes: dal.IndexSet{ + &dal.Index{ + Ident: "PRIMARY", + Type: "BTREE", + + Fields: []*dal.IndexField{ + { + AttributeIdent: "ID", + }, + }, + }, + + &dal.Index{ + Ident: "namespace", + Type: "BTREE", + + Fields: []*dal.IndexField{ + { + AttributeIdent: "NamespaceID", + }, + }, + }, + + &dal.Index{ + Ident: "uniqueHandle", + Type: "BTREE", + Unique: true, + + Fields: []*dal.IndexField{ + { + AttributeIdent: "Handle", + }, + + { + AttributeIdent: "NamespaceID", + }, + }, + }, + }, } ModuleField = &dal.Model{ @@ -361,6 +461,46 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_at"}, }, }, + + Indexes: dal.IndexSet{ + &dal.Index{ + Ident: "PRIMARY", + Type: "BTREE", + + Fields: []*dal.IndexField{ + { + AttributeIdent: "ID", + }, + }, + }, + + &dal.Index{ + Ident: "module", + Type: "BTREE", + + Fields: []*dal.IndexField{ + { + AttributeIdent: "ModuleID", + }, + }, + }, + + &dal.Index{ + Ident: "uniqueName", + Type: "BTREE", + Unique: true, + + Fields: []*dal.IndexField{ + { + AttributeIdent: "Name", + }, + + { + AttributeIdent: "ModuleID", + }, + }, + }, + }, } Namespace = &dal.Model{ @@ -419,6 +559,31 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_at"}, }, }, + + Indexes: dal.IndexSet{ + &dal.Index{ + Ident: "PRIMARY", + Type: "BTREE", + + Fields: []*dal.IndexField{ + { + AttributeIdent: "ID", + }, + }, + }, + + &dal.Index{ + Ident: "uniqueHandle", + Type: "BTREE", + Unique: true, + + Fields: []*dal.IndexField{ + { + AttributeIdent: "Slug", + }, + }, + }, + }, } Page = &dal.Model{ @@ -467,7 +632,7 @@ var ( &dal.Attribute{ Ident: "Handle", - Type: &dal.TypeText{Length: 255}, + Type: &dal.TypeText{Length: 64}, Store: &dal.CodecAlias{Ident: "handle"}, }, @@ -534,6 +699,57 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_at"}, }, }, + + Indexes: dal.IndexSet{ + &dal.Index{ + Ident: "PRIMARY", + Type: "BTREE", + + Fields: []*dal.IndexField{ + { + AttributeIdent: "ID", + }, + }, + }, + + &dal.Index{ + Ident: "namespace", + Type: "BTREE", + + Fields: []*dal.IndexField{ + { + AttributeIdent: "NamespaceID", + }, + }, + }, + + &dal.Index{ + Ident: "module", + Type: "BTREE", + + Fields: []*dal.IndexField{ + { + AttributeIdent: "ModuleID", + }, + }, + }, + + &dal.Index{ + Ident: "uniqueHandle", + Type: "BTREE", + Unique: true, + + Fields: []*dal.IndexField{ + { + AttributeIdent: "Handle", + }, + + { + AttributeIdent: "NamespaceID", + }, + }, + }, + }, } Record = &dal.Model{ @@ -636,6 +852,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_by"}, }, }, + + Indexes: dal.IndexSet{}, } ) diff --git a/compose/module.cue b/compose/module.cue index 39fe31fba..f613a80ce 100644 --- a/compose/module.cue +++ b/compose/module.cue @@ -44,15 +44,14 @@ module: { deleted_at: schema.SortableTimestampNilField } -// indexes: { -// "primary": "id" -// "namespace": "namespace_id", -// "unique_handle": { -// unique: true -// attributes: ["handle", "namespace_id"] -// predicate: "handle <> '' AND deleted_at IS NULL" -// } -// } + indexes: { + "primary": { attribute: "id" } + "namespace": { attribute: "namespace_id" }, + "unique_handle": { + fields: [{ attribute: "handle", modifiers: ["LOWERCASE"] }, { attribute: "namespace_id" }] + predicate: "handle != '' AND deleted_at IS NULL" + } + } } filter: { diff --git a/compose/module_field.cue b/compose/module_field.cue index bb542dc1a..9fdd2007c 100644 --- a/compose/module_field.cue +++ b/compose/module_field.cue @@ -14,7 +14,7 @@ moduleField: { ident: "compose_module_field" attributes: { id: schema.IdField - module_id: { + module_id: { ident: "moduleID", goType: "uint64", storeIdent: "rel_module" @@ -39,7 +39,15 @@ moduleField: { created_at: { goType: "time.Time" } updated_at: { goType: "*time.Time" } deleted_at: { goType: "*time.Time" } + } + indexes: { + "primary": { attribute: "id" } + "module": { attribute: "module_id" }, + "unique_name": { + fields: [{ attribute: "name", modifiers: ["LOWERCASE"] }, { attribute: "module_id" }] + predicate: "name != '' AND deleted_at IS NULL" + } } } diff --git a/compose/namespace.cue b/compose/namespace.cue index 39e2b0430..c4b6a1699 100644 --- a/compose/namespace.cue +++ b/compose/namespace.cue @@ -18,6 +18,14 @@ namespace: { updated_at: schema.SortableTimestampNilField deleted_at: schema.SortableTimestampNilField } + + indexes: { + "primary": { attribute: "id" } + "unique_handle": { + fields: [{ attribute: "slug", modifiers: ["LOWERCASE"] }] + predicate: "handle != '' AND deleted_at IS NULL" + } + } } filter: { diff --git a/compose/page.cue b/compose/page.cue index e31c1cba4..c3c090917 100644 --- a/compose/page.cue +++ b/compose/page.cue @@ -45,6 +45,16 @@ page: { updated_at: schema.SortableTimestampNilField deleted_at: schema.SortableTimestampNilField } + + indexes: { + "primary": { attribute: "id" } + "namespace": { attribute: "namespace_id" }, + "module": { attribute: "module_id" }, + "unique_handle": { + fields: [{ attribute: "handle", modifiers: ["LOWERCASE"] }, { attribute: "namespace_id" }] + predicate: "handle != '' AND deleted_at IS NULL" + } + } } filter: { diff --git a/federation/model/models.gen.go b/federation/model/models.gen.go index 475d43897..b21d8c779 100644 --- a/federation/model/models.gen.go +++ b/federation/model/models.gen.go @@ -111,6 +111,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_by"}, }, }, + + Indexes: dal.IndexSet{}, } NodeSync = &dal.Model{ @@ -148,6 +150,8 @@ var ( Store: &dal.CodecAlias{Ident: "time_of_action"}, }, }, + + Indexes: dal.IndexSet{}, } ExposedModule = &dal.Model{ @@ -163,7 +167,7 @@ var ( &dal.Attribute{ Ident: "Handle", - Type: &dal.TypeText{Length: 255}, + Type: &dal.TypeText{Length: 64}, Store: &dal.CodecAlias{Ident: "handle"}, }, @@ -251,6 +255,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_by"}, }, }, + + Indexes: dal.IndexSet{}, } SharedModule = &dal.Model{ @@ -266,7 +272,7 @@ var ( &dal.Attribute{ Ident: "Handle", - Type: &dal.TypeText{Length: 255}, + Type: &dal.TypeText{Length: 64}, Store: &dal.CodecAlias{Ident: "handle"}, }, @@ -348,6 +354,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_by"}, }, }, + + Indexes: dal.IndexSet{}, } ModuleMapping = &dal.Model{ @@ -385,6 +393,8 @@ var ( Store: &dal.CodecAlias{Ident: "field_mapping"}, }, }, + + Indexes: dal.IndexSet{}, } ) diff --git a/pkg/dal/model.go b/pkg/dal/model.go index 4f2c2a7ee..239036cf6 100644 --- a/pkg/dal/model.go +++ b/pkg/dal/model.go @@ -81,12 +81,37 @@ type ( AttributeSet []*Attribute Index struct { - Ident string - Expressions []string - Predicate string + Ident string + Type string + Unique bool + + Fields []*IndexField + + Predicate string + } + + IndexField struct { + AttributeIdent string + Modifiers []IndexFieldModifier + Sort IndexFieldSort + Nulls IndexFieldNulls } IndexSet []*Index + + IndexFieldModifier string + IndexFieldSort int + IndexFieldNulls int +) + +const ( + IndexFieldSortDesc IndexFieldSort = -1 + IndexFieldSortAsc IndexFieldSort = 1 + + IndexFieldNullsLast IndexFieldNulls = -1 + IndexFieldNullsFirst IndexFieldNulls = 1 + + IndexFieldModifierLower = "LOWER" ) func PrimaryAttribute(ident string, codec Codec) *Attribute { diff --git a/store/adapters/rdbms/ddl/commands.go b/store/adapters/rdbms/ddl/commands.go index 458d8f133..67b6a291b 100644 --- a/store/adapters/rdbms/ddl/commands.go +++ b/store/adapters/rdbms/ddl/commands.go @@ -3,49 +3,53 @@ package ddl import ( "context" "fmt" - "github.com/doug-martin/goqu/v9" + "github.com/cortezaproject/corteza-server/pkg/dal" "github.com/doug-martin/goqu/v9/exp" "github.com/jmoiron/sqlx" ) type ( dialect interface { - // GOQU returns goqu's dialect wrapper struct - GOQU() goqu.DialectWrapper + QuoteIdent(i string) string } CreateTable struct { - Table *Table - + Dialect dialect + Table *Table OmitIfNotExistsClause bool SuffixClause string } CreateIndex struct { + Dialect dialect Index *Index OmitIfNotExistsClause bool OmitFieldLength bool } DropIndex struct { + Dialect dialect Ident exp.IdentifierExpression TableIdent exp.IdentifierExpression } AddColumn struct { - Table exp.IdentifierExpression - Column *Column + Dialect dialect + Table exp.IdentifierExpression + Column *Column } DropColumn struct { - Table exp.IdentifierExpression - Column exp.IdentifierExpression + Dialect dialect + Table exp.IdentifierExpression + Column exp.IdentifierExpression } RenameColumn struct { - Table exp.IdentifierExpression - Old exp.IdentifierExpression - New exp.IdentifierExpression + Dialect dialect + Table exp.IdentifierExpression + Old exp.IdentifierExpression + New exp.IdentifierExpression } ) @@ -106,15 +110,15 @@ func (t *CreateTable) String() string { sql += "IF NOT EXISTS " } - sql += "\"" + t.Table.Ident + "\" (\n" - sql += GenCreateTableBody(t.Table) - sql += "\n)" + sql += t.Dialect.QuoteIdent(t.Table.Ident) + " (\n" + sql += GenCreateTableBody(t.Dialect, t.Table) + sql += ")" sql += t.SuffixClause return sql } -func GenCreateTableBody(t *Table) string { +func GenCreateTableBody(d dialect, t *Table) string { sql := "" for c, col := range t.Columns { @@ -124,7 +128,7 @@ func GenCreateTableBody(t *Table) string { sql += ", " } - sql += GenTableColumn(col) + sql += GenTableColumn(d, col) sql += "\n" } @@ -135,15 +139,16 @@ func GenCreateTableBody(t *Table) string { continue } - sql += ", " + GenPrimaryKey(pk) + "\n" + sql += "\n" + sql += ", " + GenPrimaryKey(d, pk) + "\n" break } return sql } -func GenTableColumn(col *Column) string { - sql := `"` + col.Ident + `"` + col.Type.Name + ` ` +func GenTableColumn(d dialect, col *Column) string { + sql := d.QuoteIdent(col.Ident) + " " + col.Type.Name + " " if col.Type.Null { sql += " NULL" @@ -158,13 +163,14 @@ func GenTableColumn(col *Column) string { return sql } -func GenPrimaryKey(pk *Index) string { +func GenPrimaryKey(d dialect, pk *Index) string { sql := "PRIMARY KEY (" for f, field := range pk.Fields { if f > 0 { sql += ", " } - sql += field.Column + + sql += d.QuoteIdent(field.Column) } sql += ")" @@ -184,7 +190,7 @@ func (t *CreateIndex) String() string { sql += "IF NOT EXISTS " } - sql += "\"" + t.Index.Ident + "\" ON \"" + t.Index.TableIdent + "\" (" + sql += t.Dialect.QuoteIdent(t.Index.Ident) + " ON " + t.Dialect.QuoteIdent(t.Index.TableIdent) + " (" for f, field := range t.Index.Fields { isExpr := len(field.Expression) > 0 @@ -197,15 +203,7 @@ func (t *CreateIndex) String() string { sql += "(" sql += field.Expression } else { - sql += field.Column - - } - - switch field.Sorted { - case IndexFieldSortDesc: - sql += " DESC" - case IndexFieldSortAsc: - sql += " ASC" + sql += t.Dialect.QuoteIdent(field.Column) } if field.Length > 0 && !t.OmitFieldLength { @@ -215,6 +213,21 @@ func (t *CreateIndex) String() string { if isExpr { sql += ")" } + + switch field.Sort { + case dal.IndexFieldSortDesc: + sql += " DESC" + case dal.IndexFieldSortAsc: + sql += " ASC" + } + + switch field.Nulls { + case dal.IndexFieldNullsLast: + sql += " NULLS LAST" + case dal.IndexFieldNullsFirst: + sql += " NULLS FIRST" + } + } sql += ")" diff --git a/store/adapters/rdbms/ddl/data_definer.go b/store/adapters/rdbms/ddl/data_definer.go index b0c797906..03e78738a 100644 --- a/store/adapters/rdbms/ddl/data_definer.go +++ b/store/adapters/rdbms/ddl/data_definer.go @@ -9,8 +9,10 @@ import ( type ( driverDialect interface { - // Column converts column type to type that can be used in the underlying rdbms - AttributeToColumn(attr *dal.Attribute) (col *Column, err error) + QuoteIdent(string) string + // AttributeToColumn converts attribute to column + AttributeToColumn(*dal.Attribute) (*Column, error) + IndexFieldModifiers(*dal.Attribute, ...dal.IndexFieldModifier) (string, error) } // DataDefiner describes an interface for all DDL commands @@ -76,8 +78,6 @@ type ( Meta map[string]interface{} } - IndexFieldSorted int - // IndexField describes a single field (column or expression) of the SQL index IndexField struct { // Expression or a single column @@ -89,7 +89,8 @@ type ( Expression string // Ascending or descending - Sorted IndexFieldSorted + Sort dal.IndexFieldSort + Nulls dal.IndexFieldNulls Statistics *IndexFieldStatistics @@ -109,10 +110,6 @@ type ( const ( PRIMARY_KEY = "PRIMARY" - - IndexFieldSortDesc = -1 - IndexFieldUnsorted = 0 - IndexFieldSortAsc = 1 ) func (t *Table) ColumnByIdent(i string) *Column { @@ -129,6 +126,7 @@ func (t *Table) ColumnByIdent(i string) *Column { func ConvertModel(m *dal.Model, d driverDialect) (t *Table, err error) { var ( col *Column + idx *Index ) t = &Table{Ident: m.Ident} @@ -147,7 +145,53 @@ func ConvertModel(m *dal.Model, d driverDialect) (t *Table, err error) { t.Columns = append(t.Columns, col) } - // @todo indexes + for _, i := range m.Indexes { + if idx, err = ConvertIndex(i, m.Attributes, m.Ident, d); err != nil { + return nil, fmt.Errorf("could not convert index %q: %w", i.Ident, err) + } + + t.Indexes = append(t.Indexes, idx) + } + + return +} + +// ConvertIndex converts dal.Index to ddl.Index +func ConvertIndex(i *dal.Index, aa dal.AttributeSet, table string, d driverDialect) (idx *Index, err error) { + var ( + a *dal.Attribute + idxField *IndexField + ) + + idx = &Index{ + TableIdent: table, + Ident: i.Ident, + Type: i.Type, + Unique: i.Unique, + Predicate: i.Predicate, + } + + for _, f := range i.Fields { + // ensure attribute exists + if a = aa.FindByIdent(f.AttributeIdent); a == nil { + return nil, fmt.Errorf("referenced attribute %q does not exist", f.AttributeIdent) + } + + idxField = &IndexField{ + Sort: f.Sort, + Nulls: f.Nulls, + } + + if len(f.Modifiers) > 0 { + if idxField.Expression, err = d.IndexFieldModifiers(a, f.Modifiers...); err != nil { + return nil, fmt.Errorf("could not convert index field modifiers: %w", err) + } + } else { + idxField.Column = a.StoreIdent() + } + + idx.Fields = append(idx.Fields, idxField) + } return } @@ -178,7 +222,7 @@ func DefaultNumber(set bool, precision uint, value float64) string { case precision > 0: return fmt.Sprintf("%f", value) default: - return fmt.Sprintf("%d", value) + return fmt.Sprintf("%0.0f", value) } } diff --git a/store/adapters/rdbms/ddl/data_definer_test.go b/store/adapters/rdbms/ddl/data_definer_test.go index e44e6901e..1117259e0 100644 --- a/store/adapters/rdbms/ddl/data_definer_test.go +++ b/store/adapters/rdbms/ddl/data_definer_test.go @@ -11,27 +11,42 @@ type ( mockDriver struct{} ) -func (mockDriver) Column(ct dal.Type) (*ColumnType, error) { - col := &ColumnType{ - Name: string(ct.Type()), - Null: ct.IsNullable(), +func (mockDriver) QuoteIdent(i string) string { return `"` + i + `"` } + +func (mockDriver) AttributeToColumn(attr *dal.Attribute) (col *Column, err error) { + col = &Column{ + Ident: attr.StoreIdent(), + Comment: attr.Label, + Type: &ColumnType{ + Null: attr.Type.IsNullable(), + }, } - switch ct.(type) { + switch t := attr.Type.(type) { case *dal.TypeID: - col.Name = "INT" + _ = t + col.Type.Name = "INT" + case *dal.TypeText: + _ = t + col.Type.Name = "TEXT" } return col, nil } +func (d mockDriver) IndexFieldModifiers(attr *dal.Attribute, mm ...dal.IndexFieldModifier) (string, error) { + return IndexFieldModifiers(attr, d.QuoteIdent, mm...) +} + func TestModelToTable(t *testing.T) { tests := []struct { name string m *dal.Model d driverDialect - sql string + + createTableSQL string + createIndexSQL string }{ { name: "simple", @@ -45,14 +60,60 @@ func TestModelToTable(t *testing.T) { Nullable: true, }, }, + &dal.Attribute{ + Ident: "some_txt", + Type: &dal.TypeText{}, + }, + }, + + Indexes: []*dal.Index{ + { + Ident: PRIMARY_KEY, + Fields: []*dal.IndexField{ + { + AttributeIdent: "null_ID", + }, + }, + }, + { + Ident: "first_idx", + Fields: []*dal.IndexField{ + { + AttributeIdent: "null_ID", + Modifiers: []dal.IndexFieldModifier{dal.IndexFieldModifierLower}, + Sort: dal.IndexFieldSortDesc, + }, + }, + }, + { + Ident: "second_idx", + Fields: []*dal.IndexField{ + { + AttributeIdent: "null_ID", + }, + { + AttributeIdent: "some_txt", + Modifiers: []dal.IndexFieldModifier{dal.IndexFieldModifierLower}, + Sort: dal.IndexFieldSortDesc, + Nulls: dal.IndexFieldNullsLast, + }, + }, + }, }, }, - sql: ` + createTableSQL: ` CREATE TABLE IF NOT EXISTS "simple" ( "null_ID" INT NULL +, "some_txt" TEXT NOT NULL +, PRIMARY KEY ("null_ID") )`, + + createIndexSQL: ` +CREATE INDEX IF NOT EXISTS "first_idx" ON "simple" ((LOWER("null_ID")) DESC) +CREATE INDEX IF NOT EXISTS "second_idx" ON "simple" ("null_ID", (LOWER("some_txt")) DESC NULLS LAST) +`, }, } for _, tt := range tests { @@ -62,7 +123,17 @@ CREATE TABLE IF NOT EXISTS "simple" ( tbl, err := ConvertModel(tt.m, tt.d) req.NoError(err) - req.Equal(strings.TrimSpace(tt.sql), strings.TrimSpace((&CreateTable{Table: tbl}).String())) + req.Equal( + strings.TrimSpace(tt.createTableSQL), + strings.TrimSpace((&CreateTable{Table: tbl, Dialect: tt.d}).String()), + ) + + idxSQL := "\n" + for i := 1; i < len(tbl.Indexes); i++ { + idxSQL += (&CreateIndex{Index: tbl.Indexes[i], Dialect: tt.d}).String() + "\n" + } + + req.Equal(tt.createIndexSQL, idxSQL) }) } } diff --git a/store/adapters/rdbms/drivers/dialect.go b/store/adapters/rdbms/drivers/dialect.go index 04dda7fc4..62833cf02 100644 --- a/store/adapters/rdbms/drivers/dialect.go +++ b/store/adapters/rdbms/drivers/dialect.go @@ -33,7 +33,9 @@ type ( // TypeWrap returns driver's type implementation for a particular attribute type TypeWrap(dal.Type) Type - // Column converts column type to type that can be used in the underlying rdbms + QuoteIdent(string) string + + // AttributeToColumn converts attribute to column defunition AttributeToColumn(*dal.Attribute) (*ddl.Column, error) // ExprHandler returns driver specific expression handling diff --git a/store/adapters/rdbms/drivers/json.go b/store/adapters/rdbms/drivers/json.go index 11e422fef..c344d0032 100644 --- a/store/adapters/rdbms/drivers/json.go +++ b/store/adapters/rdbms/drivers/json.go @@ -6,6 +6,7 @@ package drivers import ( "fmt" + "github.com/cortezaproject/corteza-server/pkg/dal" "strconv" "strings" @@ -85,3 +86,15 @@ func JsonPath(pp ...any) string { return path.String() } + +func IndexFieldModifiers(attr *dal.Attribute, quoteIdent func(i string) string, mm ...dal.IndexFieldModifier) (string, error) { + var ( + out = quoteIdent(attr.StoreIdent()) + ) + + for _, m := range mm { + out = fmt.Sprintf("%s(%s)", m, out) + } + + return out, nil +} diff --git a/store/adapters/rdbms/drivers/mysql/dialect.go b/store/adapters/rdbms/drivers/mysql/dialect.go index 795eaea9c..86dee249e 100644 --- a/store/adapters/rdbms/drivers/mysql/dialect.go +++ b/store/adapters/rdbms/drivers/mysql/dialect.go @@ -2,15 +2,15 @@ package mysql import ( "fmt" + "github.com/cortezaproject/corteza-server/pkg/dal" "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/drivers" "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ql" + "github.com/doug-martin/goqu/v9" + "github.com/doug-martin/goqu/v9/dialect/mysql" + "github.com/doug-martin/goqu/v9/exp" "strconv" "strings" - - "github.com/cortezaproject/corteza-server/pkg/dal" - "github.com/cortezaproject/corteza-server/store/adapters/rdbms/drivers" - "github.com/doug-martin/goqu/v9" - "github.com/doug-martin/goqu/v9/exp" ) type ( @@ -22,14 +22,18 @@ var ( dialect = &mysqlDialect{} goquDialectWrapper = goqu.Dialect("mysql") + quoteIdent = string(mysql.DialectOptions().QuoteRune) ) func Dialect() *mysqlDialect { return dialect } -func (mysqlDialect) GOQU() goqu.DialectWrapper { - return goquDialectWrapper +func (mysqlDialect) GOQU() goqu.DialectWrapper { return goquDialectWrapper } +func (mysqlDialect) QuoteIdent(i string) string { return quoteIdent + i + quoteIdent } + +func (d mysqlDialect) IndexFieldModifiers(attr *dal.Attribute, mm ...dal.IndexFieldModifier) (string, error) { + return drivers.IndexFieldModifiers(attr, d.QuoteIdent, mm...) } func (mysqlDialect) DeepIdentJSON(ident exp.IdentifierExpression, pp ...any) (exp.LiteralExpression, error) { diff --git a/store/adapters/rdbms/drivers/mysql/information_schema.go b/store/adapters/rdbms/drivers/mysql/information_schema.go index 5a3c9c6dd..e20cfedbb 100644 --- a/store/adapters/rdbms/drivers/mysql/information_schema.go +++ b/store/adapters/rdbms/drivers/mysql/information_schema.go @@ -3,6 +3,7 @@ package mysql import ( "context" "database/sql" + "github.com/cortezaproject/corteza-server/pkg/dal" "github.com/cortezaproject/corteza-server/pkg/errors" "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" "github.com/doug-martin/goqu/v9" @@ -188,9 +189,9 @@ func (i *informationSchema) scanIndexes(ctx context.Context, sd *goqu.SelectData switch a.ColumnCollation.String { case "A": - col.Sorted = ddl.IndexFieldSortAsc + col.Sort = dal.IndexFieldSortAsc case "D": - col.Sorted = ddl.IndexFieldSortDesc + col.Sort = dal.IndexFieldSortDesc } if a.Expression.Valid { diff --git a/store/adapters/rdbms/drivers/postgres/data_definer.go b/store/adapters/rdbms/drivers/postgres/data_definer.go index 0b8c374a5..20a00d2ed 100644 --- a/store/adapters/rdbms/drivers/postgres/data_definer.go +++ b/store/adapters/rdbms/drivers/postgres/data_definer.go @@ -36,7 +36,7 @@ func (dd *dataDefiner) ConvertModel(m *dal.Model) (*ddl.Table, error) { } func (dd *dataDefiner) TableCreate(ctx context.Context, t *ddl.Table) error { - return ddl.Exec(ctx, dd.conn, &ddl.CreateTable{Table: t}) + return ddl.Exec(ctx, dd.conn, &ddl.CreateTable{Table: t, Dialect: dd.d}) } func (dd *dataDefiner) TableLookup(ctx context.Context, t string) (*ddl.Table, error) { @@ -52,16 +52,18 @@ func (dd *dataDefiner) ColumnAdd(ctx context.Context, t string, c *ddl.Column) e func (dd *dataDefiner) ColumnDrop(ctx context.Context, t, col string) error { return ddl.Exec(ctx, dd.conn, &ddl.DropColumn{ - Table: exp.NewIdentifierExpression("", t, ""), - Column: exp.NewIdentifierExpression("", "", col), + Dialect: dd.d, + Table: exp.NewIdentifierExpression("", t, ""), + Column: exp.NewIdentifierExpression("", "", col), }) } func (dd *dataDefiner) ColumnRename(ctx context.Context, t string, o string, n string) error { return ddl.Exec(ctx, dd.conn, &ddl.RenameColumn{ - Table: exp.NewIdentifierExpression("", t, ""), - Old: exp.NewIdentifierExpression("", "", o), - New: exp.NewIdentifierExpression("", "", n), + Dialect: dd.d, + Table: exp.NewIdentifierExpression("", t, ""), + Old: exp.NewIdentifierExpression("", "", o), + New: exp.NewIdentifierExpression("", "", n), }) } @@ -74,11 +76,17 @@ func (dd *dataDefiner) IndexLookup(ctx context.Context, i, t string) (*ddl.Index } func (dd *dataDefiner) IndexCreate(ctx context.Context, t string, i *ddl.Index) error { - return ddl.Exec(ctx, dd.conn, &ddl.CreateIndex{Index: i}) + return ddl.Exec(ctx, dd.conn, &ddl.CreateIndex{ + Index: i, + Dialect: dd.d, + }) } func (dd *dataDefiner) IndexDrop(ctx context.Context, t, i string) error { - return ddl.Exec(ctx, dd.conn, &ddl.DropIndex{Ident: exp.NewIdentifierExpression("", t, i)}) + return ddl.Exec(ctx, dd.conn, &ddl.DropIndex{ + Ident: exp.NewIdentifierExpression("", t, i), + Dialect: dd.d, + }) } // diff --git a/store/adapters/rdbms/drivers/postgres/dialect.go b/store/adapters/rdbms/drivers/postgres/dialect.go index 8ee6afa3f..44c381331 100644 --- a/store/adapters/rdbms/drivers/postgres/dialect.go +++ b/store/adapters/rdbms/drivers/postgres/dialect.go @@ -7,6 +7,7 @@ import ( "github.com/cortezaproject/corteza-server/store/adapters/rdbms/drivers" "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ql" "github.com/doug-martin/goqu/v9" + "github.com/doug-martin/goqu/v9/dialect/postgres" "github.com/doug-martin/goqu/v9/exp" ) @@ -19,14 +20,18 @@ var ( dialect = &postgresDialect{} goquDialectWrapper = goqu.Dialect("postgres") + quoteIdent = string(postgres.DialectOptions().QuoteRune) ) func Dialect() *postgresDialect { return dialect } -func (postgresDialect) GOQU() goqu.DialectWrapper { - return goquDialectWrapper +func (postgresDialect) GOQU() goqu.DialectWrapper { return goquDialectWrapper } +func (postgresDialect) QuoteIdent(i string) string { return quoteIdent + i + quoteIdent } + +func (d postgresDialect) IndexFieldModifiers(attr *dal.Attribute, mm ...dal.IndexFieldModifier) (string, error) { + return drivers.IndexFieldModifiers(attr, d.QuoteIdent, mm...) } func (postgresDialect) DeepIdentJSON(ident exp.IdentifierExpression, pp ...any) (exp.LiteralExpression, error) { diff --git a/store/adapters/rdbms/drivers/sqlite/dialect.go b/store/adapters/rdbms/drivers/sqlite/dialect.go index dc2550f66..6abcf80d2 100644 --- a/store/adapters/rdbms/drivers/sqlite/dialect.go +++ b/store/adapters/rdbms/drivers/sqlite/dialect.go @@ -20,6 +20,7 @@ var ( dialect = &sqliteDialect{} goquDialectWrapper = goqu.Dialect("sqlite3") + quoteIdent = string(sqlite3.DialectOptions().QuoteRune) ) func init() { @@ -36,8 +37,11 @@ func Dialect() *sqliteDialect { return dialect } -func (sqliteDialect) GOQU() goqu.DialectWrapper { - return goquDialectWrapper +func (sqliteDialect) GOQU() goqu.DialectWrapper { return goquDialectWrapper } +func (sqliteDialect) QuoteIdent(i string) string { return quoteIdent + i + quoteIdent } + +func (d sqliteDialect) IndexFieldModifiers(attr *dal.Attribute, mm ...dal.IndexFieldModifier) (string, error) { + return drivers.IndexFieldModifiers(attr, d.QuoteIdent, mm...) } func (sqliteDialect) DeepIdentJSON(ident exp.IdentifierExpression, pp ...any) (exp.LiteralExpression, error) { diff --git a/store/adapters/rdbms/drivers/sqlite/information_schema.go b/store/adapters/rdbms/drivers/sqlite/information_schema.go index 5d5b9c99d..5e0f97504 100644 --- a/store/adapters/rdbms/drivers/sqlite/information_schema.go +++ b/store/adapters/rdbms/drivers/sqlite/information_schema.go @@ -3,6 +3,7 @@ package sqlite import ( "context" "database/sql" + "github.com/cortezaproject/corteza-server/pkg/dal" "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" @@ -117,9 +118,9 @@ func (i *informationSchema) scanIndexes(ctx context.Context, sd *goqu.SelectData switch a.ColumnCollation.String { case "A": - col.Sorted = ddl.IndexFieldSortAsc + col.Sort = dal.IndexFieldSortAsc case "D": - col.Sorted = ddl.IndexFieldSortDesc + col.Sort = dal.IndexFieldSortDesc } if a.Expression.Valid { diff --git a/store/adapters/rdbms/upgrade.go b/store/adapters/rdbms/upgrade.go index 2824c4721..a2c22d4ab 100644 --- a/store/adapters/rdbms/upgrade.go +++ b/store/adapters/rdbms/upgrade.go @@ -81,6 +81,12 @@ func createTablesFromModels(ctx context.Context, log *zap.Logger, dd ddl.DataDef if err = dd.TableCreate(ctx, tbl); err != nil { return fmt.Errorf("can not create table from model %q: %w", m.Ident, err) } + + for _, idx := range tbl.Indexes { + if err = dd.IndexCreate(ctx, tbl.Ident, idx); err != nil { + return fmt.Errorf("can not create index %q on table %q: %w", idx.Ident, tbl.Ident, err) + } + } } } diff --git a/system/model/models.gen.go b/system/model/models.gen.go index a4eeec0ca..d1cdd7d9b 100644 --- a/system/model/models.gen.go +++ b/system/model/models.gen.go @@ -87,6 +87,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_at"}, }, }, + + Indexes: dal.IndexSet{}, } Application = &dal.Model{ @@ -151,6 +153,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_at"}, }, }, + + Indexes: dal.IndexSet{}, } ApigwRoute = &dal.Model{ @@ -248,6 +252,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_by"}, }, }, + + Indexes: dal.IndexSet{}, } ApigwFilter = &dal.Model{ @@ -351,6 +357,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_by"}, }, }, + + Indexes: dal.IndexSet{}, } AuthClient = &dal.Model{ @@ -366,7 +374,7 @@ var ( &dal.Attribute{ Ident: "Handle", - Type: &dal.TypeText{Length: 255}, + Type: &dal.TypeText{Length: 64}, Store: &dal.CodecAlias{Ident: "handle"}, }, @@ -495,6 +503,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_by"}, }, }, + + Indexes: dal.IndexSet{}, } AuthConfirmedClient = &dal.Model{ @@ -520,6 +530,8 @@ var ( Store: &dal.CodecAlias{Ident: "confirmed_at"}, }, }, + + Indexes: dal.IndexSet{}, } AuthSession = &dal.Model{ @@ -572,6 +584,8 @@ var ( Store: &dal.CodecAlias{Ident: "user_agent"}, }, }, + + Indexes: dal.IndexSet{}, } AuthOa2token = &dal.Model{ @@ -648,6 +662,8 @@ var ( Store: &dal.CodecAlias{Ident: "user_agent"}, }, }, + + Indexes: dal.IndexSet{}, } Credential = &dal.Model{ @@ -724,6 +740,8 @@ var ( Store: &dal.CodecAlias{Ident: "expires_at"}, }, }, + + Indexes: dal.IndexSet{}, } DataPrivacyRequest = &dal.Model{ @@ -833,6 +851,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_by"}, }, }, + + Indexes: dal.IndexSet{}, } DataPrivacyRequestComment = &dal.Model{ @@ -912,6 +932,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_by"}, }, }, + + Indexes: dal.IndexSet{}, } Queue = &dal.Model{ @@ -997,6 +1019,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_by"}, }, }, + + Indexes: dal.IndexSet{}, } QueueMessage = &dal.Model{ @@ -1034,6 +1058,8 @@ var ( Store: &dal.CodecAlias{Ident: "created"}, }, }, + + Indexes: dal.IndexSet{}, } Reminder = &dal.Model{ @@ -1122,6 +1148,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_at"}, }, }, + + Indexes: dal.IndexSet{}, } Report = &dal.Model{ @@ -1137,7 +1165,7 @@ var ( &dal.Attribute{ Ident: "Handle", - Type: &dal.TypeText{Length: 255}, + Type: &dal.TypeText{Length: 64}, Store: &dal.CodecAlias{Ident: "handle"}, }, @@ -1230,6 +1258,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_by"}, }, }, + + Indexes: dal.IndexSet{}, } ResourceTranslation = &dal.Model{ @@ -1332,6 +1362,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_by"}, }, }, + + Indexes: dal.IndexSet{}, } Role = &dal.Model{ @@ -1353,7 +1385,7 @@ var ( &dal.Attribute{ Ident: "Handle", - Type: &dal.TypeText{Length: 255}, + Type: &dal.TypeText{Length: 64}, Store: &dal.CodecAlias{Ident: "handle"}, }, @@ -1390,6 +1422,8 @@ var ( Store: &dal.CodecAlias{Ident: "archived_at"}, }, }, + + Indexes: dal.IndexSet{}, } RoleMember = &dal.Model{ @@ -1409,6 +1443,8 @@ var ( Store: &dal.CodecAlias{Ident: "rel_role"}, }, }, + + Indexes: dal.IndexSet{}, } SettingValue = &dal.Model{ @@ -1446,6 +1482,8 @@ var ( Store: &dal.CodecAlias{Ident: "updated_at"}, }, }, + + Indexes: dal.IndexSet{}, } Template = &dal.Model{ @@ -1461,7 +1499,7 @@ var ( &dal.Attribute{ Ident: "Handle", - Type: &dal.TypeText{Length: 255}, + Type: &dal.TypeText{Length: 64}, Store: &dal.CodecAlias{Ident: "handle"}, }, @@ -1528,6 +1566,8 @@ var ( Store: &dal.CodecAlias{Ident: "last_used_at"}, }, }, + + Indexes: dal.IndexSet{}, } User = &dal.Model{ @@ -1543,7 +1583,7 @@ var ( &dal.Attribute{ Ident: "Handle", - Type: &dal.TypeText{Length: 255}, + Type: &dal.TypeText{Length: 64}, Store: &dal.CodecAlias{Ident: "handle"}, }, @@ -1610,6 +1650,8 @@ var ( Store: &dal.CodecAlias{Ident: "suspended_at"}, }, }, + + Indexes: dal.IndexSet{}, } DalConnection = &dal.Model{ @@ -1625,7 +1667,7 @@ var ( &dal.Attribute{ Ident: "Handle", - Type: &dal.TypeText{Length: 255}, + Type: &dal.TypeText{Length: 64}, Store: &dal.CodecAlias{Ident: "handle"}, }, @@ -1701,6 +1743,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_by"}, }, }, + + Indexes: dal.IndexSet{}, } DalSensitivityLevel = &dal.Model{ @@ -1716,7 +1760,7 @@ var ( &dal.Attribute{ Ident: "Handle", - Type: &dal.TypeText{Length: 255}, + Type: &dal.TypeText{Length: 64}, Store: &dal.CodecAlias{Ident: "handle"}, }, @@ -1786,6 +1830,8 @@ var ( Store: &dal.CodecAlias{Ident: "deleted_by"}, }, }, + + Indexes: dal.IndexSet{}, } )