diff --git a/compose/dalutils/capabilities.go b/compose/dalutils/capabilities.go deleted file mode 100644 index 91a498c22..000000000 --- a/compose/dalutils/capabilities.go +++ /dev/null @@ -1,47 +0,0 @@ -package dalutils - -import ( - "github.com/cortezaproject/corteza-server/compose/types" - "github.com/cortezaproject/corteza-server/pkg/dal/capabilities" -) - -func recCreateCapabilities(m *types.Module) (out capabilities.Set) { - return capabilities.CreateCapabilities(m.ModelConfig.Capabilities...) -} - -func recUpdateCapabilities(m *types.Module) (out capabilities.Set) { - return capabilities.UpdateCapabilities(m.ModelConfig.Capabilities...) -} - -func recDeleteCapabilities(m *types.Module) (out capabilities.Set) { - return capabilities.DeleteCapabilities(m.ModelConfig.Capabilities...) -} - -func recFilterCapabilities(f types.RecordFilter) (out capabilities.Set) { - if f.PageCursor != nil { - out = append(out, capabilities.Paging) - } - - if f.IncPageNavigation { - out = append(out, capabilities.Paging) - } - - if f.IncTotal { - out = append(out, capabilities.Stats) - } - - if f.Sort != nil { - out = append(out, capabilities.Sorting) - } - - return -} - -func recSearchCapabilities(m *types.Module, f types.RecordFilter) (out capabilities.Set) { - return capabilities.SearchCapabilities(m.ModelConfig.Capabilities...). - Union(recFilterCapabilities(f)) -} - -func recLookupCapabilities(m *types.Module) (out capabilities.Set) { - return capabilities.LookupCapabilities(m.ModelConfig.Capabilities...) -} diff --git a/compose/dalutils/modules.go b/compose/dalutils/modules.go deleted file mode 100644 index 4f2133879..000000000 --- a/compose/dalutils/modules.go +++ /dev/null @@ -1,427 +0,0 @@ -package dalutils - -import ( - "context" - "fmt" - "strconv" - "strings" - - "github.com/cortezaproject/corteza-server/compose/types" - "github.com/cortezaproject/corteza-server/pkg/dal" - "github.com/cortezaproject/corteza-server/pkg/handle" - "github.com/cortezaproject/corteza-server/store" - systemTypes "github.com/cortezaproject/corteza-server/system/types" -) - -type ( - identFormatter interface { - ModelIdentFormatter(connectionID uint64) (f *dal.IdentFormatter, err error) - } - - modelReloader interface { - identFormatter - ReloadModel(ctx context.Context, models ...*dal.Model) (err error) - } - - modelCreator interface { - identFormatter - CreateModel(ctx context.Context, models ...*dal.Model) (err error) - } - - modelUpdater interface { - identFormatter - UpdateModel(ctx context.Context, old, new *dal.Model) (err error) - } - - attributeUpdater interface { - identFormatter - UpdateModelAttribute(ctx context.Context, model *dal.Model, old, new *dal.Attribute, trans ...dal.TransformationFunction) (err error) - } - - modelDeleter interface { - identFormatter - DeleteModel(ctx context.Context, models ...*dal.Model) (err error) - } -) - -const ( - // https://www.rfc-editor.org/errata/eid1690 - emailLength = 254 - - // Generally the upper most limit - urlLength = 2048 - - sysID = "ID" - sysNamespaceID = "namespaceID" - sysModuleID = "moduleID" - sysCreatedAt = "createdAt" - sysCreatedBy = "createdBy" - sysUpdatedAt = "updatedAt" - sysUpdatedBy = "updatedBy" - sysDeletedAt = "deletedAt" - sysDeletedBy = "deletedBy" - sysOwnedBy = "ownedBy" - - colSysID = "id" - colSysNamespaceID = "rel_namespace" - colSysModuleID = "module_id" - colSysCreatedAt = "created_at" - colSysCreatedBy = "created_by" - colSysUpdatedAt = "updated_at" - colSysUpdatedBy = "updated_by" - colSysDeletedAt = "deleted_at" - colSysDeletedBy = "deleted_by" - colSysOwnedBy = "owned_by" -) - -func ComposeModulesReload(ctx context.Context, s store.Storer, r modelReloader) (err error) { - var ( - namespaces types.NamespaceSet - modules types.ModuleSet - fields types.ModuleFieldSet - models dal.ModelSet - ) - - namespaces, _, err = s.SearchComposeNamespaces(ctx, types.NamespaceFilter{}) - if err != nil { - return - } - - var model *dal.Model - for _, ns := range namespaces { - modules, _, err = s.SearchComposeModules(ctx, types.ModuleFilter{ - NamespaceID: ns.ID, - }) - if err != nil { - return - } - - for _, mod := range modules { - fields, _, err = s.SearchComposeModuleFields(ctx, types.ModuleFieldFilter{ - ModuleID: []uint64{mod.ID}, - }) - if err != nil { - return - } - - mod.Fields = append(mod.Fields, fields...) - - model, err = ModuleToModel(ctx, r, ns, mod) - if err != nil { - return - } - - models = append(models, model) - } - } - - return r.ReloadModel(ctx, models...) -} - -func ComposeModuleCreate(ctx context.Context, c modelCreator, ns *types.Namespace, mod *types.Module) (err error) { - model, err := ModuleToModel(ctx, c, ns, mod) - if err != nil { - return - } - if err = c.CreateModel(ctx, model); err != nil { - return - } - - return -} - -func ComposeModuleUpdate(ctx context.Context, u modelUpdater, ns *types.Namespace, old, new *types.Module) (err error) { - oldModel, err := ModuleToModel(ctx, u, ns, old) - if err != nil { - return - } - newModel, err := ModuleToModel(ctx, u, ns, new) - if err != nil { - return - } - - if err = u.UpdateModel(ctx, oldModel, newModel); err != nil { - return - } - - return -} - -func ComposeModuleFieldsUpdate(ctx context.Context, u attributeUpdater, ns *types.Namespace, old, new *types.Module) (err error) { - oldModel, err := ModuleToModel(ctx, u, ns, old) - if err != nil { - return - } - newModel, err := ModuleToModel(ctx, u, ns, new) - if err != nil { - return - } - - diff := oldModel.Diff(newModel) - for _, d := range diff { - if err = u.UpdateModelAttribute(ctx, oldModel, d.Original, d.Asserted); err != nil { - return - } - } - - return -} - -func ComposeModuleDelete(ctx context.Context, d modelDeleter, ns *types.Namespace, mod *types.Module) (err error) { - model, err := ModuleToModel(ctx, d, ns, mod) - if err != nil { - return - } - if err = d.DeleteModel(ctx, model); err != nil { - return - } - - return -} - -func ComposeModuleModelFormatter(f identFormatter, ns *types.Namespace, mod *types.Module) (formatter *dal.IdentFormatter, tplParts []string, err error) { - formatter, err = f.ModelIdentFormatter(mod.ModelConfig.ConnectionID) - if err != nil { - return - } - - modHandle, _ := handle.Cast(nil, mod.Handle, strconv.FormatUint(mod.ID, 10)) - nsHandle, _ := handle.Cast(nil, ns.Slug, strconv.FormatUint(ns.ID, 10)) - tplParts = []string{ - "module", modHandle, - "namespace", nsHandle, - } - - return -} - -// // // // // // // // // // // // // // // // // // // // // // // // // -// Utilities - -func ModuleToModel(ctx context.Context, f identFormatter, ns *types.Namespace, mod *types.Module) (*dal.Model, error) { - formatter, tplParts, err := ComposeModuleModelFormatter(f, ns, mod) - if err != nil { - return nil, err - } - - // Metadata - out := &dal.Model{ - ConnectionID: mod.ModelConfig.ConnectionID, - Label: mod.Handle, - - Attributes: make(dal.AttributeSet, len(mod.Fields)), - - SensitivityLevel: mod.Privacy.SensitivityLevel, - - ResourceID: mod.ID, - ResourceType: types.ModuleResourceType, - Resource: mod.RbacResource(), - } - - var ok bool - out.Ident, ok = formatter.ModelIdent(ctx, mod.ModelConfig.Partitioned, mod.ModelConfig.PartitionFormat, tplParts...) - if !ok { - return nil, fmt.Errorf("invalid model identifier generated: %s", out.Ident) - } - - getCodec := moduleFieldCodecBuilder(mod.ModelConfig.Partitioned, formatter) - - // Handle user-defined fields - for i, f := range mod.Fields { - out.Attributes[i], err = moduleFieldToAttribute(getCodec, ns, mod, f) - if err != nil { - return nil, err - } - } - - // Handle system fields; either default or user defined - if !mod.ModelConfig.Partitioned { - // When not partitioned the default system fields should be defined along side the `values` column - out.Attributes = append(out.Attributes, moduleModelDefaultSysAttributes()...) - } else { - // When partitioned, we use store codec defined on the module - out.Attributes = append(out.Attributes, moduleModelSysAttributes(mod, getCodec)...) - } - - return out, nil -} - -func moduleModelSysAttributes(mod *types.Module, getCodec func(f *types.ModuleField) dal.Codec) (out dal.AttributeSet) { - sysEnc := mod.ModelConfig.SystemFieldEncoding - - if sysEnc.ID != nil { - out = append(out, dal.PrimaryAttribute(sysID, getCodec(&types.ModuleField{Name: sysID, EncodingStrategy: *sysEnc.ID}))) - } - - if sysEnc.ModuleID != nil { - out = append(out, dal.FullAttribute(sysModuleID, &dal.TypeID{}, getCodec(&types.ModuleField{Name: sysModuleID, EncodingStrategy: *sysEnc.ModuleID}))) - } - if sysEnc.NamespaceID != nil { - out = append(out, dal.FullAttribute(sysNamespaceID, &dal.TypeID{}, getCodec(&types.ModuleField{Name: sysNamespaceID, EncodingStrategy: *sysEnc.NamespaceID}))) - } - - if sysEnc.OwnedBy != nil { - out = append(out, dal.FullAttribute(sysOwnedBy, &dal.TypeID{}, getCodec(&types.ModuleField{Name: sysOwnedBy, EncodingStrategy: *sysEnc.OwnedBy}))) - } - - if sysEnc.CreatedAt != nil { - out = append(out, dal.FullAttribute(sysCreatedAt, &dal.TypeTimestamp{}, getCodec(&types.ModuleField{Name: sysCreatedAt, EncodingStrategy: *sysEnc.CreatedAt}))) - } - if sysEnc.CreatedBy != nil { - out = append(out, dal.FullAttribute(sysCreatedBy, &dal.TypeID{}, getCodec(&types.ModuleField{Name: sysCreatedBy, EncodingStrategy: *sysEnc.CreatedBy}))) - } - - if sysEnc.UpdatedAt != nil { - out = append(out, dal.FullAttribute(sysUpdatedAt, &dal.TypeTimestamp{Nullable: true}, getCodec(&types.ModuleField{Name: sysUpdatedAt, EncodingStrategy: *sysEnc.UpdatedAt}))) - } - if sysEnc.UpdatedBy != nil { - out = append(out, dal.FullAttribute(sysUpdatedBy, &dal.TypeID{Nullable: true}, getCodec(&types.ModuleField{Name: sysUpdatedBy, EncodingStrategy: *sysEnc.UpdatedBy}))) - } - - if sysEnc.DeletedAt != nil { - out = append(out, dal.FullAttribute(sysDeletedAt, &dal.TypeTimestamp{Nullable: true}, getCodec(&types.ModuleField{Name: sysDeletedAt, EncodingStrategy: *sysEnc.DeletedAt}))) - } - if sysEnc.DeletedBy != nil { - out = append(out, dal.FullAttribute(sysDeletedBy, &dal.TypeID{Nullable: true}, getCodec(&types.ModuleField{Name: sysDeletedBy, EncodingStrategy: *sysEnc.DeletedBy}))) - } - - return -} - -func moduleModelDefaultSysAttributes() dal.AttributeSet { - return dal.AttributeSet{ - dal.PrimaryAttribute(sysID, &dal.CodecAlias{Ident: colSysID}), - - dal.FullAttribute(sysModuleID, &dal.TypeID{}, &dal.CodecAlias{Ident: colSysModuleID}), - dal.FullAttribute(sysNamespaceID, &dal.TypeID{}, &dal.CodecAlias{Ident: colSysNamespaceID}), - - dal.FullAttribute(sysOwnedBy, &dal.TypeID{}, &dal.CodecAlias{Ident: colSysOwnedBy}), - - dal.FullAttribute(sysCreatedAt, &dal.TypeTimestamp{}, &dal.CodecAlias{Ident: colSysCreatedAt}), - dal.FullAttribute(sysCreatedBy, &dal.TypeID{}, &dal.CodecAlias{Ident: colSysCreatedBy}), - - dal.FullAttribute(sysUpdatedAt, &dal.TypeTimestamp{Nullable: true}, &dal.CodecAlias{Ident: colSysUpdatedAt}), - dal.FullAttribute(sysUpdatedBy, &dal.TypeID{Nullable: true}, &dal.CodecAlias{Ident: colSysUpdatedBy}), - - dal.FullAttribute(sysDeletedAt, &dal.TypeTimestamp{Nullable: true}, &dal.CodecAlias{Ident: colSysDeletedAt}), - dal.FullAttribute(sysDeletedBy, &dal.TypeID{Nullable: true}, &dal.CodecAlias{Ident: colSysDeletedBy}), - } -} - -func moduleFieldToAttribute(getCodec func(f *types.ModuleField) dal.Codec, ns *types.Namespace, mod *types.Module, f *types.ModuleField) (out *dal.Attribute, err error) { - kind := f.Kind - if kind == "" { - kind = "String" - } - - switch strings.ToLower(kind) { - case "bool", "boolean": - at := &dal.TypeBoolean{} - out = dal.FullAttribute(f.Name, at, getCodec(f)) - case "datetime": - switch { - case f.IsDateOnly(): - at := &dal.TypeDate{} - out = dal.FullAttribute(f.Name, at, getCodec(f)) - case f.IsTimeOnly(): - at := &dal.TypeTime{} - out = dal.FullAttribute(f.Name, at, getCodec(f)) - default: - at := &dal.TypeTimestamp{} - out = dal.FullAttribute(f.Name, at, getCodec(f)) - } - case "email": - at := &dal.TypeText{Length: emailLength} - out = dal.FullAttribute(f.Name, at, getCodec(f)) - case "file": - at := &dal.TypeRef{ - RefModel: &dal.Model{Resource: "corteza::system:attachment"}, - RefAttribute: &dal.Attribute{Ident: "id"}, - } - out = dal.FullAttribute(f.Name, at, getCodec(f)) - case "number": - at := &dal.TypeNumber{ - Precision: f.Options.Precision(), - } - out = dal.FullAttribute(f.Name, at, getCodec(f)) - case "record": - at := &dal.TypeRef{ - RefModel: &dal.Model{ - ResourceID: f.Options.UInt64("moduleID"), - ResourceType: types.ModuleResourceType, - }, - RefAttribute: &dal.Attribute{ - Ident: "id", - }, - } - out = dal.FullAttribute(f.Name, at, getCodec(f)) - case "select": - at := &dal.TypeEnum{ - Values: f.SelectOptions(), - } - out = dal.FullAttribute(f.Name, at, getCodec(f)) - case "string": - at := &dal.TypeText{ - Length: 0, - } - out = dal.FullAttribute(f.Name, at, getCodec(f)) - case "url": - at := &dal.TypeText{ - Length: urlLength, - } - out = dal.FullAttribute(f.Name, at, getCodec(f)) - case "user": - at := &dal.TypeRef{ - RefModel: &dal.Model{ - ResourceType: systemTypes.UserResourceType, - }, - RefAttribute: &dal.Attribute{ - Ident: "id", - }, - } - out = dal.FullAttribute(f.Name, at, getCodec(f)) - - default: - return nil, fmt.Errorf("invalid field %s: kind %s not supported", f.Name, f.Kind) - } - - out.SensitivityLevel = f.Privacy.SensitivityLevel - out.Label = f.Name - - return -} - -func moduleFieldCodecBuilder(partitioned bool, formatter *dal.IdentFormatter) func(f *types.ModuleField) dal.Codec { - return func(f *types.ModuleField) dal.Codec { - return moduleFieldCodec(f, partitioned, formatter) - } -} - -func moduleFieldCodec(f *types.ModuleField, partitioned bool, formatter *dal.IdentFormatter) (strat dal.Codec) { - if partitioned { - strat = &dal.CodecPlain{} - } else { - ident, ok := formatter.AttributeIdent(partitioned, f.Name) - if !ok { - // @todo put in configs or something - ident = "values" - } - - strat = &dal.CodecRecordValueSetJSON{ - Ident: ident, - } - } - - switch { - case f.EncodingStrategy.EncodingStrategyAlias != nil: - strat = &dal.CodecAlias{ - Ident: f.EncodingStrategy.EncodingStrategyAlias.Ident, - } - case f.EncodingStrategy.EncodingStrategyJSON != nil: - strat = &dal.CodecRecordValueSetJSON{ - Ident: f.EncodingStrategy.EncodingStrategyJSON.Ident, - } - } - - return -} diff --git a/compose/dalutils/records.go b/compose/dalutils/records.go index 69868aeb8..91a8f6127 100644 --- a/compose/dalutils/records.go +++ b/compose/dalutils/records.go @@ -258,3 +258,44 @@ func recToGetters(rr ...*types.Record) (out []dal.ValueGetter) { return } + +func recCreateCapabilities(m *types.Module) (out capabilities.Set) { + return capabilities.CreateCapabilities(m.ModelConfig.Capabilities...) +} + +func recUpdateCapabilities(m *types.Module) (out capabilities.Set) { + return capabilities.UpdateCapabilities(m.ModelConfig.Capabilities...) +} + +func recDeleteCapabilities(m *types.Module) (out capabilities.Set) { + return capabilities.DeleteCapabilities(m.ModelConfig.Capabilities...) +} + +func recFilterCapabilities(f types.RecordFilter) (out capabilities.Set) { + if f.PageCursor != nil { + out = append(out, capabilities.Paging) + } + + if f.IncPageNavigation { + out = append(out, capabilities.Paging) + } + + if f.IncTotal { + out = append(out, capabilities.Stats) + } + + if f.Sort != nil { + out = append(out, capabilities.Sorting) + } + + return +} + +func recSearchCapabilities(m *types.Module, f types.RecordFilter) (out capabilities.Set) { + return capabilities.SearchCapabilities(m.ModelConfig.Capabilities...). + Union(recFilterCapabilities(f)) +} + +func recLookupCapabilities(m *types.Module) (out capabilities.Set) { + return capabilities.LookupCapabilities(m.ModelConfig.Capabilities...) +} diff --git a/compose/service/dal_interfaces.go b/compose/service/dal_interfaces.go index 3c9b7b44e..8361f6e96 100644 --- a/compose/service/dal_interfaces.go +++ b/compose/service/dal_interfaces.go @@ -10,13 +10,11 @@ import ( type ( dalModeler interface { - ModelIdentFormatter(connectionID uint64) (f *dal.IdentFormatter, err error) + SearchModels(ctx context.Context) (out dal.ModelSet, err error) + ReplaceModel(ctx context.Context, model *dal.Model) (err error) + RemoveModel(ctx context.Context, connectionID, ID uint64) (err error) - ReloadModel(ctx context.Context, models ...*dal.Model) (err error) - CreateModel(ctx context.Context, models ...*dal.Model) (err error) - UpdateModel(ctx context.Context, old, new *dal.Model) (err error) - UpdateModelAttribute(ctx context.Context, model *dal.Model, old, new *dal.Attribute, trans ...dal.TransformationFunction) (err error) - DeleteModel(ctx context.Context, models ...*dal.Model) (err error) + GetConnectionMeta(ctx context.Context, ID uint64) (cm dal.ConnectionMeta, err error) SearchModelIssues(connectionID, resourceID uint64) (out []error) } diff --git a/compose/service/module.go b/compose/service/module.go index 641892400..dae2d3d0b 100644 --- a/compose/service/module.go +++ b/compose/service/module.go @@ -3,13 +3,14 @@ package service import ( "context" "fmt" - "github.com/cortezaproject/corteza-server/pkg/auth" - "github.com/cortezaproject/corteza-server/pkg/dal" "reflect" "sort" "strconv" + "strings" + + "github.com/cortezaproject/corteza-server/pkg/auth" + "github.com/cortezaproject/corteza-server/pkg/dal" - "github.com/cortezaproject/corteza-server/compose/dalutils" "github.com/cortezaproject/corteza-server/compose/service/event" "github.com/cortezaproject/corteza-server/compose/service/values" "github.com/cortezaproject/corteza-server/compose/types" @@ -21,6 +22,7 @@ import ( "github.com/cortezaproject/corteza-server/pkg/locale" "github.com/cortezaproject/corteza-server/pkg/slice" "github.com/cortezaproject/corteza-server/store" + systemTypes "github.com/cortezaproject/corteza-server/system/types" ) type ( @@ -31,7 +33,7 @@ type ( store store.Storer locale ResourceTranslationsManagerService - dal dalService + dal dalModelManager } moduleAccessController interface { @@ -63,6 +65,20 @@ type ( moduleUpdateHandler func(ctx context.Context, ns *types.Namespace, c *types.Module) (moduleChanges, error) moduleChanges uint8 + + // Model management on DAL Service + dalModelManager interface { + GetConnectionMeta(ctx context.Context, ID uint64) (cm dal.ConnectionMeta, err error) + + ReplaceModel(context.Context, *dal.Model) error + RemoveModel(ctx context.Context, connectionID, ID uint64) error + ReplaceModelAttribute(ctx context.Context, model *dal.Model, old, new *dal.Attribute, trans ...dal.TransformationFunction) (err error) + SearchModelIssues(connectionID, ID uint64) []error + } + + dalIdentFormatter interface { + Format(ctx context.Context, template string) (out string, ok bool) + } ) const ( @@ -72,6 +88,36 @@ const ( moduleFieldsChanged moduleChanges = 4 ) +const ( + // https://www.rfc-editor.org/errata/eid1690 + emailLength = 254 + + // Generally the upper most limit + urlLength = 2048 + + sysID = "ID" + sysNamespaceID = "namespaceID" + sysModuleID = "moduleID" + sysCreatedAt = "createdAt" + sysCreatedBy = "createdBy" + sysUpdatedAt = "updatedAt" + sysUpdatedBy = "updatedBy" + sysDeletedAt = "deletedAt" + sysDeletedBy = "deletedBy" + sysOwnedBy = "ownedBy" + + colSysID = "id" + colSysNamespaceID = "rel_namespace" + colSysModuleID = "module_id" + colSysCreatedAt = "created_at" + colSysCreatedBy = "created_by" + colSysUpdatedAt = "updated_at" + colSysUpdatedBy = "updated_by" + colSysDeletedAt = "deleted_at" + colSysDeletedBy = "deleted_by" + colSysOwnedBy = "owned_by" +) + var ( systemFields = slice.ToStringBoolMap([]string{ "recordID", @@ -352,7 +398,7 @@ func (svc module) Create(ctx context.Context, new *types.Module) (*types.Module, return } - if err = dalutils.ComposeModuleCreate(ctx, svc.dal, ns, new); err != nil { + if err = dalModelReplace(ctx, svc.dal, ns, new); err != nil { return err } @@ -381,7 +427,7 @@ func (svc module) UndeleteByID(ctx context.Context, namespaceID, moduleID uint64 // // Directly using store so we don't spam the action log func (svc *module) ReloadDALModels(ctx context.Context) (err error) { - return dalutils.ComposeModulesReload(ctx, svc.store, svc.dal) + return dalModelReload(ctx, svc.store, svc.dal) } // FindSensitive will list all module with at least one private module field @@ -513,17 +559,17 @@ func (svc module) updater(ctx context.Context, namespaceID, moduleID uint64, act if err = svc.eventbus.WaitFor(ctx, event.ModuleAfterUpdate(m, old, ns)); err != nil { return err } - if err = dalutils.ComposeModuleUpdate(ctx, svc.dal, ns, old, m); err != nil { + if err = dalModelReplace(ctx, svc.dal, ns, old, m); err != nil { return err } - if err = dalutils.ComposeModuleFieldsUpdate(ctx, svc.dal, ns, old, m); err != nil { + if err = dalAttributeReplace(ctx, svc.dal, ns, old, m); err != nil { return err } } else { if err = svc.eventbus.WaitFor(ctx, event.ModuleAfterDelete(nil, old, ns)); err != nil { return } - if err = dalutils.ComposeModuleDelete(ctx, svc.dal, ns, m); err != nil { + if err = dalModelRemove(ctx, svc.dal, m); err != nil { return err } } @@ -899,7 +945,7 @@ func loadModuleFields(ctx context.Context, s store.Storer, mm ...*types.Module) for _, m := range mm { m.Fields = ff.FilterByModule(m.ID) - _ = m.Fields.Walk(func(f *types.ModuleField) error { + m.Fields.Walk(func(f *types.ModuleField) error { f.NamespaceID = m.NamespaceID return nil }) @@ -979,3 +1025,406 @@ func loadModuleLabels(ctx context.Context, s store.Labels, set ...*types.Module) return nil } + +// dalModelReload reloads all defined compose modules into the DAL +func dalModelReload(ctx context.Context, s store.Storer, dmm dalModelManager) (err error) { + // Get all available namespaces + nn, _, err := store.SearchComposeNamespaces(ctx, s, types.NamespaceFilter{}) + if err != nil { + return + } + + // Get all available connections + mm, _, err := store.SearchComposeModules(ctx, s, types.ModuleFilter{}) + if err != nil { + return + } + err = loadModuleFields(ctx, s, mm...) + if err != nil { + return err + } + + // Reload! + for _, ns := range nn { + err = dalModelReplace(ctx, dmm, ns, modulesForNamespace(ns, mm)...) + if err != nil { + return + } + } + + return +} + +// modulesForNamespace returns all of the modules belonging to that namespace +// @todo implement some indexing at an earlier step for faster processing; will do for now +func modulesForNamespace(ns *types.Namespace, mm types.ModuleSet) (out types.ModuleSet) { + out = make(types.ModuleSet, 0, len(mm)) + for _, m := range mm { + if m.NamespaceID == ns.ID { + out = append(out, m) + } + } + return +} + +// Replaces all given connections +func dalModelReplace(ctx context.Context, dmm dalModelManager, ns *types.Namespace, modules ...*types.Module) (err error) { + var ( + models dal.ModelSet + ) + + models, err = moduleToModel(ctx, dmm, ns, modules...) + if err != nil { + return + } + + for _, m := range models { + err = dmm.ReplaceModel(ctx, m) + if err != nil { + return + } + } + + return +} + +func dalAttributeReplace(ctx context.Context, dmm dalModelManager, ns *types.Namespace, old, new *types.Module) (err error) { + oldModel, err := moduleToModel(ctx, dmm, ns, old) + if err != nil { + return + } + newModel, err := moduleToModel(ctx, dmm, ns, new) + if err != nil { + return + } + + diff := oldModel[0].Diff(newModel[0]) + for _, d := range diff { + if err = dmm.ReplaceModelAttribute(ctx, oldModel[0], d.Original, d.Asserted); err != nil { + return + } + } + + return +} + +// Removes a connection from DAL service +func dalModelRemove(ctx context.Context, dmm dalModelManager, mm ...*types.Module) (err error) { + for _, m := range mm { + if err = dmm.RemoveModel(ctx, m.ModelConfig.ConnectionID, m.ID); err != nil { + return err + } + } + + return +} + +func moduleToModel(ctx context.Context, dmm dalModelManager, ns *types.Namespace, modules ...*types.Module) (out dal.ModelSet, err error) { + var ( + cm dal.ConnectionMeta + attrAux dal.AttributeSet + ok bool + ) + + for connectionID, modules := range modulesByConnection(modules...) { + // Get the connection meta + cm, err = dmm.GetConnectionMeta(ctx, connectionID) + if err != nil { + return + } + + // Prepare the ident formatter for this connection + ff := dal.IdentFormatter(formatterNamespaceParams(ns)...) + if cm.PartitionValidator != "" { + ff, err = ff.WithValidationE(cm.PartitionValidator, nil) + if err != nil { + return + } + } + + // Convert all modules to models + for _, mod := range modules { + // - base params + model := &dal.Model{ + ConnectionID: connectionID, + Label: mod.Handle, + Resource: mod.RbacResource(), + ResourceID: mod.ID, + ResourceType: types.ModuleResourceType, + SensitivityLevel: mod.Privacy.SensitivityLevel, + Capabilities: mod.ModelConfig.Capabilities, + } + + // - make the model ident + ident := cm.DefaultModelIdent + if mod.ModelConfig.Partitioned { + tpl := mod.ModelConfig.PartitionFormat + if tpl == "" { + tpl = cm.DefaultPartitionFormat + } + + ident, ok = ff.Format(ctx, tpl, formatterModuleParams(mod)...) + if !ok { + err = fmt.Errorf("invalid model ident generated: %s", ident) + return + } + } + model.Ident = ident + + // Convert user-defined fields to attributes + attrAux, err = moduleFieldsToAttributes(ctx, cm, ns, mod) + if err != nil { + return + } + model.Attributes = append(model.Attributes, attrAux...) + + // Convert system fields to attribute + attrAux, err = moduleSystemFieldsToAttributes(ctx, cm, ns, mod) + if err != nil { + return + } + model.Attributes = append(model.Attributes, attrAux...) + + out = append(out, model) + } + } + + return +} + +// moduleFieldsToAttributes converts all user-defined module fields to attributes +func moduleFieldsToAttributes(ctx context.Context, cm dal.ConnectionMeta, ns *types.Namespace, mod *types.Module) (out dal.AttributeSet, err error) { + out = make(dal.AttributeSet, 0, len(mod.Fields)) + var ( + attr *dal.Attribute + ) + + for _, f := range mod.Fields { + attr, err = moduleFieldToAttribute(ctx, cm, mod, f) + if err != nil { + return + } + out = append(out, attr) + } + + return +} + +// moduleSystemFieldsToAttributes converts all system-defined module fields to attributes +func moduleSystemFieldsToAttributes(ctx context.Context, cm dal.ConnectionMeta, ns *types.Namespace, mod *types.Module) (out dal.AttributeSet, err error) { + if mod.ModelConfig.Partitioned { + return partitionedModuleSystemFieldsToAttributes(cm, mod), nil + } + return defaultModuleSystemFieldsToAttributes(), nil +} + +// partitionedModuleSystemFieldsToAttributes converts all system-defined module fields to attributes +// keeping user-defined codec in mind +func partitionedModuleSystemFieldsToAttributes(cm dal.ConnectionMeta, mod *types.Module) (out dal.AttributeSet) { + sysEnc := mod.ModelConfig.SystemFieldEncoding + + if sysEnc.ID != nil { + out = append(out, dal.PrimaryAttribute(sysID, modelFieldCodec(cm, mod, &types.ModuleField{Name: sysID, EncodingStrategy: *sysEnc.ID}))) + } + + if sysEnc.ModuleID != nil { + out = append(out, dal.FullAttribute(sysModuleID, &dal.TypeID{}, modelFieldCodec(cm, mod, &types.ModuleField{Name: sysModuleID, EncodingStrategy: *sysEnc.ModuleID}))) + } + if sysEnc.NamespaceID != nil { + out = append(out, dal.FullAttribute(sysNamespaceID, &dal.TypeID{}, modelFieldCodec(cm, mod, &types.ModuleField{Name: sysNamespaceID, EncodingStrategy: *sysEnc.NamespaceID}))) + } + + if sysEnc.OwnedBy != nil { + out = append(out, dal.FullAttribute(sysOwnedBy, &dal.TypeID{}, modelFieldCodec(cm, mod, &types.ModuleField{Name: sysOwnedBy, EncodingStrategy: *sysEnc.OwnedBy}))) + } + + if sysEnc.CreatedAt != nil { + out = append(out, dal.FullAttribute(sysCreatedAt, &dal.TypeTimestamp{}, modelFieldCodec(cm, mod, &types.ModuleField{Name: sysCreatedAt, EncodingStrategy: *sysEnc.CreatedAt}))) + } + if sysEnc.CreatedBy != nil { + out = append(out, dal.FullAttribute(sysCreatedBy, &dal.TypeID{}, modelFieldCodec(cm, mod, &types.ModuleField{Name: sysCreatedBy, EncodingStrategy: *sysEnc.CreatedBy}))) + } + + if sysEnc.UpdatedAt != nil { + out = append(out, dal.FullAttribute(sysUpdatedAt, &dal.TypeTimestamp{Nullable: true}, modelFieldCodec(cm, mod, &types.ModuleField{Name: sysUpdatedAt, EncodingStrategy: *sysEnc.UpdatedAt}))) + } + if sysEnc.UpdatedBy != nil { + out = append(out, dal.FullAttribute(sysUpdatedBy, &dal.TypeID{Nullable: true}, modelFieldCodec(cm, mod, &types.ModuleField{Name: sysUpdatedBy, EncodingStrategy: *sysEnc.UpdatedBy}))) + } + + if sysEnc.DeletedAt != nil { + out = append(out, dal.FullAttribute(sysDeletedAt, &dal.TypeTimestamp{Nullable: true}, modelFieldCodec(cm, mod, &types.ModuleField{Name: sysDeletedAt, EncodingStrategy: *sysEnc.DeletedAt}))) + } + if sysEnc.DeletedBy != nil { + out = append(out, dal.FullAttribute(sysDeletedBy, &dal.TypeID{Nullable: true}, modelFieldCodec(cm, mod, &types.ModuleField{Name: sysDeletedBy, EncodingStrategy: *sysEnc.DeletedBy}))) + } + + return +} + +// defaultModuleSystemFieldsToAttributes converts all system-defined module fields to attributes +// assuming no user-defined codec provided +func defaultModuleSystemFieldsToAttributes() dal.AttributeSet { + return dal.AttributeSet{ + dal.PrimaryAttribute(sysID, &dal.CodecAlias{Ident: colSysID}), + + dal.FullAttribute(sysModuleID, &dal.TypeID{}, &dal.CodecAlias{Ident: colSysModuleID}), + dal.FullAttribute(sysNamespaceID, &dal.TypeID{}, &dal.CodecAlias{Ident: colSysNamespaceID}), + + dal.FullAttribute(sysOwnedBy, &dal.TypeID{}, &dal.CodecAlias{Ident: colSysOwnedBy}), + + dal.FullAttribute(sysCreatedAt, &dal.TypeTimestamp{}, &dal.CodecAlias{Ident: colSysCreatedAt}), + dal.FullAttribute(sysCreatedBy, &dal.TypeID{}, &dal.CodecAlias{Ident: colSysCreatedBy}), + + dal.FullAttribute(sysUpdatedAt, &dal.TypeTimestamp{Nullable: true}, &dal.CodecAlias{Ident: colSysUpdatedAt}), + dal.FullAttribute(sysUpdatedBy, &dal.TypeID{Nullable: true}, &dal.CodecAlias{Ident: colSysUpdatedBy}), + + dal.FullAttribute(sysDeletedAt, &dal.TypeTimestamp{Nullable: true}, &dal.CodecAlias{Ident: colSysDeletedAt}), + dal.FullAttribute(sysDeletedBy, &dal.TypeID{Nullable: true}, &dal.CodecAlias{Ident: colSysDeletedBy}), + } +} + +// moduleFieldToAttribute converts the given module field to a DAL attribute +func moduleFieldToAttribute(ctx context.Context, cm dal.ConnectionMeta, mod *types.Module, f *types.ModuleField) (out *dal.Attribute, err error) { + kind := f.Kind + if kind == "" { + kind = "String" + } + + switch strings.ToLower(kind) { + case "bool", "boolean": + at := &dal.TypeBoolean{} + out = dal.FullAttribute(f.Name, at, modelFieldCodec(cm, mod, f)) + case "datetime": + switch { + case f.IsDateOnly(): + at := &dal.TypeDate{} + out = dal.FullAttribute(f.Name, at, modelFieldCodec(cm, mod, f)) + case f.IsTimeOnly(): + at := &dal.TypeTime{} + out = dal.FullAttribute(f.Name, at, modelFieldCodec(cm, mod, f)) + default: + at := &dal.TypeTimestamp{} + out = dal.FullAttribute(f.Name, at, modelFieldCodec(cm, mod, f)) + } + case "email": + at := &dal.TypeText{Length: emailLength} + out = dal.FullAttribute(f.Name, at, modelFieldCodec(cm, mod, f)) + case "file": + at := &dal.TypeRef{ + RefModel: &dal.Model{Resource: "corteza::system:attachment"}, + RefAttribute: &dal.Attribute{Ident: "id"}, + } + out = dal.FullAttribute(f.Name, at, modelFieldCodec(cm, mod, f)) + case "number": + at := &dal.TypeNumber{ + Precision: f.Options.Precision(), + } + out = dal.FullAttribute(f.Name, at, modelFieldCodec(cm, mod, f)) + case "record": + at := &dal.TypeRef{ + RefModel: &dal.Model{ + ResourceID: f.Options.UInt64("moduleID"), + ResourceType: types.ModuleResourceType, + }, + RefAttribute: &dal.Attribute{ + Ident: "id", + }, + } + out = dal.FullAttribute(f.Name, at, modelFieldCodec(cm, mod, f)) + case "select": + at := &dal.TypeEnum{ + Values: f.SelectOptions(), + } + out = dal.FullAttribute(f.Name, at, modelFieldCodec(cm, mod, f)) + case "string": + at := &dal.TypeText{ + Length: 0, + } + out = dal.FullAttribute(f.Name, at, modelFieldCodec(cm, mod, f)) + case "url": + at := &dal.TypeText{ + Length: urlLength, + } + out = dal.FullAttribute(f.Name, at, modelFieldCodec(cm, mod, f)) + case "user": + at := &dal.TypeRef{ + RefModel: &dal.Model{ + ResourceType: systemTypes.UserResourceType, + }, + RefAttribute: &dal.Attribute{ + Ident: "id", + }, + } + out = dal.FullAttribute(f.Name, at, modelFieldCodec(cm, mod, f)) + + default: + return nil, fmt.Errorf("invalid field %s: kind %s not supported", f.Name, f.Kind) + } + + out.SensitivityLevel = f.Privacy.SensitivityLevel + out.Label = f.Name + + return +} + +// modulesByConnection groups given modules by the common connectionID +func modulesByConnection(modules ...*types.Module) map[uint64]types.ModuleSet { + out := make(map[uint64]types.ModuleSet) + for _, mod := range modules { + out[mod.ModelConfig.ConnectionID] = append(out[mod.ModelConfig.ConnectionID], mod) + } + + return out +} + +// formatterNamespaceParams returns the base namespace params used for ident formatting +func formatterNamespaceParams(ns *types.Namespace) []string { + nsHandle, _ := handle.Cast(nil, ns.Slug, strconv.FormatUint(ns.ID, 10)) + return []string{ + "namespace", nsHandle, + } +} + +// formatterModuleParams returns the base module params used for ident formatting +func formatterModuleParams(mod *types.Module) []string { + modHandle, _ := handle.Cast(nil, mod.Handle, strconv.FormatUint(mod.ID, 10)) + return []string{ + "module", modHandle, + } +} + +// modelFieldCodec returns the DAL codec the given module field should use +func modelFieldCodec(cm dal.ConnectionMeta, mod *types.Module, f *types.ModuleField) (c dal.Codec) { + c = baseModelFieldCodec(cm, mod, f) + + switch { + case f.EncodingStrategy.EncodingStrategyAlias != nil: + c = &dal.CodecAlias{ + Ident: f.EncodingStrategy.EncodingStrategyAlias.Ident, + } + case f.EncodingStrategy.EncodingStrategyJSON != nil: + c = &dal.CodecRecordValueSetJSON{ + Ident: f.EncodingStrategy.EncodingStrategyJSON.Ident, + } + } + + return +} + +// baseModelFieldCodec returns the DAL codec the given module field should use by default +func baseModelFieldCodec(cm dal.ConnectionMeta, mod *types.Module, f *types.ModuleField) dal.Codec { + if mod.ModelConfig.Partitioned { + return &dal.CodecPlain{} + } + + ident := cm.DefaultAttributeIdent + if ident == "" { + // @todo put in configs or something + ident = "values" + } + + return &dal.CodecRecordValueSetJSON{ + Ident: ident, + } +} diff --git a/pkg/dal/errors.go b/pkg/dal/errors.go index 5b5f4a830..6acd2ff39 100644 --- a/pkg/dal/errors.go +++ b/pkg/dal/errors.go @@ -51,9 +51,6 @@ func errModelCreateProblematicConnection(connectionID, modelID uint64) error { func errModelCreateMissingConnection(connectionID, modelID uint64) error { return fmt.Errorf("cannot create model %d on connection %d: connection does not exist", modelID, connectionID) } -func errModelCreateDuplicate(connectionID, modelID uint64) error { - return fmt.Errorf("cannot create model %d on connection %d: model already exists", modelID, connectionID) -} func errModelCreateMissingSensitivityLevel(connectionID, modelID, sensitivityLevelID uint64) error { return fmt.Errorf("cannot create model %d on connection %d: sensitivity level %d does not exist", modelID, connectionID, sensitivityLevelID) } diff --git a/pkg/dal/ident_formatter.go b/pkg/dal/ident_formatter.go new file mode 100644 index 000000000..fc4ad814a --- /dev/null +++ b/pkg/dal/ident_formatter.go @@ -0,0 +1,120 @@ +package dal + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/PaesslerAG/gval" + "github.com/cortezaproject/corteza-server/pkg/expr" +) + +type ( + identFormatter struct { + identValidator string + identValidatorP gval.Evaluable + identValidatorParams map[string]any + + params []string + } +) + +// IdentFormatter returns an initialized ident formatter preconfigured with given +// base params. +// +// The ident formatter is primarily used for defining model and attribute identifiers. +// +// Base params are provided in key,value pairs and the function panics if an odd number of +// parameters is provided. +func IdentFormatter(baseParams ...string) identFormatter { + out := identFormatter{} + + // Validate params; following what string replacer does + err := out.validateFormatParams(baseParams...) + if err != nil { + panic(fmt.Sprintf("cannot initialize identFormatter: %s", err.Error())) + } + + // Some preprocessing + out.params = out.prepareFormatParams(baseParams...) + return out +} + +// WithValidation binds the given validation expression to the ident formatter +// +// The initial formatter remains unchanged. +func (f identFormatter) WithValidationE(validator string, params map[string]any) (_ identFormatter, err error) { + f.identValidator = validator + f.identValidatorP, err = expr.Parser().NewEvaluable(validator) + f.identValidatorParams = params + return f, err +} + +// WithValidation binds the given validation expression to the ident formatter +// +// The initial formatter remains unchanged. +// +// The function panics if the expression can not be parsed. +func (f identFormatter) WithValidation(validator string, params map[string]any) identFormatter { + out, err := f.WithValidationE(validator, params) + if err != nil { + panic(err) + } + return out +} + +// Format returns a formatted identifier and a flag wether the resulting identifier is valid or not +// +// Parameters are provided in key,value pairs and the function panics if an odd number of +// parameters is provided. +func (f identFormatter) Format(ctx context.Context, template string, params ...string) (out string, ok bool) { + var err error + ok = true + + err = f.validateFormatParams(params...) + if err != nil { + panic(fmt.Sprintf("cannot format template: %s", err.Error())) + } + + f.params = append(f.params, f.prepareFormatParams(params...)...) + + rpl := strings.NewReplacer(f.params...) + out = rpl.Replace(template) + + if f.identValidator != "" { + ok, err = f.identValidatorP.EvalBool(ctx, f.getEvalParams(out)) + ok = ok && (err == nil) + } + + return +} + +// getEvalParams is a helper to get a KV map of parameters for gval ident validation +func (f identFormatter) getEvalParams(ident string) (out map[string]any) { + out = map[string]any{ + "ident": ident, + } + + for k, v := range f.identValidatorParams { + out[k] = v + } + + return +} + +func (f identFormatter) validateFormatParams(params ...string) error { + if len(params)%2 == 1 { + return errors.New("expecting even number of parameters") + } + + return nil +} + +func (f identFormatter) prepareFormatParams(params ...string) []string { + for i := 0; i < len(params); i += 2 { + params[i] = fmt.Sprintf("{{%s}}", params[i]) + } + + return params +} diff --git a/pkg/dal/ident_formatter_test.go b/pkg/dal/ident_formatter_test.go new file mode 100644 index 000000000..cbfe5e3ae --- /dev/null +++ b/pkg/dal/ident_formatter_test.go @@ -0,0 +1,109 @@ +package dal + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIdentFormatterInit_noParams(t *testing.T) { + IdentFormatter() +} + +func TestIdentFormatterInit_okParams(t *testing.T) { + IdentFormatter("k", "v") +} + +func TestIdentFormatterInit_nokParams(t *testing.T) { + assert.Panics(t, func() { + IdentFormatter("k") + }) +} + +func TestWithValidation(t *testing.T) { + _, err := IdentFormatter().WithValidationE("true", nil) + assert.NoError(t, err) +} + +func TestWithValidation_nokExpr(t *testing.T) { + _, err := IdentFormatter().WithValidationE("1variable", nil) + assert.Error(t, err) +} + +func TestWithValidation_nokExpr_panic(t *testing.T) { + assert.Panics(t, func() { + IdentFormatter().WithValidation("1variable", nil) + }) +} + +func TestFormatting(t *testing.T) { + tcc := []struct { + name string + tpl string + params []string + validator string + ident string + ok bool + }{ + { + name: "template without params", + tpl: "identifier", + ident: "identifier", + ok: true, + }, + { + name: "template with params", + tpl: "identifier_{{k}}", + params: []string{"k", "v"}, + ident: "identifier_v", + ok: true, + }, + + { + name: "template without params; validated ok", + tpl: "identifier", + ident: "identifier", + validator: "true", + ok: true, + }, + { + name: "template with params; validated ok", + tpl: "identifier_{{k}}", + params: []string{"k", "v"}, + ident: "identifier_v", + validator: "true", + ok: true, + }, + + { + name: "template without params; validated nok", + tpl: "identifier", + ident: "identifier", + validator: "false", + ok: false, + }, + { + name: "template with params; validated nok", + tpl: "identifier_{{k}}", + params: []string{"k", "v"}, + ident: "identifier_v", + validator: "false", + ok: false, + }, + } + + ctx := context.Background() + for _, c := range tcc { + t.Run(c.name, func(t *testing.T) { + f := IdentFormatter() + if c.validator != "" { + f = f.WithValidation(c.validator, nil) + } + ident, ok := f.Format(ctx, c.tpl, c.params...) + + assert.Equal(t, c.ident, ident) + assert.Equal(t, c.ok, ok) + }) + } +} diff --git a/pkg/dal/model.go b/pkg/dal/model.go index 3ed802ada..af8abb90f 100644 --- a/pkg/dal/model.go +++ b/pkg/dal/model.go @@ -1,28 +1,15 @@ package dal import ( - "context" "fmt" "strings" - "github.com/PaesslerAG/gval" "github.com/cortezaproject/corteza-server/pkg/dal/capabilities" "github.com/cortezaproject/corteza-server/pkg/handle" "github.com/modern-go/reflect2" ) type ( - IdentFormatter struct { - defaultModelIdent string - defaultAttributeIdent string - - defaultPartitionFormat string - - things map[string]any - - partitionFormatValidator gval.Evaluable - } - // ModelFilter is used to retrieve a model from the DAL based on given params ModelFilter struct { ConnectionID uint64 @@ -215,84 +202,3 @@ func (m Model) Validate() error { return nil } - -func (f *IdentFormatter) AddEvalParam(params map[string]any) { - if f.things == nil { - f.things = make(map[string]any) - } - - for k, v := range params { - f.things[k] = v - } - -} - -func (f IdentFormatter) getEvalParams(ident string) (out map[string]any) { - out = map[string]any{ - "ident": ident, - } - - for k, v := range f.things { - out[k] = v - } - - return -} - -func (f IdentFormatter) ModelIdent(ctx context.Context, partitioned bool, tpl string, kv ...string) (out string, ok bool) { - ok = true - - if !partitioned { - return f.defaultModelIdent, ok - } - - // A bit of preprocessing - { - if len(kv)%2 != 0 { - panic("ModelIdentFormatter.ModelIdent requires key/value pairs") - } - for i := 0; i < len(kv); i += 2 { - kv[i] = fmt.Sprintf("{{%s}}", kv[i]) - } - } - - // Template preparation with defaulting based on provided KV - { - if tpl == "" { - tpl = f.defaultPartitionFormat - } - if tpl == "" { - pts := make([]string, 0, len(kv)/2) - for i := 0; i < len(kv); i += 2 { - pts = append(pts, kv[i]) - } - tpl = strings.Join(pts, "_") - } - if tpl == "" { - return "", false - } - } - - rpl := strings.NewReplacer(kv...) - out = rpl.Replace(tpl) - - if f.partitionFormatValidator != nil { - var err error - ok, err = f.partitionFormatValidator.EvalBool(ctx, f.getEvalParams(out)) - ok = ok && (err == nil) - } - - return -} - -func (f IdentFormatter) AttributeIdent(partitioned bool, ident string) (out string, ok bool) { - if !partitioned { - if f.defaultAttributeIdent != "" { - return f.defaultAttributeIdent, true - } - return "", false - } - - return ident, ident != "" - -} diff --git a/pkg/dal/service.go b/pkg/dal/service.go index 08cb6b297..b2c0d8e02 100644 --- a/pkg/dal/service.go +++ b/pkg/dal/service.go @@ -6,7 +6,6 @@ import ( "strconv" "github.com/cortezaproject/corteza-server/pkg/dal/capabilities" - "github.com/cortezaproject/corteza-server/pkg/expr" "github.com/cortezaproject/corteza-server/pkg/filter" "go.uber.org/zap" ) @@ -100,6 +99,19 @@ func Service() *service { return gSvc } +// Purge resets the service to the initial zero state +// @todo will probably need to change but for now this is ok +// +// Primarily used for testing reasons +func (svc *service) Purge(ctx context.Context) { + svc.connections = make(map[uint64]*ConnectionWrap) + svc.defConnID = 0 + svc.models = make(map[uint64]ModelSet) + svc.sensitivityLevels = SensitivityLevelIndex() + svc.connectionIssues = make(dalIssueIndex) + svc.modelIssues = make(dalIssueIndex) +} + // // // // // // // // // // // // // // // // // // // // // // // // // // meta @@ -313,6 +325,25 @@ func (svc *service) ReplaceConnection(ctx context.Context, cw *ConnectionWrap, i return nil } +// GetConnectionMeta returns the metadata of the given DAL connection +// +// The function is primarily used by services which need to know a little bit +// about the connection their resources are located in (ident formatting for example). +func (svc *service) GetConnectionMeta(ctx context.Context, ID uint64) (cm ConnectionMeta, err error) { + if ID == 0 { + ID = svc.defConnID + } + + cw := svc.getConnectionByID(ID) + if cw == nil { + err = errConnectionNotFound(ID) + return + } + + cm = cw.meta + return +} + // RemoveConnection removes the given connection from the DAL func (svc *service) RemoveConnection(ctx context.Context, ID uint64) (err error) { var ( @@ -464,25 +495,9 @@ func (svc *service) storeOpPrep(ctx context.Context, mf ModelFilter, capabilitie // // // // // // // // // // // // // // // // // // // // // // // // // // DDL -// ReloadModel unregister old models and register the new ones -func (svc *service) ReloadModel(ctx context.Context, models ...*Model) (err error) { - svc.logger.Debug("reloading models", zap.Int("count", len(models))) - - // Clear up the old ones - // @todo profile if manually removing nested pointers makes it faster - svc.models = make(map[uint64]ModelSet) - svc.clearModelIssues() - - err = svc.CreateModel(ctx, models...) - if err != nil { - return - } - - svc.logger.Debug("reloaded models") - - return -} - +// SearchModels returns a list of modules registered under DAL +// +// Primarily used for testing (for data truncate). func (svc *service) SearchModels(ctx context.Context) (out ModelSet, err error) { out = make(ModelSet, 0, 100) for _, models := range svc.models { @@ -491,297 +506,202 @@ func (svc *service) SearchModels(ctx context.Context) (out ModelSet, err error) return } -// AddModel adds support for a new model -func (svc *service) CreateModel(ctx context.Context, models ...*Model) (err error) { +// ReplaceModel adds new or updates an existing model +// +// ReplaceModel only affects the metadata (the connection, identifier, ...) +// and leaves attributes untouched. +// Use the ReplaceModelAttribute to upsert attributes. +// +// We rely on the user to provide stable and valid model definitions. +func (svc *service) ReplaceModel(ctx context.Context, model *Model) (err error) { var ( - log = svc.logger.Named("models") - issues = newIssueHelper() + ID = model.ResourceID + oldModel *Model + issues = newIssueHelper().addModel(ID) auxIssues = newIssueHelper() - ) + upd bool - svc.logger.Debug("creating", zap.Int("count", len(models))) + log = svc.logger.Named("models").With( + zap.Uint64("ID", ID), + zap.String("ident", model.Ident), + zap.Any("label", model.Label), + ) + ) defer svc.updateIssues(issues) - // Validate models - for _, model := range models { - issues.addModel(model.ResourceID) - - if model.ConnectionID == 0 { - // Replace model's connection ID with default one when zero - model.ConnectionID = svc.defConnID - } - - // Assure the connection has no issues - if svc.hasConnectionIssues(model.ConnectionID) { - issues.addModelIssue(model.ConnectionID, model.ResourceID, errModelCreateProblematicConnection(model.ConnectionID, model.ResourceID)) - } - - // Check the connection exists - conn := svc.getConnectionByID(model.ConnectionID) - if conn == nil { - issues.addModelIssue(model.ConnectionID, model.ResourceID, errModelCreateMissingConnection(model.ConnectionID, model.ResourceID)) - } - - // Check if model for the given resource already exists - existing := svc.FindModelByResourceID(model.ConnectionID, model.ResourceID) - if existing != nil { - issues.addModelIssue(model.ConnectionID, model.ResourceID, errModelCreateDuplicate(model.ConnectionID, model.ResourceID)) - } - - // Check sensitivity levels - // - model - if !svc.sensitivityLevels.includes(model.SensitivityLevel) { - issues.addModelIssue(model.ConnectionID, model.ResourceID, errModelCreateMissingSensitivityLevel(model.ConnectionID, model.ResourceID, model.SensitivityLevel)) - } else { - // Only check if it is present - if !svc.sensitivityLevels.isSubset(model.SensitivityLevel, conn.meta.SensitivityLevel) { - issues.addModelIssue(model.ConnectionID, model.ResourceID, errModelCreateGreaterSensitivityLevel(model.ConnectionID, model.ResourceID, model.SensitivityLevel, conn.meta.SensitivityLevel)) - } - } - // - attributes - for _, attr := range model.Attributes { - if !svc.sensitivityLevels.includes(attr.SensitivityLevel) { - issues.addModelIssue(model.ConnectionID, model.ResourceID, errModelCreateMissingAttributeSensitivityLevel(model.ConnectionID, model.ResourceID, attr.SensitivityLevel)) - } else { - if !svc.sensitivityLevels.isSubset(attr.SensitivityLevel, model.SensitivityLevel) { - issues.addModelIssue(model.ConnectionID, model.ResourceID, errModelCreateGreaterAttributeSensitivityLevel(model.ConnectionID, model.ResourceID, attr.SensitivityLevel, model.SensitivityLevel)) - } - } - } - - log.Debug("validated", - zap.Uint64("ID", model.ResourceID), - zap.String("ident", model.Ident), - zap.String("label", model.Label), - ) + // Replace model's connection ID with default one when zero + if model.ConnectionID == 0 { + model.ConnectionID = svc.defConnID } - // Add models to corresponding connections - for connection, models := range svc.modelByConnection(models) { - for _, model := range models { - mLog := log.With( - zap.Uint64("connectionID", model.ConnectionID), - zap.Uint64("ID", model.ResourceID), - zap.String("ident", model.Ident), - zap.String("label", model.Label), - ) + // Check if update + if oldModel = svc.FindModelByResourceID(model.ConnectionID, model.ResourceID); oldModel != nil { + log.Debug("found existing") - connectionIssues := svc.hasConnectionIssues(model.ConnectionID) - modelIssues := svc.hasModelIssues(model.ConnectionID, model.ResourceID) - - if !modelIssues && !connectionIssues { - mLog.Debug("adding model to connection") - - // Add model to connection - auxIssues, err = svc.registerModelToConnection(ctx, connection, model) - issues.mergeWith(auxIssues) - if err != nil { - return - } - } else { - if connectionIssues { - mLog.Warn("not adding to connection due to connection issues") - } - if modelIssues { - mLog.Warn("not adding to connection due to model issues") - } - } - - // Add model to internal registry - svc.models[model.ConnectionID] = append(svc.models[model.ConnectionID], model) + if oldModel.ConnectionID != model.ConnectionID { + log.Warn("changed model connection, existing data potentially unavailable") } + + upd = true } - log.Debug("done") + // Validation + svc.validateModel(issues, model, oldModel) + for _, attr := range model.Attributes { + svc.validateAttribute(issues, model, attr) + } + + // Add to connection + connectionIssues := svc.hasConnectionIssues(model.ConnectionID) + modelIssues := svc.hasModelIssues(model.ConnectionID, model.ResourceID) + if connectionIssues { + log.Warn("not adding to connection due to connection issues") + } + if modelIssues { + log.Warn("not adding to connection due to model issues") + } + + connection := svc.getConnectionByID(model.ConnectionID) + if !modelIssues && !connectionIssues { + log.Debug("adding to connection") + auxIssues, err = svc.registerModelToConnection(ctx, connection, model) + issues.mergeWith(auxIssues) + if err != nil { + log.Error("failed with errors", zap.Error(err)) + return + } + log.Debug("added to connection") + } + + // Add to registry + svc.addModelToRegistry(model, upd) + log.Debug("added") return } -// DeleteModel removes support for the model and deletes it from the connection -func (svc *service) DeleteModel(ctx context.Context, models ...*Model) (err error) { - +// RemoveModel removes the given model from DAL +// +// @todo potentially add more interaction with the connection as in letting it know a model was removed. +func (svc *service) RemoveModel(ctx context.Context, connectionID, ID uint64) (err error) { var ( - log = svc.logger.Named("models") - issues = newIssueHelper() + old *Model + + log = svc.logger.Named("models").With( + zap.Uint64("connectionID", connectionID), + zap.Uint64("ID", ID), + ) + issues = newIssueHelper().addModel(ID) ) - log.Debug("deleting", zap.Int("count", len(models))) + log.Debug("deleting") + + if connectionID == 0 { + connectionID = svc.defConnID + } defer svc.updateIssues(issues) - // validation - skip := make(map[uint64]bool) - for _, model := range models { - if model.ConnectionID == 0 { - // Replace model's connection ID with default one when zero - model.ConnectionID = svc.defConnID - } - - issues.addModel(model.ResourceID) - - // Validate existence - old := svc.FindModelByResourceIdent(model.ConnectionID, model.ResourceType, model.Resource) - if old == nil { - skip[model.ResourceID] = true - continue - } - - // Validate no leftover references - // @todo we can probably expand on this quitea bit - // for _, registered := range svc.models { - // refs := registered.FilterByReferenced(model) - // if len(refs) > 0 { - // return fmt.Errorf("cannot remove model %s: referenced by other models", model.Resource) - // } - // } + // Check we have something to remove + if old = svc.FindModelByResourceID(connectionID, ID); old == nil { + return } - // Work - for _, model := range models { - mLog := log.With( - zap.Uint64("connectionID", model.ConnectionID), - zap.Uint64("ID", model.ResourceID), - zap.String("ident", model.Ident), - zap.String("label", model.Label), - ) + // Validate no leftover references + // @todo we can probably expand on this quitea bit + // for _, registered := range svc.models { + // refs := registered.FilterByReferenced(model) + // if len(refs) > 0 { + // return fmt.Errorf("cannot remove model %s: referenced by other models", model.Resource) + // } + // } - if skip[model.ResourceID] { - mLog.Debug("model does not exist; skipping") - continue - } + // @todo should the underlying store be notified about this? + // how should this be handled; a straight up delete doesn't sound sane to me + // anymore - oldModels := svc.models[model.ConnectionID] - svc.models[model.ConnectionID] = make(ModelSet, 0, len(oldModels)) - for _, o := range oldModels { - if o.Resource == model.Resource { - continue - } - - svc.models[model.ConnectionID] = append(svc.models[model.ConnectionID], o) - } - - // @todo should the underlying store be notified about this? - // how should this be handled; a straight up delete doesn't sound sane to me - // anymore - - mLog.Debug("deleted") - } + svc.removeModelFromRegistry(old) + log.Debug("removed") return nil } -func (svc *service) UpdateModel(ctx context.Context, old *Model, new *Model) (err error) { - if old.ConnectionID == 0 { - // Replace old model's connection ID with default one when zero - old.ConnectionID = svc.defConnID +func (svc *service) validateModel(issues *issueHelper, model, oldModel *Model) { + // Connection ok? + if svc.hasConnectionIssues(model.ConnectionID) { + issues.addModelIssue(model.ConnectionID, model.ResourceID, errModelCreateProblematicConnection(model.ConnectionID, model.ResourceID)) + } + // Connection exists? + conn := svc.getConnectionByID(model.ConnectionID) + if conn == nil { + issues.addModelIssue(model.ConnectionID, model.ResourceID, errModelCreateMissingConnection(model.ConnectionID, model.ResourceID)) } - if new.ConnectionID == 0 { - // Replace new model's connection ID with default one when zero - new.ConnectionID = svc.defConnID + // If ident changed, check for duplicate + if oldModel != nil && oldModel.Ident != model.Ident { + if tmp := svc.FindModelByIdent(model.ConnectionID, model.Ident); tmp == nil { + issues.addModelIssue(oldModel.ConnectionID, oldModel.ResourceID, errModelUpdateDuplicate(model.ConnectionID, model.ResourceID)) + } } - var ( - log = svc.logger.Named("models").With( - zap.Uint64("connection", new.ConnectionID), - zap.Uint64("model", new.ResourceID), - zap.Uint64("ID", new.ResourceID), - zap.String("ident", new.Ident), - zap.String("label", new.Label), - ) - - conn *ConnectionWrap - - issues = newIssueHelper().addModel(old.ResourceID) - ) - - log.Debug("updating", zap.Uint64("model", old.ResourceID)) - - defer svc.updateIssues(issues) - - // Validation - { - // Assure the connection has no issues - if svc.hasConnectionIssues(old.ConnectionID) { - issues.addModelIssue(old.ConnectionID, old.ResourceID, errModelUpdateProblematicConnection(old.ConnectionID, old.ResourceID)) - } - // Check the connection exists - conn = svc.getConnectionByID(old.ConnectionID) - if conn == nil { - issues.addModelIssue(old.ConnectionID, old.ResourceID, errModelUpdateMissingConnection(old.ConnectionID, old.ResourceID)) - } - - // Check if old one exists - if tmp := svc.FindModelByResourceID(old.ConnectionID, old.ResourceID); tmp == nil { - issues.addModelIssue(old.ConnectionID, old.ResourceID, errModelUpdateMissingOldModel(old.ConnectionID, old.ResourceID)) - } - - // Check if the new one can fit in - // - if ident changed, check if it's duplicated - if old.Ident != new.Ident { - if tmp := svc.FindModelByIdent(new.ConnectionID, new.Ident); tmp == nil { - issues.addModelIssue(old.ConnectionID, old.ResourceID, errModelUpdateDuplicate(new.ConnectionID, new.ResourceID)) - } - } - // - assure same connection - // @todo some migration between different connections - if old.ConnectionID != new.ConnectionID { - issues.addModelIssue(new.ConnectionID, new.ResourceID, errModelUpdateConnectionMissmatch(old.ConnectionID, old.ResourceID)) - } - - // Sensitivity levels - // - model - if !svc.sensitivityLevels.includes(new.SensitivityLevel) { - issues.addModelIssue(new.ConnectionID, new.ResourceID, errModelUpdateMissingSensitivityLevel(new.ConnectionID, new.ResourceID, new.SensitivityLevel)) - } else { - if !svc.sensitivityLevels.isSubset(new.SensitivityLevel, conn.meta.SensitivityLevel) { - issues.addModelIssue(new.ConnectionID, new.ResourceID, errModelUpdateGreaterSensitivityLevel(new.ConnectionID, new.ResourceID, new.SensitivityLevel, conn.meta.SensitivityLevel)) - } - } - - // @note attribute check should be done in update model attribute so it's omitted here - } - - // Update connection - connectionIssues := svc.hasConnectionIssues(new.ConnectionID) - modelIssues := svc.hasModelIssues(new.ConnectionID, new.ResourceID) - - if !modelIssues && !connectionIssues { - log.Debug("updating connection's model") - - err = conn.connection.UpdateModel(ctx, old, new) - if err != nil { - issues.addModelIssue(new.ConnectionID, new.ResourceID, err) - } + // Sensitivity level ok and valid? + if !svc.sensitivityLevels.includes(model.SensitivityLevel) { + issues.addModelIssue(model.ConnectionID, model.ResourceID, errModelCreateMissingSensitivityLevel(model.ConnectionID, model.ResourceID, model.SensitivityLevel)) } else { - if connectionIssues { - log.Warn("not updating connection's model due to connection issues") + // Only check if it is present + if !svc.sensitivityLevels.isSubset(model.SensitivityLevel, conn.meta.SensitivityLevel) { + issues.addModelIssue(model.ConnectionID, model.ResourceID, errModelCreateGreaterSensitivityLevel(model.ConnectionID, model.ResourceID, model.SensitivityLevel, conn.meta.SensitivityLevel)) } - if modelIssues { - log.Warn("not updating connection's model due to model issues") + } +} + +func (svc *service) validateAttribute(issues *issueHelper, model *Model, attr *Attribute) { + if !svc.sensitivityLevels.includes(attr.SensitivityLevel) { + issues.addModelIssue(model.ConnectionID, model.ResourceID, errModelCreateMissingAttributeSensitivityLevel(model.ConnectionID, model.ResourceID, attr.SensitivityLevel)) + } else { + if !svc.sensitivityLevels.isSubset(attr.SensitivityLevel, model.SensitivityLevel) { + issues.addModelIssue(model.ConnectionID, model.ResourceID, errModelCreateGreaterAttributeSensitivityLevel(model.ConnectionID, model.ResourceID, attr.SensitivityLevel, model.SensitivityLevel)) } } - // Update registry +} + +func (svc *service) addModelToRegistry(model *Model, upd bool) { + if !upd { + svc.models[model.ConnectionID] = append(svc.models[model.ConnectionID], model) + return + } + ok := false - for i, model := range svc.models[old.ConnectionID] { - if model.ResourceID == old.ResourceID { - svc.models[old.ConnectionID][i] = new + for i, old := range svc.models[model.ConnectionID] { + if old.ResourceID == model.ResourceID { + svc.models[model.ConnectionID][i] = model ok = true break } } if !ok { - svc.models[old.ConnectionID] = append(svc.models[old.ConnectionID], new) + svc.models[model.ConnectionID] = append(svc.models[model.ConnectionID], model) } - - log.Debug("updated") - - return } -func (svc *service) UpdateModelAttribute(ctx context.Context, model *Model, old, new *Attribute, trans ...TransformationFunction) (err error) { +func (svc *service) removeModelFromRegistry(model *Model) { + oldModels := svc.models[model.ConnectionID] + svc.models[model.ConnectionID] = make(ModelSet, 0, len(oldModels)) + for _, o := range oldModels { + if o.Resource == model.Resource { + continue + } + + svc.models[model.ConnectionID] = append(svc.models[model.ConnectionID], o) + } +} + +// ReplaceModelAttribute adds new or updates an existing attribute for the given model +// +// We rely on the user to provide stable and valid attribute definitions. +func (svc *service) ReplaceModelAttribute(ctx context.Context, model *Model, old, new *Attribute, trans ...TransformationFunction) (err error) { svc.logger.Debug("updating model attribute", zap.Uint64("model", model.ResourceID)) var ( @@ -790,6 +710,10 @@ func (svc *service) UpdateModelAttribute(ctx context.Context, model *Model, old, ) defer svc.updateIssues(issues) + if model.ConnectionID == 0 { + model.ConnectionID = gSvc.defConnID + } + // Validation { // Connection issues @@ -798,8 +722,8 @@ func (svc *service) UpdateModelAttribute(ctx context.Context, model *Model, old, } // Check if it exists - model := svc.FindModelByResourceID(model.ConnectionID, model.ResourceID) - if model == nil { + auxModel := svc.FindModelByResourceID(model.ConnectionID, model.ResourceID) + if auxModel == nil { issues.addModelIssue(model.ConnectionID, model.ResourceID, errAttributeUpdateMissingModel(model.ConnectionID, model.ResourceID)) } @@ -868,34 +792,6 @@ func (svc *service) UpdateModelAttribute(ctx context.Context, model *Model, old, return } -func (svc *service) ModelIdentFormatter(connectionID uint64) (f *IdentFormatter, err error) { - if connectionID == 0 { - connectionID = svc.defConnID - } - - c := svc.getConnectionByID(connectionID) - - if c == nil { - err = errConnectionNotFound(connectionID) - return - } - - f = &IdentFormatter{ - defaultModelIdent: c.meta.DefaultModelIdent, - defaultAttributeIdent: c.meta.DefaultAttributeIdent, - defaultPartitionFormat: c.meta.DefaultPartitionFormat, - } - - if c.meta.PartitionValidator != "" { - f.partitionFormatValidator, err = expr.Parser().NewEvaluable(c.meta.PartitionValidator) - if err != nil { - return - } - } - - return -} - func (svc *service) FindModelByResourceID(connectionID uint64, resourceID uint64) *Model { return svc.models[connectionID].FindByResourceID(resourceID) } @@ -947,18 +843,6 @@ func (svc *service) getConnection(connectionID uint64, cc ...capabilities.Capabi return } -// modelByConnection maps the given models by their CRS -func (svc *service) modelByConnection(models ModelSet) (out map[*ConnectionWrap]ModelSet) { - out = make(map[*ConnectionWrap]ModelSet) - - for _, model := range models { - c := svc.getConnectionByID(model.ConnectionID) - out[c] = append(out[c], model) - } - - return -} - func (svc *service) registerModelToConnection(ctx context.Context, cw *ConnectionWrap, model *Model) (issues *issueHelper, err error) { issues = newIssueHelper()