From 4bb82701896060088ef8f9b7745be9ed4b465e6e Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Mon, 22 Aug 2022 16:55:46 +0200 Subject: [PATCH] Fix DAL implementation with Envoy --- app/boot_dal.go | 59 +++++++++++++++ app/boot_levels.go | 14 ++-- compose/service/module.go | 16 ++--- compose/service/module_test.go | 10 +-- pkg/envoy/store/compose.go | 3 + pkg/envoy/store/compose_module_marshal.go | 65 ++++++++++------- pkg/envoy/store/compose_record_marshal.go | 2 +- pkg/provision/config.go | 3 +- pkg/provision/dal.go | 48 ------------- pkg/provision/provision.go | 2 - provision/700_crm/1500_record_settings.yaml | 50 ++++++------- .../1500_record_settings.yaml | 22 +++--- system/service/dal_connection.go | 71 ++++++++++--------- 13 files changed, 196 insertions(+), 169 deletions(-) delete mode 100644 pkg/provision/dal.go diff --git a/app/boot_dal.go b/app/boot_dal.go index 538f9864d..7ef7bbd7b 100644 --- a/app/boot_dal.go +++ b/app/boot_dal.go @@ -4,7 +4,12 @@ import ( "context" "fmt" "github.com/cortezaproject/corteza-server/pkg/dal" + "github.com/cortezaproject/corteza-server/pkg/id" + "github.com/cortezaproject/corteza-server/store" + "github.com/cortezaproject/corteza-server/system/service" + "github.com/cortezaproject/corteza-server/system/types" "go.uber.org/zap" + "time" ) func (app *CortezaApp) initDAL(ctx context.Context, log *zap.Logger) (err error) { @@ -19,8 +24,26 @@ func (app *CortezaApp) initDAL(ctx context.Context, log *zap.Logger) (err error) return fmt.Errorf("primary store not connected") } + var ( + conn *types.DalConnection + cw *dal.ConnectionWrap + ) + + // load/create primary connection + if conn, err = provisionPrimaryDalConnection(ctx, app.Store); err != nil { + return + } + + // Convert connection to dal.ConnectionWrap + if cw, err = service.MakeDalConnection(conn, app.Store.ToDalConn()); err != nil { + return + } + // Init DAL and prepare default connection dal.SetGlobal(dal.New(log.Named("dal"), app.Opt.Environment.IsDevelopment())) + if err = dal.Service().ReplaceConnection(ctx, cw, true); err != nil { + return err + } // disabled for now until we have a real need (RDBMS table gen for example) for it // @@ -40,3 +63,39 @@ func (app *CortezaApp) initDAL(ctx context.Context, log *zap.Logger) (err error) return } + +// Creates entry for primary connection in the store +func provisionPrimaryDalConnection(ctx context.Context, s store.DalConnections) (conn *types.DalConnection, err error) { + conn, err = store.LookupDalConnectionByHandle(ctx, s, types.DalPrimaryConnectionHandle) + if err != nil && err != store.ErrNotFound { + return + } + + // Already exists + if conn != nil { + return + } + + conn = &types.DalConnection{ + // Using id.Next since we dropped "special" ids a while ago. + // If needed, use the handle + ID: id.Next(), + Handle: types.DalPrimaryConnectionHandle, + Type: types.DalPrimaryConnectionResourceType, + + Meta: types.ConnectionMeta{ + Name: "Primary Database", + }, + + Config: types.ConnectionConfig{ + DAL: &types.ConnectionConfigDAL{ + ModelIdent: "compose_record", + Operations: dal.FullOperations(), + }, + }, + + CreatedAt: time.Now(), + } + + return conn, store.CreateDalConnection(ctx, s, conn) +} diff --git a/app/boot_levels.go b/app/boot_levels.go index 1ee74083d..63e6dce39 100644 --- a/app/boot_levels.go +++ b/app/boot_levels.go @@ -223,6 +223,13 @@ func (app *CortezaApp) InitStore(ctx context.Context) (err error) { } } + { + // Initialize Data Access Layer (DAL) + if err = app.initDAL(ctx, app.Log); err != nil { + return fmt.Errorf("can not initialize DAL: %w", err) + } + } + app.lvl = bootLevelStoreInitialized return nil } @@ -352,13 +359,6 @@ func (app *CortezaApp) InitServices(ctx context.Context) (err error) { return fmt.Errorf("could not reload resource translations: %w", err) } - { - // Initialize Data Access Layer (DAL) - if err = app.initDAL(ctx, app.Log); err != nil { - return fmt.Errorf("can not initialize DAL: %w", err) - } - } - // Initializes system services // // Note: this is a legacy approach, all services from all 3 apps diff --git a/compose/service/module.go b/compose/service/module.go index de7ce4443..0ba0d3def 100644 --- a/compose/service/module.go +++ b/compose/service/module.go @@ -1218,7 +1218,7 @@ func modulesToModelSet(dmm dalModelManager, ns *types.Namespace, mm ...*types.Mo } // convert each module to model - model, err = moduleToModel(mod, conn) + model, err = ModuleToModel(mod, conn.Config.ModelIdent) if err != nil { return } @@ -1262,10 +1262,10 @@ func modulesToModelSet(dmm dalModelManager, ns *types.Namespace, mm ...*types.Mo return } -// moduleToModel converts a module with fields to DAL model and attributes +// ModuleToModel converts a module with fields to DAL model and attributes // // note: this function does not do any partition placeholder replacements -func moduleToModel(mod *types.Module, conn *dal.ConnectionWrap) (model *dal.Model, err error) { +func ModuleToModel(mod *types.Module, inhIdent string) (model *dal.Model, err error) { var ( attrAux dal.AttributeSet ) @@ -1282,11 +1282,11 @@ func moduleToModel(mod *types.Module, conn *dal.ConnectionWrap) (model *dal.Mode if model.Ident = mod.Config.DAL.Ident; model.Ident == "" { // try with explicitly set ident on module's DAL config // and fallback connection's default if it is empty - model.Ident = conn.Config.ModelIdent + model.Ident = inhIdent } // Convert user-defined fields to attributes - attrAux, err = moduleFieldsToAttributes(mod, conn) + attrAux, err = moduleFieldsToAttributes(mod) if err != nil { return } @@ -1303,14 +1303,14 @@ func moduleToModel(mod *types.Module, conn *dal.ConnectionWrap) (model *dal.Mode } // moduleFieldsToAttributes converts all user-defined module fields to attributes -func moduleFieldsToAttributes(mod *types.Module, conn *dal.ConnectionWrap) (out dal.AttributeSet, err error) { +func moduleFieldsToAttributes(mod *types.Module) (out dal.AttributeSet, err error) { out = make(dal.AttributeSet, 0, len(mod.Fields)) var ( attr *dal.Attribute ) for _, f := range mod.Fields { - attr, err = moduleFieldToAttribute(f, conn) + attr, err = moduleFieldToAttribute(f) if err != nil { return } @@ -1363,7 +1363,7 @@ func moduleSystemFieldsToAttributes(mod *types.Module) (out dal.AttributeSet, er } // moduleFieldToAttribute converts the given module field to a DAL attribute -func moduleFieldToAttribute(f *types.ModuleField, conn *dal.ConnectionWrap) (out *dal.Attribute, err error) { +func moduleFieldToAttribute(f *types.ModuleField) (out *dal.Attribute, err error) { var ( // generate dal.Codec for each attribute // using encoding strategy for that attribute diff --git a/compose/service/module_test.go b/compose/service/module_test.go index 7f67ed57c..7d7d5ad89 100644 --- a/compose/service/module_test.go +++ b/compose/service/module_test.go @@ -296,12 +296,6 @@ func TestModuleToModel(t *testing.T) { model *dal.Model err error - w = &dal.ConnectionWrap{ - Config: dal.ConnectionConfig{ - ModelIdent: "ident-from-conn-config", - }, - } - m = &types.Module{ ID: 1, Handle: "model-handle", @@ -312,13 +306,13 @@ func TestModuleToModel(t *testing.T) { ) t.Log("ident on DAL config not set, use ident from connection config") - model, err = moduleToModel(m, w) + model, err = ModuleToModel(m, "ident-from-conn-config") req.NoError(err) req.Equal("ident-from-conn-config", model.Ident) t.Log("explicit ident in module's DAL config should override the handle") m.Config.DAL.Ident = "explicit-ident" - model, err = moduleToModel(m, w) + model, err = ModuleToModel(m, "ident-from-conn-config") req.NoError(err) req.Equal("explicit-ident", model.Ident) } diff --git a/pkg/envoy/store/compose.go b/pkg/envoy/store/compose.go index eff57b4dc..9d1a1483a 100644 --- a/pkg/envoy/store/compose.go +++ b/pkg/envoy/store/compose.go @@ -37,6 +37,9 @@ type ( Search(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, f filter.Filter) (dal.Iterator, error) Delete(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, pkv ...dal.ValueGetter) (err error) Update(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, pkv ...dal.ValueGetter) (err error) + + ReplaceModel(context.Context, *dal.Model) error + GetConnectionByID(uint64) *dal.ConnectionWrap } composeDecoder struct { diff --git a/pkg/envoy/store/compose_module_marshal.go b/pkg/envoy/store/compose_module_marshal.go index 06a27cc9b..339dbad45 100644 --- a/pkg/envoy/store/compose_module_marshal.go +++ b/pkg/envoy/store/compose_module_marshal.go @@ -2,6 +2,8 @@ package store import ( "context" + "github.com/cortezaproject/corteza-server/compose/service" + "github.com/cortezaproject/corteza-server/pkg/dal" "strconv" "github.com/cortezaproject/corteza-server/compose/types" @@ -260,38 +262,49 @@ func (n *composeModule) Encode(ctx context.Context, pl *payload) (err error) { if err != nil { return err } + } else { + // Update existing module + switch n.cfg.OnExisting { + case resource.Skip: + return nil - return nil + case resource.MergeLeft: + res = mergeComposeModule(n.mod, res) + res.Fields = mergeComposeModuleFields(n.mod.Fields, res.Fields) + + case resource.MergeRight: + res = mergeComposeModule(res, n.mod) + res.Fields = mergeComposeModuleFields(res.Fields, n.mod.Fields) + } + + err = store.UpdateComposeModule(ctx, pl.s, res) + if err != nil { + return err + } + + err = store.DeleteComposeModuleField(ctx, pl.s, n.mod.Fields...) + if err != nil { + return err + } + err = store.CreateComposeModuleField(ctx, pl.s, res.Fields...) + if err != nil { + return err + } + + n.res.Res = res } - // Update existing module - switch n.cfg.OnExisting { - case resource.Skip: - return nil - - case resource.MergeLeft: - res = mergeComposeModule(n.mod, res) - res.Fields = mergeComposeModuleFields(n.mod.Fields, res.Fields) - - case resource.MergeRight: - res = mergeComposeModule(res, n.mod) - res.Fields = mergeComposeModuleFields(res.Fields, n.mod.Fields) + var model *dal.Model + // convert module to model and assume compose_record for default ident + if model, err = service.ModuleToModel(res, "compose_record"); err != nil { + return } - err = store.UpdateComposeModule(ctx, pl.s, res) - if err != nil { - return err + model.ConnectionID = pl.dal.GetConnectionByID(0).ID + + if err = pl.dal.ReplaceModel(ctx, model); err != nil { + return } - err = store.DeleteComposeModuleField(ctx, pl.s, n.mod.Fields...) - if err != nil { - return err - } - err = store.CreateComposeModuleField(ctx, pl.s, res.Fields...) - if err != nil { - return err - } - - n.res.Res = res return nil } diff --git a/pkg/envoy/store/compose_record_marshal.go b/pkg/envoy/store/compose_record_marshal.go index d9d073373..01e1f6c9a 100644 --- a/pkg/envoy/store/compose_record_marshal.go +++ b/pkg/envoy/store/compose_record_marshal.go @@ -329,7 +329,7 @@ func (n *composeRecord) Encode(ctx context.Context, pl *payload) (err error) { } } - rec.OwnedBy = service.CalcRecordOwner(old.OwnedBy, rec.OwnedBy, pl.invokerID) + rec.OwnedBy = service.CalcRecordOwner(0, rec.OwnedBy, pl.invokerID) rvs := make(composeTypes.RecordValueSet, 0, len(r.Values)) for k, v := range r.Values { diff --git a/pkg/provision/config.go b/pkg/provision/config.go index 820c70d2c..44e67406e 100644 --- a/pkg/provision/config.go +++ b/pkg/provision/config.go @@ -3,6 +3,7 @@ package provision import ( "context" "fmt" + "github.com/cortezaproject/corteza-server/pkg/dal" "path/filepath" "strings" @@ -32,7 +33,7 @@ func importConfig(ctx context.Context, log *zap.Logger, s store.Storer, paths st yd = yaml.Decoder() nn = make([]resource.Interface, 0, 200) // @todo - se = es.NewStoreEncoder(s, nil, &es.EncoderConfig{OnExisting: resource.MergeLeft}) + se = es.NewStoreEncoder(s, dal.Service(), &es.EncoderConfig{OnExisting: resource.MergeLeft}) bld = envoy.NewBuilder(se) sources = make([]string, 0, 16) diff --git a/pkg/provision/dal.go b/pkg/provision/dal.go deleted file mode 100644 index cf0f71a0c..000000000 --- a/pkg/provision/dal.go +++ /dev/null @@ -1,48 +0,0 @@ -package provision - -import ( - "context" - - "github.com/cortezaproject/corteza-server/pkg/auth" - "github.com/cortezaproject/corteza-server/pkg/dal" - "github.com/cortezaproject/corteza-server/pkg/id" - "github.com/cortezaproject/corteza-server/store" - "github.com/cortezaproject/corteza-server/system/types" -) - -// Injects primary connection -func defaultDalConnection(ctx context.Context, s store.DalConnections) error { - conn, err := store.LookupDalConnectionByHandle(ctx, s, types.DalPrimaryConnectionHandle) - if err != nil && err != store.ErrNotFound { - return err - } - - // Already exists - if conn != nil { - return nil - } - - conn = &types.DalConnection{ - // Using id.Next since we dropped "special" ids a while ago. - // If needed, use the handle - ID: id.Next(), - Handle: types.DalPrimaryConnectionHandle, - Type: types.DalPrimaryConnectionResourceType, - - Meta: types.ConnectionMeta{ - Name: "Primary Database", - }, - - Config: types.ConnectionConfig{ - DAL: &types.ConnectionConfigDAL{ - ModelIdent: "compose_record", - Operations: dal.FullOperations(), - }, - }, - - CreatedAt: *now(), - CreatedBy: auth.ServiceUser().ID, - } - - return store.CreateDalConnection(ctx, s, conn) -} diff --git a/pkg/provision/provision.go b/pkg/provision/provision.go index 9f7021557..9bd915d81 100644 --- a/pkg/provision/provision.go +++ b/pkg/provision/provision.go @@ -41,8 +41,6 @@ func Run(ctx context.Context, log *zap.Logger, s store.Storer, provisionOpt opti func() error { return authAddExternals(ctx, log.Named("auth.externals"), s) }, func() error { return oidcAutoDiscovery(ctx, log.Named("auth.oidc-auto-discovery"), s, authOpt) }, func() error { return defaultAuthClient(ctx, log.Named("auth.clients"), s, authOpt) }, - - func() error { return defaultDalConnection(ctx, s) }, } for _, fn := range ffn { diff --git a/provision/700_crm/1500_record_settings.yaml b/provision/700_crm/1500_record_settings.yaml index 718b325fd..ca784a5c9 100644 --- a/provision/700_crm/1500_record_settings.yaml +++ b/provision/700_crm/1500_record_settings.yaml @@ -1,27 +1,27 @@ namespace: crm records: -# Settings: -# - (envoy): { skipIf: "!missing" } -# values: -# QuoteExpirationDays: "30" -# QuoteNextNumber: "1" -# OpportunityCloseDateDays: "30" -# OpportunityProbability: "10" -# OpportunityForecaseCategory: "Pipeline" -# OpportunityStagename: "Qualification" -# ContractNextNumber: "1" -# ContractDefaultTime: "12" -# CaseNextNumber: "1" -# SolutionNextNumber: "1" -# ScoreBudget: "1" -# ScoreAuthority: "1" -# ScoreNeed: "1" -# ScoreTimescale: "1" -# ScoreTurnover: "0" -# ScoreYearsInBusiness: "1" -# ScoreSource: "0" -# ScoreProductKnowlege: "0" -# ScoreTechnicalDepth: "0" -# ScoreCompetition: "1" -# ScoreCriticality: "1" -# ScoreRepeatPurchase: "1" + Settings: + - (envoy): { skipIf: "!missing" } + values: + QuoteExpirationDays: "30" + QuoteNextNumber: "1" + OpportunityCloseDateDays: "30" + OpportunityProbability: "10" + OpportunityForecaseCategory: "Pipeline" + OpportunityStagename: "Qualification" + ContractNextNumber: "1" + ContractDefaultTime: "12" + CaseNextNumber: "1" + SolutionNextNumber: "1" + ScoreBudget: "1" + ScoreAuthority: "1" + ScoreNeed: "1" + ScoreTimescale: "1" + ScoreTurnover: "0" + ScoreYearsInBusiness: "1" + ScoreSource: "0" + ScoreProductKnowlege: "0" + ScoreTechnicalDepth: "0" + ScoreCompetition: "1" + ScoreCriticality: "1" + ScoreRepeatPurchase: "1" diff --git a/provision/701_service-solution/1500_record_settings.yaml b/provision/701_service-solution/1500_record_settings.yaml index a5d64f597..66fbff710 100644 --- a/provision/701_service-solution/1500_record_settings.yaml +++ b/provision/701_service-solution/1500_record_settings.yaml @@ -1,13 +1,13 @@ namespace: service-solution records: -# Department: -# - (envoy): { skipIf: "!missing" } -# values: -# Name: "Service" -# HourCost: "50" -# Settings: -# - (envoy): { skipIf: "!missing" } -# values: -# KBNextNumber: "1" -# DefaultCostPerHour: "50" -# DefaultTimeUpdate: "0.25" + Department: + - (envoy): { skipIf: "!missing" } + values: + Name: "Service" + HourCost: "50" + Settings: + - (envoy): { skipIf: "!missing" } + values: + KBNextNumber: "1" + DefaultCostPerHour: "50" + DefaultTimeUpdate: "0.25" diff --git a/system/service/dal_connection.go b/system/service/dal_connection.go index 0a173f4dc..47bc7f235 100644 --- a/system/service/dal_connection.go +++ b/system/service/dal_connection.go @@ -334,53 +334,60 @@ func dalConnectionReplace(ctx context.Context, primary dal.Connection, dcm dalCo var ( cw *dal.ConnectionWrap isPrimary bool - - connConfig dal.ConnectionConfig - - conn dal.Connection ) for _, c := range cc { isPrimary = c.Type == types.DalPrimaryConnectionResourceType + if isPrimary { + // reuse primary connection + cw, err = MakeDalConnection(c, primary) + } else { + cw, err = MakeDalConnection(c, nil) + } + + if err != nil { + return + } + + if err = dcm.ReplaceConnection(ctx, cw, isPrimary); err != nil { + return + } + } + + return +} + +// MakeDalConnection converts types.DalConnection to dal.ConnectionWrap and returns it. +func MakeDalConnection(c *types.DalConnection, existing dal.Connection) (cw *dal.ConnectionWrap, err error) { + var ( connConfig = dal.ConnectionConfig{ SensitivityLevelID: c.Config.Privacy.SensitivityLevelID, ModelIdent: c.Config.DAL.ModelIdent, Label: c.Handle, } + ) - if checks := len(c.Config.DAL.ModelIdentCheck); checks > 0 { - connConfig.ModelIdentCheck = make([]*regexp.Regexp, checks) - for i, m := range c.Config.DAL.ModelIdentCheck { - if connConfig.ModelIdentCheck[i], err = regexp.Compile(m); err != nil { - return fmt.Errorf("could not prepare connection model ident check for %q: %w", c.Handle, err) - } + if checks := len(c.Config.DAL.ModelIdentCheck); checks > 0 { + connConfig.ModelIdentCheck = make([]*regexp.Regexp, checks) + for i, m := range c.Config.DAL.ModelIdentCheck { + if connConfig.ModelIdentCheck[i], err = regexp.Compile(m); err != nil { + return nil, fmt.Errorf("could not prepare connection model ident check for %q: %w", c.Handle, err) } } - - if isPrimary { - // reuse primary connection - conn = primary - } else { - conn = nil - } - - cw = dal.MakeConnection( - c.ID, - conn, - dal.ConnectionParams{ - Type: c.Config.DAL.Type, - Params: c.Config.DAL.Params, - }, - connConfig, - c.Config.DAL.Operations..., - ) - - if err = dcm.ReplaceConnection(ctx, cw, isPrimary); err != nil { - return err - } } + cw = dal.MakeConnection( + c.ID, + existing, + dal.ConnectionParams{ + Type: c.Config.DAL.Type, + Params: c.Config.DAL.Params, + }, + connConfig, + c.Config.DAL.Operations..., + ) + return }