DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Fix DAL implementation with Envoy

Browse Source
This commit is contained in:
Denis Arh
2022-08-22 16:55:46 +02:00
parent 6799d3f8de
commit 4bb8270189
13 changed files with 196 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/boot_dal.go
+59
View File
@@ -4,7 +4,12 @@ import (
"context"
"fmt"
"github.com/cortezaproject/corteza-server/pkg/dal"
"github.com/cortezaproject/corteza-server/pkg/id"
"github.com/cortezaproject/corteza-server/store"
"github.com/cortezaproject/corteza-server/system/service"
"github.com/cortezaproject/corteza-server/system/types"
"go.uber.org/zap"
"time"
)
func (app *CortezaApp) initDAL(ctx context.Context, log *zap.Logger) (err error) {
@@ -19,8 +24,26 @@ func (app *CortezaApp) initDAL(ctx context.Context, log *zap.Logger) (err error)
return fmt.Errorf("primary store not connected")
}
var (
conn *types.DalConnection
cw *dal.ConnectionWrap
)
// load/create primary connection
if conn, err = provisionPrimaryDalConnection(ctx, app.Store); err != nil {
return
}
// Convert connection to dal.ConnectionWrap
if cw, err = service.MakeDalConnection(conn, app.Store.ToDalConn()); err != nil {
return
}
// Init DAL and prepare default connection
dal.SetGlobal(dal.New(log.Named("dal"), app.Opt.Environment.IsDevelopment()))
if err = dal.Service().ReplaceConnection(ctx, cw, true); err != nil {
return err
}
// disabled for now until we have a real need (RDBMS table gen for example) for it
//
@@ -40,3 +63,39 @@ func (app *CortezaApp) initDAL(ctx context.Context, log *zap.Logger) (err error)
return
}
// Creates entry for primary connection in the store
func provisionPrimaryDalConnection(ctx context.Context, s store.DalConnections) (conn *types.DalConnection, err error) {
conn, err = store.LookupDalConnectionByHandle(ctx, s, types.DalPrimaryConnectionHandle)
if err != nil && err != store.ErrNotFound {
return
}
// Already exists
if conn != nil {
return
}
conn = &types.DalConnection{
// Using id.Next since we dropped "special" ids a while ago.
// If needed, use the handle
ID: id.Next(),
Handle: types.DalPrimaryConnectionHandle,
Type: types.DalPrimaryConnectionResourceType,
Meta: types.ConnectionMeta{
Name: "Primary Database",
},
Config: types.ConnectionConfig{
DAL: &types.ConnectionConfigDAL{
ModelIdent: "compose_record",
Operations: dal.FullOperations(),
},
},
CreatedAt: time.Now(),
}
return conn, store.CreateDalConnection(ctx, s, conn)
}
app/boot_levels.go
+7 -7
View File
@@ -223,6 +223,13 @@ func (app *CortezaApp) InitStore(ctx context.Context) (err error) {
}
}
{
// Initialize Data Access Layer (DAL)
if err = app.initDAL(ctx, app.Log); err != nil {
return fmt.Errorf("can not initialize DAL: %w", err)
}
}
app.lvl = bootLevelStoreInitialized
return nil
}
@@ -352,13 +359,6 @@ func (app *CortezaApp) InitServices(ctx context.Context) (err error) {
return fmt.Errorf("could not reload resource translations: %w", err)
}
{
// Initialize Data Access Layer (DAL)
if err = app.initDAL(ctx, app.Log); err != nil {
return fmt.Errorf("can not initialize DAL: %w", err)
}
}
// Initializes system services
//
// Note: this is a legacy approach, all services from all 3 apps
compose/service/module.go
+8 -8
View File
@@ -1218,7 +1218,7 @@ func modulesToModelSet(dmm dalModelManager, ns *types.Namespace, mm ...*types.Mo
}
// convert each module to model
model, err = moduleToModel(mod, conn)
model, err = ModuleToModel(mod, conn.Config.ModelIdent)
if err != nil {
return
}
@@ -1262,10 +1262,10 @@ func modulesToModelSet(dmm dalModelManager, ns *types.Namespace, mm ...*types.Mo
return
}
// moduleToModel converts a module with fields to DAL model and attributes
// ModuleToModel converts a module with fields to DAL model and attributes
//
// note: this function does not do any partition placeholder replacements
func moduleToModel(mod *types.Module, conn *dal.ConnectionWrap) (model *dal.Model, err error) {
func ModuleToModel(mod *types.Module, inhIdent string) (model *dal.Model, err error) {
var (
attrAux dal.AttributeSet
)
@@ -1282,11 +1282,11 @@ func moduleToModel(mod *types.Module, conn *dal.ConnectionWrap) (model *dal.Mode
if model.Ident = mod.Config.DAL.Ident; model.Ident == "" {
// try with explicitly set ident on module's DAL config
// and fallback connection's default if it is empty
model.Ident = conn.Config.ModelIdent
model.Ident = inhIdent
}
// Convert user-defined fields to attributes
attrAux, err = moduleFieldsToAttributes(mod, conn)
attrAux, err = moduleFieldsToAttributes(mod)
if err != nil {
return
}
@@ -1303,14 +1303,14 @@ func moduleToModel(mod *types.Module, conn *dal.ConnectionWrap) (model *dal.Mode
}
// moduleFieldsToAttributes converts all user-defined module fields to attributes
func moduleFieldsToAttributes(mod *types.Module, conn *dal.ConnectionWrap) (out dal.AttributeSet, err error) {
func moduleFieldsToAttributes(mod *types.Module) (out dal.AttributeSet, err error) {
out = make(dal.AttributeSet, 0, len(mod.Fields))
var (
attr *dal.Attribute
)
for _, f := range mod.Fields {
attr, err = moduleFieldToAttribute(f, conn)
attr, err = moduleFieldToAttribute(f)
if err != nil {
return
}
@@ -1363,7 +1363,7 @@ func moduleSystemFieldsToAttributes(mod *types.Module) (out dal.AttributeSet, er
}
// moduleFieldToAttribute converts the given module field to a DAL attribute
func moduleFieldToAttribute(f *types.ModuleField, conn *dal.ConnectionWrap) (out *dal.Attribute, err error) {
func moduleFieldToAttribute(f *types.ModuleField) (out *dal.Attribute, err error) {
var (
// generate dal.Codec for each attribute
// using encoding strategy for that attribute
compose/service/module_test.go
+2 -8
View File
@@ -296,12 +296,6 @@ func TestModuleToModel(t *testing.T) {
model *dal.Model
err error
w = &dal.ConnectionWrap{
Config: dal.ConnectionConfig{
ModelIdent: "ident-from-conn-config",
},
}
m = &types.Module{
ID: 1,
Handle: "model-handle",
@@ -312,13 +306,13 @@ func TestModuleToModel(t *testing.T) {
)
t.Log("ident on DAL config not set, use ident from connection config")
model, err = moduleToModel(m, w)
model, err = ModuleToModel(m, "ident-from-conn-config")
req.NoError(err)
req.Equal("ident-from-conn-config", model.Ident)
t.Log("explicit ident in module's DAL config should override the handle")
m.Config.DAL.Ident = "explicit-ident"
model, err = moduleToModel(m, w)
model, err = ModuleToModel(m, "ident-from-conn-config")
req.NoError(err)
req.Equal("explicit-ident", model.Ident)
}
pkg/envoy/store/compose.go
+3
View File
@@ -37,6 +37,9 @@ type (
Search(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, f filter.Filter) (dal.Iterator, error)
Delete(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, pkv ...dal.ValueGetter) (err error)
Update(ctx context.Context, m dal.ModelRef, operations dal.OperationSet, pkv ...dal.ValueGetter) (err error)
ReplaceModel(context.Context, *dal.Model) error
GetConnectionByID(uint64) *dal.ConnectionWrap
}
composeDecoder struct {
pkg/envoy/store/compose_module_marshal.go
+39 -26
View File
@@ -2,6 +2,8 @@ package store
import (
"context"
"github.com/cortezaproject/corteza-server/compose/service"
"github.com/cortezaproject/corteza-server/pkg/dal"
"strconv"
"github.com/cortezaproject/corteza-server/compose/types"
@@ -260,38 +262,49 @@ func (n *composeModule) Encode(ctx context.Context, pl *payload) (err error) {
if err != nil {
return err
}
} else {
// Update existing module
switch n.cfg.OnExisting {
case resource.Skip:
return nil
return nil
case resource.MergeLeft:
res = mergeComposeModule(n.mod, res)
res.Fields = mergeComposeModuleFields(n.mod.Fields, res.Fields)
case resource.MergeRight:
res = mergeComposeModule(res, n.mod)
res.Fields = mergeComposeModuleFields(res.Fields, n.mod.Fields)
}
err = store.UpdateComposeModule(ctx, pl.s, res)
if err != nil {
return err
}
err = store.DeleteComposeModuleField(ctx, pl.s, n.mod.Fields...)
if err != nil {
return err
}
err = store.CreateComposeModuleField(ctx, pl.s, res.Fields...)
if err != nil {
return err
}
n.res.Res = res
}
// Update existing module
switch n.cfg.OnExisting {
case resource.Skip:
return nil
case resource.MergeLeft:
res = mergeComposeModule(n.mod, res)
res.Fields = mergeComposeModuleFields(n.mod.Fields, res.Fields)
case resource.MergeRight:
res = mergeComposeModule(res, n.mod)
res.Fields = mergeComposeModuleFields(res.Fields, n.mod.Fields)
var model *dal.Model
// convert module to model and assume compose_record for default ident
if model, err = service.ModuleToModel(res, "compose_record"); err != nil {
return
}
err = store.UpdateComposeModule(ctx, pl.s, res)
if err != nil {
return err
model.ConnectionID = pl.dal.GetConnectionByID(0).ID
if err = pl.dal.ReplaceModel(ctx, model); err != nil {
return
}
err = store.DeleteComposeModuleField(ctx, pl.s, n.mod.Fields...)
if err != nil {
return err
}
err = store.CreateComposeModuleField(ctx, pl.s, res.Fields...)
if err != nil {
return err
}
n.res.Res = res
return nil
}
pkg/envoy/store/compose_record_marshal.go
+1 -1
View File
@@ -329,7 +329,7 @@ func (n *composeRecord) Encode(ctx context.Context, pl *payload) (err error) {
}
}
rec.OwnedBy = service.CalcRecordOwner(old.OwnedBy, rec.OwnedBy, pl.invokerID)
rec.OwnedBy = service.CalcRecordOwner(0, rec.OwnedBy, pl.invokerID)
rvs := make(composeTypes.RecordValueSet, 0, len(r.Values))
for k, v := range r.Values {
pkg/provision/config.go
+2 -1
View File
@@ -3,6 +3,7 @@ package provision
import (
"context"
"fmt"
"github.com/cortezaproject/corteza-server/pkg/dal"
"path/filepath"
"strings"
@@ -32,7 +33,7 @@ func importConfig(ctx context.Context, log *zap.Logger, s store.Storer, paths st
yd = yaml.Decoder()
nn = make([]resource.Interface, 0, 200)
// @todo
se = es.NewStoreEncoder(s, nil, &es.EncoderConfig{OnExisting: resource.MergeLeft})
se = es.NewStoreEncoder(s, dal.Service(), &es.EncoderConfig{OnExisting: resource.MergeLeft})
bld = envoy.NewBuilder(se)
sources = make([]string, 0, 16)
pkg/provision/dal.go
-48
View File
@@ -1,48 +0,0 @@
package provision
import (
"context"
"github.com/cortezaproject/corteza-server/pkg/auth"
"github.com/cortezaproject/corteza-server/pkg/dal"
"github.com/cortezaproject/corteza-server/pkg/id"
"github.com/cortezaproject/corteza-server/store"
"github.com/cortezaproject/corteza-server/system/types"
)
// Injects primary connection
func defaultDalConnection(ctx context.Context, s store.DalConnections) error {
conn, err := store.LookupDalConnectionByHandle(ctx, s, types.DalPrimaryConnectionHandle)
if err != nil && err != store.ErrNotFound {
return err
}
// Already exists
if conn != nil {
return nil
}
conn = &types.DalConnection{
// Using id.Next since we dropped "special" ids a while ago.
// If needed, use the handle
ID: id.Next(),
Handle: types.DalPrimaryConnectionHandle,
Type: types.DalPrimaryConnectionResourceType,
Meta: types.ConnectionMeta{
Name: "Primary Database",
},
Config: types.ConnectionConfig{
DAL: &types.ConnectionConfigDAL{
ModelIdent: "compose_record",
Operations: dal.FullOperations(),
},
},
CreatedAt: *now(),
CreatedBy: auth.ServiceUser().ID,
}
return store.CreateDalConnection(ctx, s, conn)
}
pkg/provision/provision.go
-2
View File
@@ -41,8 +41,6 @@ func Run(ctx context.Context, log *zap.Logger, s store.Storer, provisionOpt opti
func() error { return authAddExternals(ctx, log.Named("auth.externals"), s) },
func() error { return oidcAutoDiscovery(ctx, log.Named("auth.oidc-auto-discovery"), s, authOpt) },
func() error { return defaultAuthClient(ctx, log.Named("auth.clients"), s, authOpt) },
func() error { return defaultDalConnection(ctx, s) },
}
for _, fn := range ffn {
provision/700_crm/1500_record_settings.yaml
+25 -25
View File
@@ -1,27 +1,27 @@
namespace: crm
records:
# Settings:
# - (envoy): { skipIf: "!missing" }
# values:
# QuoteExpirationDays: "30"
# QuoteNextNumber: "1"
# OpportunityCloseDateDays: "30"
# OpportunityProbability: "10"
# OpportunityForecaseCategory: "Pipeline"
# OpportunityStagename: "Qualification"
# ContractNextNumber: "1"
# ContractDefaultTime: "12"
# CaseNextNumber: "1"
# SolutionNextNumber: "1"
# ScoreBudget: "1"
# ScoreAuthority: "1"
# ScoreNeed: "1"
# ScoreTimescale: "1"
# ScoreTurnover: "0"
# ScoreYearsInBusiness: "1"
# ScoreSource: "0"
# ScoreProductKnowlege: "0"
# ScoreTechnicalDepth: "0"
# ScoreCompetition: "1"
# ScoreCriticality: "1"
# ScoreRepeatPurchase: "1"
Settings:
- (envoy): { skipIf: "!missing" }
values:
QuoteExpirationDays: "30"
QuoteNextNumber: "1"
OpportunityCloseDateDays: "30"
OpportunityProbability: "10"
OpportunityForecaseCategory: "Pipeline"
OpportunityStagename: "Qualification"
ContractNextNumber: "1"
ContractDefaultTime: "12"
CaseNextNumber: "1"
SolutionNextNumber: "1"
ScoreBudget: "1"
ScoreAuthority: "1"
ScoreNeed: "1"
ScoreTimescale: "1"
ScoreTurnover: "0"
ScoreYearsInBusiness: "1"
ScoreSource: "0"
ScoreProductKnowlege: "0"
ScoreTechnicalDepth: "0"
ScoreCompetition: "1"
ScoreCriticality: "1"
ScoreRepeatPurchase: "1"
provision/701_service-solution/1500_record_settings.yaml
+11 -11
View File
@@ -1,13 +1,13 @@
namespace: service-solution
records:
# Department:
# - (envoy): { skipIf: "!missing" }
# values:
# Name: "Service"
# HourCost: "50"
# Settings:
# - (envoy): { skipIf: "!missing" }
# values:
# KBNextNumber: "1"
# DefaultCostPerHour: "50"
# DefaultTimeUpdate: "0.25"
Department:
- (envoy): { skipIf: "!missing" }
values:
Name: "Service"
HourCost: "50"
Settings:
- (envoy): { skipIf: "!missing" }
values:
KBNextNumber: "1"
DefaultCostPerHour: "50"
DefaultTimeUpdate: "0.25"
system/service/dal_connection.go
+39 -32
View File
@@ -334,53 +334,60 @@ func dalConnectionReplace(ctx context.Context, primary dal.Connection, dcm dalCo
var (
cw *dal.ConnectionWrap
isPrimary bool
connConfig dal.ConnectionConfig
conn dal.Connection
)
for _, c := range cc {
isPrimary = c.Type == types.DalPrimaryConnectionResourceType
if isPrimary {
// reuse primary connection
cw, err = MakeDalConnection(c, primary)
} else {
cw, err = MakeDalConnection(c, nil)
}
if err != nil {
return
}
if err = dcm.ReplaceConnection(ctx, cw, isPrimary); err != nil {
return
}
}
return
}
// MakeDalConnection converts types.DalConnection to dal.ConnectionWrap and returns it.
func MakeDalConnection(c *types.DalConnection, existing dal.Connection) (cw *dal.ConnectionWrap, err error) {
var (
connConfig = dal.ConnectionConfig{
SensitivityLevelID: c.Config.Privacy.SensitivityLevelID,
ModelIdent: c.Config.DAL.ModelIdent,
Label: c.Handle,
}
)
if checks := len(c.Config.DAL.ModelIdentCheck); checks > 0 {
connConfig.ModelIdentCheck = make([]*regexp.Regexp, checks)
for i, m := range c.Config.DAL.ModelIdentCheck {
if connConfig.ModelIdentCheck[i], err = regexp.Compile(m); err != nil {
return fmt.Errorf("could not prepare connection model ident check for %q: %w", c.Handle, err)
}
if checks := len(c.Config.DAL.ModelIdentCheck); checks > 0 {
connConfig.ModelIdentCheck = make([]*regexp.Regexp, checks)
for i, m := range c.Config.DAL.ModelIdentCheck {
if connConfig.ModelIdentCheck[i], err = regexp.Compile(m); err != nil {
return nil, fmt.Errorf("could not prepare connection model ident check for %q: %w", c.Handle, err)
}
}
if isPrimary {
// reuse primary connection
conn = primary
} else {
conn = nil
}
cw = dal.MakeConnection(
c.ID,
conn,
dal.ConnectionParams{
Type: c.Config.DAL.Type,
Params: c.Config.DAL.Params,
},
connConfig,
c.Config.DAL.Operations...,
)
if err = dcm.ReplaceConnection(ctx, cw, isPrimary); err != nil {
return err
}
}
cw = dal.MakeConnection(
c.ID,
existing,
dal.ConnectionParams{
Type: c.Config.DAL.Type,
Params: c.Config.DAL.Params,
},
connConfig,
c.Config.DAL.Operations...,
)
return
}