DeforaNetworks/corteza
DeforaNetworks/corteza
3
0
Fork 0
Code

Refactor pkg/dal implementation

Browse Source 
* Reworked errors to not brick the system (things keep track of
  issues.
* Reworked internal state management -- keeping invalid things
  present, cleanning up the code, utilizing issues.
* Cleanup/improve error messages
This commit is contained in:
Tomaž Jerman 2022-06-14 10:51:40 +02:00
parent 7bac115ceb
commit 8eb062293f
10 changed files with 1063 additions and 263 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
pkg/dal/diff.go
View File

@ -5,8 +5,10 @@ type (
// ModelDiff defines one identified missmatch between two models
ModelDiff struct {
Type modelDiffType
Type modelDiffType
// Original will be nil when a new attribute is being added
Original *Attribute
// Asserted will be nil wen an existing attribute is being removed
Asserted *Attribute
}
@ -14,8 +16,9 @@ type (
)
const (
AttributeMissing modelDiffType = "attributeMissing"
AttributeTypeMissmatch modelDiffType = "typeMissmatch"
AttributeMissing modelDiffType = "attributeMissing"
AttributeTypeMissmatch modelDiffType = "typeMissmatch"
AttributeSensitivityMissmatch modelDiffType = "sensitivityMissmatch"
)
// Diff calculates the diff between models a and b where a is used as base
@ -34,6 +37,21 @@ func (a *Model) Diff(b *Model) (out ModelDiffSet) {
}
}
aIndex := make(map[string]struct {
found bool
attr *Attribute
})
for _, _attr := range a.Attributes {
attr := _attr
aIndex[attr.Ident] = struct {
found bool
attr *Attribute
}{
attr: attr,
}
}
// Deleted and update ones
for _, _attrA := range a.Attributes {
attrA := _attrA
@ -55,6 +73,32 @@ func (a *Model) Diff(b *Model) (out ModelDiffSet) {
Asserted: attrBAux.attr,
})
}
// Other stuff
// @todo improve; for now it'll do
if attrA.SensitivityLevel != attrBAux.attr.SensitivityLevel {
out = append(out, &ModelDiff{
Type: AttributeSensitivityMissmatch,
Original: attrA,
Asserted: attrBAux.attr,
})
}
}
// New
for _, _attrB := range b.Attributes {
attrB := _attrB
// Missmatches
_, ok := aIndex[attrB.Ident]
if !ok {
out = append(out, &ModelDiff{
Type: AttributeMissing,
Original: nil,
Asserted: attrB,
})
continue
}
}
return

2
pkg/dal/driver.go
View File

@ -79,7 +79,7 @@ type (
// Specific operations require data transformations (type change).
// Some basic ops. should be implemented on DB driver level, but greater controll can be
// achieved via the trans functions.
UpdateModelAttribute(ctx context.Context, sch *Model, old Attribute, new Attribute, trans ...TransformationFunction) error
UpdateModelAttribute(ctx context.Context, sch *Model, old, new *Attribute, trans ...TransformationFunction) error
}
ConnectionCloser interface {

179
pkg/dal/errors.go
View File

@ -1,44 +1,153 @@
package dal
import (
"github.com/cortezaproject/corteza-server/pkg/errors"
"github.com/cortezaproject/corteza-server/pkg/locale"
"fmt"
)
func errModelHigherSensitivity(model, connection string) error {
return errors.New(
errors.KindSensitiveData,
// Generic errors
"model sensitivity surpasses connection sensitivity",
errors.Meta("type", "invalid sensitivity"),
// Translation namespace & key
errors.Meta(locale.ErrorMetaNamespace{}, "internal"),
errors.Meta(locale.ErrorMetaKey{}, "dal.sensitivity.model-exceeds-connection"),
errors.Meta("model", model),
errors.Meta("connection", connection),
errors.StackSkip(1),
errors.StackTrimAtFn("http.HandlerFunc.ServeHTTP"),
)
func errModelNotFound(modelID uint64) error {
return fmt.Errorf("model %d does not exist", modelID)
}
func errAttributeHigherSensitivity(model, attribute string) error {
return errors.New(
errors.KindSensitiveData,
"attribute sensitivity surpasses model sensitivity",
errors.Meta("type", "invalid sensitivity"),
// Translation namespace & key
errors.Meta(locale.ErrorMetaNamespace{}, "internal"),
errors.Meta(locale.ErrorMetaKey{}, "dal.sensitivity.attribute-exceeds-model"),
errors.Meta("model", model),
errors.Meta("attribute", attribute),
errors.StackSkip(1),
errors.StackTrimAtFn("http.HandlerFunc.ServeHTTP"),
)
func errConnectionNotFound(connectionID uint64) error {
return fmt.Errorf("connection %d does not exist", connectionID)
}
// Connection errors
// - create
func errConnectionCreateMissingSensitivityLevel(connectionID, sensitivityLevelID uint64) error {
return fmt.Errorf("cannot create connection %d: sensitivity level does not exist %d", connectionID, sensitivityLevelID)
}
func errConnectionCreateConnectionFailed(connectionID uint64, err error) error {
return fmt.Errorf("cannot create connection %d: connection failed: %v", connectionID, err)
}
func errConnectionDeleteNotFound(connectionID uint64) error {
return fmt.Errorf("cannot delete connection %d: connection does not exist", connectionID)
}
func errConnectionDeleteCloserFailed(connectionID uint64, err error) error {
return fmt.Errorf("cannot delete connection %d: connection's driver failed to close: %v", connectionID, err)
}
// - update
func errConnectionUpdateNotFound(connectionID uint64) error {
return fmt.Errorf("cannot update connection %d: connection does not exist", connectionID)
}
func errConnectionUpdateMissingSensitivityLevel(connectionID, sensitivityLevelID uint64) error {
return fmt.Errorf("cannot update connection %d: sensitivity level %d does not exist", connectionID, sensitivityLevelID)
}
// Model errors
// - create
func errModelCreateProblematicConnection(connectionID, modelID uint64) error {
return fmt.Errorf("cannot create model %d on connection %d: connection has issues", modelID, connectionID)
}
func errModelCreateMissingConnection(connectionID, modelID uint64) error {
return fmt.Errorf("cannot create model %d on connection %d: connection does not exist", modelID, connectionID)
}
func errModelCreateDuplicate(connectionID, modelID uint64) error {
return fmt.Errorf("cannot create model %d on connection %d: model already exists", modelID, connectionID)
}
func errModelCreateMissingSensitivityLevel(connectionID, modelID, sensitivityLevelID uint64) error {
return fmt.Errorf("cannot create model %d on connection %d: sensitivity level %d does not exist", modelID, connectionID, sensitivityLevelID)
}
func errModelCreateGreaterSensitivityLevel(connectionID, modelID, modelSensitivityLevelID, connSensitivityLevelID uint64) error {
return fmt.Errorf("cannot create model %d on connection %d: sensitivity level %d exceeds connection supported sensitivity level %d", modelID, connectionID, modelSensitivityLevelID, connSensitivityLevelID)
}
func errModelCreateMissingAttributeSensitivityLevel(connectionID, modelID, sensitivityLevelID uint64) error {
return fmt.Errorf("cannot create model %d on connection %d: attribute sensitivity level %d does not exist", modelID, connectionID, sensitivityLevelID)
}
func errModelCreateGreaterAttributeSensitivityLevel(connectionID, modelID, attrSensitivityLevelID, modelSensitivityLevelID uint64) error {
return fmt.Errorf("cannot create model %d on connection %d: attribute sensitivity level %d exceeds model supported sensitivity level %d", modelID, connectionID, attrSensitivityLevelID, modelSensitivityLevelID)
}
func errModelCreateConnectionModelUnsupported(connectionID, modelID uint64) error {
return fmt.Errorf("cannot create model %d on connection %d: model already exists for connection but is not compatible with provided definition", modelID, connectionID)
}
// - update
func errModelUpdateProblematicConnection(connectionID, modelID uint64) error {
return fmt.Errorf("cannot update model %d on connection %d: connection has issues", modelID, connectionID)
}
func errModelUpdateMissingConnection(connectionID, modelID uint64) error {
return fmt.Errorf("cannot update model %d on connection %d: connection does not exist", modelID, connectionID)
}
func errModelUpdateConnectionModelUnsupported(connectionID, modelID uint64) error {
return fmt.Errorf("cannot update model %d on connection %d: model already exists for connection but is not compatible with provided definition", modelID, connectionID)
}
func errModelUpdateMissingOldModel(connectionID, modelID uint64) error {
return fmt.Errorf("cannot update model %d on connection %d: model does not exist", modelID, connectionID)
}
func errModelUpdateDuplicate(connectionID, modelID uint64) error {
return fmt.Errorf("cannot update model %d on connection %d: model already exists", modelID, connectionID)
}
func errModelUpdateConnectionMissmatch(connectionID, modelID uint64) error {
return fmt.Errorf("cannot update model %d on connection %d: cannot change model connection", modelID, connectionID)
}
func errModelUpdateMissingSensitivityLevel(connectionID, modelID, sensitivityLevelID uint64) error {
return fmt.Errorf("cannot update model %d on connection %d: sensitivity level %d does not exist", modelID, connectionID, sensitivityLevelID)
}
func errModelUpdateGreaterSensitivityLevel(connectionID, modelID, modelSensitivityLevelID, connSensitivityLevelID uint64) error {
return fmt.Errorf("cannot update model %d on connection %d: sensitivity level %d exceeds connection supported sensitivity level %d", modelID, connectionID, modelSensitivityLevelID, connSensitivityLevelID)
}
// Attribute errors
// - Update
func errAttributeUpdateProblematicConnection(connectionID, modelID uint64) error {
return fmt.Errorf("cannot update attribute for model %d on connection %d: connection has issues", modelID, connectionID)
}
func errAttributeUpdateMissingModel(connectionID, modelID uint64) error {
return fmt.Errorf("cannot update attribute for model %d on connection %d: model does not exist", modelID, connectionID)
}
func errAttributeUpdateMissingSensitivityLevel(connectionID, modelID, sensitivityLevelID uint64) error {
return fmt.Errorf("cannot update attribute for model %d on connection %d: sensitivity level %d does not exist", modelID, connectionID, sensitivityLevelID)
}
func errAttributeUpdateGreaterSensitivityLevel(connectionID, modelID, attrSensitivityLevelID, modelSensitivityLevelID uint64) error {
return fmt.Errorf("cannot update attribute for model %d on connection %d: sensitivity level %d exceeds model supported sensitivity level %d", modelID, connectionID, attrSensitivityLevelID, modelSensitivityLevelID)
}
// Record errors
func errRecordOpProblematicConnection(connectionID uint64) error {
return fmt.Errorf("cannot perform record operation: connection %d has issues", connectionID)
}
func errRecordOpProblematicModel(modelID uint64) error {
return fmt.Errorf("cannot perform record operation: model %d has issues", modelID)
}
// func errModelHigherSensitivity(model, connection string) error {
// return errors.New(
// errors.KindSensitiveData,
// "model sensitivity surpasses connection sensitivity",
// errors.Meta("type", "invalid sensitivity"),
// // Translation namespace & key
// errors.Meta(locale.ErrorMetaNamespace{}, "internal"),
// errors.Meta(locale.ErrorMetaKey{}, "dal.sensitivity.model-exceeds-connection"),
// errors.Meta("model", model),
// errors.Meta("connection", connection),
// errors.StackSkip(1),
// errors.StackTrimAtFn("http.HandlerFunc.ServeHTTP"),
// )
// }
// func errAttributeHigherSensitivity(model, attribute string) error {
// return errors.New(
// errors.KindSensitiveData,
// "attribute sensitivity surpasses model sensitivity",
// errors.Meta("type", "invalid sensitivity"),
// // Translation namespace & key
// errors.Meta(locale.ErrorMetaNamespace{}, "internal"),
// errors.Meta(locale.ErrorMetaKey{}, "dal.sensitivity.attribute-exceeds-model"),
// errors.Meta("model", model),
// errors.Meta("attribute", attribute),
// errors.StackSkip(1),
// errors.StackTrimAtFn("http.HandlerFunc.ServeHTTP"),
// )
// }

134
pkg/dal/issues.go Normal file
View File

@ -0,0 +1,134 @@
package dal
type (
issue struct {
kind issueKind
err error
}
issueSet []issue
issueHelper struct {
// these two will be used to help clear out unneeded errors
connections []uint64
models []uint64
connectionIssues dalIssueIndex
modelIssues dalIssueIndex
}
issueKind string
dalIssueIndex map[uint64]issueSet
)
const (
connectionIssue issueKind = "connection"
modelIssue issueKind = "model"
)
func newIssueHelper() *issueHelper {
return &issueHelper{
connectionIssues: make(dalIssueIndex),
modelIssues: make(dalIssueIndex),
}
}
func makeIssue(kind issueKind, err error) issue {
return issue{
kind: kind,
err: err,
}
}
func (svc *service) SearchConnectionIssues(connectionID uint64) (out []error) {
for _, issue := range svc.connectionIssues[connectionID] {
out = append(out, issue.err)
}
return
}
func (svc *service) SearchModelIssues(connectionID, resourceID uint64) (out []error) {
// @todo index by connection as well
// if _, ok := svc.modelIssues[connectionID]; !ok {
// return
// }
for _, issue := range svc.modelIssues[resourceID] {
out = append(out, issue.err)
}
return
}
func (svc *service) hasConnectionIssues(connectionID uint64) bool {
return len(svc.SearchConnectionIssues(connectionID)) > 0
}
func (svc *service) hasModelIssues(connectionID, modelID uint64) bool {
return len(svc.SearchModelIssues(connectionID, modelID)) > 0
}
func (svc *service) updateIssues(issues *issueHelper) {
for _, connectionID := range issues.connections {
delete(svc.connectionIssues, connectionID)
}
for connectionID, issues := range issues.connectionIssues {
svc.connectionIssues[connectionID] = issues
}
for _, modelID := range issues.models {
delete(svc.modelIssues, modelID)
}
for modelID, issues := range issues.modelIssues {
svc.modelIssues[modelID] = issues
}
}
func (svc *service) clearModelIssues() {
svc.modelIssues = make(dalIssueIndex)
}
func (rd *issueHelper) addConnection(connectionID uint64) *issueHelper {
rd.connections = append(rd.connections, connectionID)
return rd
}
func (rd *issueHelper) addModel(modelID uint64) *issueHelper {
rd.models = append(rd.models, modelID)
return rd
}
func (rd *issueHelper) addConnectionIssue(connectionID uint64, err error) {
rd.connectionIssues[connectionID] = append(rd.connectionIssues[connectionID], makeIssue(connectionIssue, err))
}
func (rd *issueHelper) addModelIssue(connectionID, resourceID uint64, err error) {
rd.modelIssues[resourceID] = append(rd.modelIssues[resourceID], makeIssue(modelIssue, err))
}
func (a *issueHelper) mergeWith(b *issueHelper) {
if b == nil {
return
}
for connectionID, issues := range b.connectionIssues {
a.connectionIssues[connectionID] = append(a.connectionIssues[connectionID], issues...)
}
for modelID, issues := range b.modelIssues {
a.modelIssues[modelID] = append(a.modelIssues[modelID], issues...)
}
}
// Op check utils
func (svc *service) canOpRecord(connectionID, modelID uint64) (err error) {
if svc.hasConnectionIssues(connectionID) {
return errRecordOpProblematicConnection(connectionID)
}
if svc.hasModelIssues(connectionID, modelID) {
return errRecordOpProblematicModel(modelID)
}
return nil
}

33
pkg/dal/model.go
View File

@ -6,6 +6,7 @@ import (
"strings"
"github.com/PaesslerAG/gval"
"github.com/cortezaproject/corteza-server/pkg/dal/capabilities"
"github.com/cortezaproject/corteza-server/pkg/handle"
"github.com/modern-go/reflect2"
)
@ -45,6 +46,8 @@ type (
SensitivityLevel uint64
Attributes AttributeSet
Capabilities capabilities.Set
}
ModelSet []*Model
@ -110,20 +113,27 @@ func (a *Attribute) WithMultiValue() *Attribute {
return a
}
// FindByResource returns the model that matches the resource
func (mm ModelSet) FindByResource(resType string, resource string) *Model {
func (mm ModelSet) FindByResourceID(resourceID uint64) *Model {
for _, m := range mm {
if m.ResourceType == resType && m.Resource == resource {
if m.ResourceID == resourceID {
return m
}
}
return nil
}
func (mm ModelSet) FindByID(id uint64) *Model {
func (mm ModelSet) FindByResourceIdent(resourceType, resourceIdent string) *Model {
for _, m := range mm {
if m.ResourceID == id {
if m.ResourceType == resourceType && m.Resource == resourceIdent {
return m
}
}
return nil
}
func (mm ModelSet) FindByIdent(ident string) *Model {
for _, m := range mm {
if m.Ident == ident {
return m
}
}
@ -151,6 +161,17 @@ func (aa ModelSet) FilterByReferenced(b *Model) (out ModelSet) {
return
}
func (m Model) ToFilter() ModelFilter {
return ModelFilter{
ConnectionID: m.ConnectionID,
ResourceID: m.ResourceID,
ResourceType: m.ResourceType,
Resource: m.Resource,
}
}
// HasAttribute returns true when the model includes the specified attribute
func (m Model) HasAttribute(ident string) bool {
return m.Attributes.FindByIdent(ident) != nil

21
pkg/dal/sensitivity_level.go
View File

@ -4,6 +4,7 @@ type (
SensitivityLevel struct {
Handle string
ID uint64
Level int
}
SensitivityLevelSet []SensitivityLevel
@ -15,11 +16,31 @@ type (
}
)
func SensitivityLevelIndex(levels ...SensitivityLevel) *sensitivityLevelIndex {
out := &sensitivityLevelIndex{
set: make(SensitivityLevelSet, len(levels)),
byHandle: make(map[string]int),
byID: make(map[uint64]int),
}
for i, l := range levels {
out.set[i] = l
out.byHandle[l.Handle] = i
out.byID[l.ID] = i
}
return out
}
func (sli sensitivityLevelIndex) includes(l uint64) (ok bool) {
if l == 0 {
return true
}
if sli.byID == nil {
return false
}
_, ok = sli.byID[l]
return
}

898
pkg/dal/service.go
View File

File diff suppressed because it is too large Load Diff

5
store/adapters/rdbms/dal/connection.go
View File

@ -108,15 +108,18 @@ func (c *connection) CreateModel(ctx context.Context, model *dal.Model, model2 .
func (c *connection) DeleteModel(ctx context.Context, model *dal.Model, model2 ...*dal.Model) error {
//TODO implement me
return nil
panic("implement me")
}
func (c *connection) UpdateModel(ctx context.Context, old *dal.Model, new *dal.Model) error {
//TODO implement me
return nil
panic("implement me")
}
func (c *connection) UpdateModelAttribute(ctx context.Context, sch *dal.Model, old dal.Attribute, new dal.Attribute, trans ...dal.TransformationFunction) error {
func (c *connection) UpdateModelAttribute(ctx context.Context, sch *dal.Model, old, new *dal.Attribute, trans ...dal.TransformationFunction) error {
//TODO implement me
return nil
panic("implement me")
}

2
store/adapters/rdbms/drivers/postgres/dal.go
View File

@ -29,5 +29,5 @@ func dalConnector(ctx context.Context, dsn string, cc ...capabilities.Capability
return
}
return rdbmsdal.Connection(db, Dialect()), nil
return rdbmsdal.Connection(db, Dialect(), cc...), nil
}

2
store/adapters/rdbms/drivers/sqlite/dal.go
View File

@ -29,5 +29,5 @@ func dalConnector(ctx context.Context, dsn string, cc ...capabilities.Capability
return
}
return rdbmsdal.Connection(db, Dialect()), nil
return rdbmsdal.Connection(db, Dialect(), cc...), nil
}