From 1e2ee1bb00ca12296fe4af75da3eabce55ec8429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Thu, 5 Nov 2020 20:23:59 +0100 Subject: [PATCH] Rework how records are handled --- pkg/envoy/resource/compose_record.go | 48 +++--- pkg/envoy/tmp/compose_record.go | 156 ++++++++++++++++++++ pkg/envoy/tmp/compose_record_set.go | 13 -- pkg/envoy/tmp/compose_record_set_preproc.go | 89 ----------- pkg/envoy/yaml/compose_record.go | 111 +++++++------- 5 files changed, 236 insertions(+), 181 deletions(-) create mode 100644 pkg/envoy/tmp/compose_record.go delete mode 100644 pkg/envoy/tmp/compose_record_set.go delete mode 100644 pkg/envoy/tmp/compose_record_set_preproc.go diff --git a/pkg/envoy/resource/compose_record.go b/pkg/envoy/resource/compose_record.go index 6e6d89e87..b78ba99f5 100644 --- a/pkg/envoy/resource/compose_record.go +++ b/pkg/envoy/resource/compose_record.go @@ -1,36 +1,46 @@ package resource import ( - "fmt" - "github.com/cortezaproject/corteza-server/compose/types" ) -const ( - COMPOSE_RECORD_RESOURCE_TYPE = "ComposeRecordSet" -) - type ( + ComposeRecordRaw struct { + ID string + Values map[string]string + SysValues map[string]string + RefUser map[string]string + } + ComposeRecordRawSet []*ComposeRecordRaw + + crsWalker func(f func(r *ComposeRecordRaw) error) error + ComposeRecord struct { *base - Res *types.Record - nsRef string - modRef string - userRef map[string]string + // Res *types.Record + + Walker crsWalker + + NsRef *Ref + ModRef *Ref + ModFields types.ModuleFieldSet + UserRef map[string]string } ) -func NewComposeRecord(res *types.Record, nsRef, modRef string, userRef map[string]string) *ComposeRecord { +func NewComposeRecordSet(w crsWalker, nsRef, modRef string) *ComposeRecord { r := &ComposeRecord{base: &base{}} r.SetResourceType(COMPOSE_RECORD_RESOURCE_TYPE) - r.Res = res - r.nsRef = nsRef - r.modRef = modRef + r.Walker = w - r.AddIdentifier(identifiers(res.ID)...) + r.AddIdentifier(identifiers(modRef)...) - r.AddRef(COMPOSE_NAMESPACE_RESOURCE_TYPE, nsRef) - r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, modRef) + // for _, u := range userRef { + // r.AddRef(USER_RESOURCE_TYPE, u) + // } + + r.NsRef = r.AddRef(COMPOSE_NAMESPACE_RESOURCE_TYPE, nsRef) + r.ModRef = r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, modRef) return r } @@ -38,9 +48,5 @@ func NewComposeRecord(res *types.Record, nsRef, modRef string, userRef map[strin func (m *ComposeRecord) SearchQuery() types.RecordFilter { f := types.RecordFilter{} - if m.Res.ID > 0 { - f.Query = fmt.Sprintf("recordID=%d", m.Res.ID) - } - return f } diff --git a/pkg/envoy/tmp/compose_record.go b/pkg/envoy/tmp/compose_record.go new file mode 100644 index 000000000..986b192cc --- /dev/null +++ b/pkg/envoy/tmp/compose_record.go @@ -0,0 +1,156 @@ +package tmp + +import ( + "context" + "errors" + "time" + + "github.com/cortezaproject/corteza-server/compose/types" + "github.com/cortezaproject/corteza-server/pkg/envoy" + "github.com/cortezaproject/corteza-server/pkg/envoy/resource" + "github.com/cortezaproject/corteza-server/store" +) + +type ( + composeRecordPreproc struct { + es *encoderState + s store.Storer + } +) + +func NewComposeRecordPreproc(is *encoderState, s store.Storer) *composeRecordPreproc { + return &composeRecordPreproc{ + es: is, + s: s, + } +} + +func (p *composeRecordPreproc) Process(ctx context.Context, state *envoy.ExecState) error { + res, is := state.Res.(*resource.ComposeRecord) + if !is { + return nil + } + + // Get relate namespace + ns, err := findNamespace(ctx, p.s, state.ParentResources, res.NsRef.Identifiers) + if err != nil { + return err + } + if ns == nil { + return errors.New("@todo couldn't resolve namespace") + } + p.es.Set(res, res.NsRef.ResourceType, ns.ID, res.NsRef.Identifiers.StringSlice()...) + + // Get relate namespace + mod, err := findModule(ctx, p.s, state.ParentResources, res.ModRef.Identifiers) + if err != nil { + return err + } + if mod == nil { + return errors.New("@todo couldn't resolve module") + } + p.es.Set(res, res.ModRef.ResourceType, mod.ID, res.ModRef.Identifiers.StringSlice()...) + + // @todo existing records + // + // Hookup with labels to determine existing records + + ff, err := loadComposeModuleFields(ctx, p.s, mod) + if err != nil { + return err + } + res.ModFields = ff + + for _, f := range ff { + switch f.Kind { + case "Record": + refM := f.Options.String("module") + if refM != "" && refM != "0" { + // Make a reference with that module's records + res.AddRef(resource.COMPOSE_RECORD_RESOURCE_TYPE, refM) + } + } + } + + return nil +} + +func encodeComposeRecord(ctx context.Context, ectx *encodingContext, s store.Storer, state resRefs, res *resource.ComposeRecord) (resRefs, error) { + var err error + rState := make(resRefs) + + // Namespace... + nsID := uint64(0) + for _, v := range state[resource.COMPOSE_NAMESPACE_RESOURCE_TYPE] { + nsID = v + break + } + + // Module... + modID := uint64(0) + for _, v := range state[resource.COMPOSE_MODULE_RESOURCE_TYPE] { + modID = v + break + } + mod := &types.Module{ + NamespaceID: nsID, + ID: modID, + Fields: res.ModFields, + } + + return rState, res.Walker(func(r *resource.ComposeRecordRaw) error { + rec := &types.Record{ + NamespaceID: nsID, + ModuleID: modID, + } + + rec.ID = state[res.ResourceType()][r.ID] + if rec.ID <= 0 { + rec.ID = nextID() + } + rState.Set(resource.COMPOSE_RECORD_RESOURCE_TYPE, rec.ID, r.ID) + + for k, v := range r.SysValues { + if v == "" { + continue + } + + switch k { + case "createdAt": + // @todo set time + rec.CreatedAt = time.Now() + + case "updatedAt": + // @todo set time + rec.UpdatedAt = nil + + case "deletedAt": + // @todo set time + rec.DeletedAt = nil + } + } + + rvs := make(types.RecordValueSet, 0, len(r.Values)) + for k, v := range r.Values { + rv := &types.RecordValue{ + RecordID: rec.ID, + Name: k, + Value: v, + Updated: true, + } + + rvs = append(rvs, rv) + } + + rec.Values = rvSanitizer.Run(mod, rvs) + + if !ectx.partial && !ectx.exists { + err = store.CreateComposeRecord(ctx, s, mod, rec) + if err != nil { + return err + } + } + + return nil + }) +} diff --git a/pkg/envoy/tmp/compose_record_set.go b/pkg/envoy/tmp/compose_record_set.go deleted file mode 100644 index 124d471de..000000000 --- a/pkg/envoy/tmp/compose_record_set.go +++ /dev/null @@ -1,13 +0,0 @@ -package tmp - -import ( - "context" - - "github.com/cortezaproject/corteza-server/pkg/envoy/resource" - "github.com/cortezaproject/corteza-server/store" -) - -func encodeComposeRecordSet(ctx context.Context, s store.Storer, rec *resource.ComposeRecordSet, rm resMap) (uint64, error) { - // @todo... - return 0, nil -} diff --git a/pkg/envoy/tmp/compose_record_set_preproc.go b/pkg/envoy/tmp/compose_record_set_preproc.go deleted file mode 100644 index b7de03b29..000000000 --- a/pkg/envoy/tmp/compose_record_set_preproc.go +++ /dev/null @@ -1,89 +0,0 @@ -package tmp - -import ( - "context" - - "github.com/cortezaproject/corteza-server/pkg/envoy" - "github.com/cortezaproject/corteza-server/pkg/envoy/resource" - "github.com/cortezaproject/corteza-server/store" -) - -type ( - composeRecordSetPreproc struct { - is *importState - s store.Storer - } -) - -func NewComposeRecordSetPreproc(is *importState, s store.Storer) *composeRecordSetPreproc { - return &composeRecordSetPreproc{ - is: is, - s: s, - } -} - -func (p *composeRecordSetPreproc) Process(ctx context.Context, state *envoy.ExecState) error { - // @todo can we/should we have the same pattern as with decoder's CanDecode? - res, is := state.Res.(*resource.ComposeRecordSet) - if !is { - return nil - } - - nsID, err := p.namespace(ctx, res, state) - if err != nil { - return err - } else if nsID <= 0 { - // If the namespace doesn't exist, no underlying resource is able to exist. - // @todo generate an error set to show as warnings? - return nil - } - - modID, err := p.module(ctx, res, state) - if err != nil { - return err - } else if modID <= 0 { - // If the module doesn't exist, no underlying resource is able to exist. - // @todo generate an error set to show as warnings? - return nil - } - - // @todo existing records, related records - - return nil -} - -func (p *composeRecordSetPreproc) namespace(ctx context.Context, res *resource.ComposeRecordSet, state *envoy.ExecState) (nsID uint64, err error) { - nss := filterComposeNamespaceResources(state.ParentResources) - if len(nss) > 0 { - nsID = p.is.Existint(nss[0]) - } else { - ns, idd, err := findMissingComposeNamespace(ctx, p.s, state.MissingDeps) - if err != nil { - return 0, err - } - if ns != nil { - nsID = ns.ID - p.is.AddRefMapping(res, "compose:namespace", nsID, idd.StringSlice()...) - } - } - - return nsID, nil -} - -func (p *composeRecordSetPreproc) module(ctx context.Context, res *resource.ComposeRecordSet, state *envoy.ExecState) (modID uint64, err error) { - modd := filterComposeModuleResources(state.ParentResources) - if len(modd) > 0 { - modID = p.is.Existint(modd[0]) - } else { - mod, idd, err := findMissingComposeModule(ctx, p.s, state.MissingDeps) - if err != nil { - return 0, err - } - if mod != nil { - modID = mod.ID - p.is.AddRefMapping(res, "compose:module", modID, idd.StringSlice()...) - } - } - - return modID, nil -} diff --git a/pkg/envoy/yaml/compose_record.go b/pkg/envoy/yaml/compose_record.go index 051770663..5e0574a05 100644 --- a/pkg/envoy/yaml/compose_record.go +++ b/pkg/envoy/yaml/compose_record.go @@ -4,14 +4,15 @@ import ( "fmt" "github.com/cortezaproject/corteza-server/compose/types" - "github.com/cortezaproject/corteza-server/pkg/envoy" "github.com/cortezaproject/corteza-server/pkg/envoy/resource" "gopkg.in/yaml.v3" ) type ( composeRecord struct { - res *types.Record `yaml:",inline"` + // res *types.Record `yaml:",inline"` + values map[string]string + sysValues map[string]string refModule string refNamespace string @@ -76,13 +77,50 @@ func (wset *composeRecordSet) UnmarshalYAML(n *yaml.Node) error { func (wset composeRecordSet) MarshalEnvoy() ([]resource.Interface, error) { nn := make([]resource.Interface, 0, len(wset)) + type ( + rw struct { + rr resource.ComposeRecordRawSet + nsRef string + modRef string + } + ) + + // moduleRef to values set + recMap := make(map[string]*rw) + for _, res := range wset { - if tmp, err := res.MarshalEnvoy(); err != nil { - return nil, err - } else { - nn = append(nn, tmp...) + if recMap[res.refModule] == nil { + recMap[res.refModule] = &rw{ + rr: make(resource.ComposeRecordRawSet, 0, 10), + nsRef: res.refNamespace, + modRef: res.refModule, + } } + r := &resource.ComposeRecordRaw{ + // @todo change this probably + ID: res.values["id"], + + Values: res.values, + RefUser: res.refUser, + SysValues: res.sysValues, + } + recMap[res.refModule].rr = append(recMap[res.refModule].rr, r) + } + + for _, w := range recMap { + walker := func(f func(r *resource.ComposeRecordRaw) error) error { + for _, r := range w.rr { + err := f(r) + if err != nil { + return err + } + } + return nil + } + + n := resource.NewComposeRecordSet(walker, w.nsRef, w.modRef) + nn = append(nn, n) } return nn, nil @@ -100,18 +138,15 @@ func (wset composeRecordSet) setNamespaceRef(ref string) error { return nil } -func (wrap composeRecord) MarshalEnvoy() ([]resource.Interface, error) { - return envoy.CollectNodes( - resource.NewComposeRecord(wrap.res, wrap.refNamespace, wrap.refModule, wrap.refUser), - ) -} - func (wrap *composeRecord) UnmarshalYAML(n *yaml.Node) (err error) { if wrap.refUser == nil { wrap.refUser = make(map[string]string) } - if wrap.res == nil { - wrap.res = &types.Record{} + if wrap.values == nil { + wrap.values = make(map[string]string) + } + if wrap.sysValues == nil { + wrap.sysValues = make(map[string]string) } // @todo enable when records are ready for RBAC @@ -126,20 +161,17 @@ func (wrap *composeRecord) UnmarshalYAML(n *yaml.Node) (err error) { case "values": // Use aux structure to decode record values into RVS - aux := composeRecordValues{} - if err := v.Decode(&aux); err != nil { + if err := v.Decode(&wrap.values); err != nil { return err } - - wrap.res.Values = aux.rvs return nil case "createdAt": - return v.Decode(&wrap.res.CreatedAt) + return v.Decode(wrap.sysValues["createdAt"]) case "updatedAt": - return v.Decode(&wrap.res.UpdatedAt) + return v.Decode(wrap.sysValues["updatedAt"]) case "deletedAt": - return v.Decode(&wrap.res.DeletedAt) + return v.Decode(wrap.sysValues["deletedAt"]) case "createdBy": return v.Decode(wrap.refUser["createdBy"]) case "updatedBy": @@ -154,40 +186,3 @@ func (wrap *composeRecord) UnmarshalYAML(n *yaml.Node) (err error) { return nil }) } - -// UnmarshalYAML resolves record values definitioons -// -// { : ... , .... } -// { : [ ], .... } -func (wset *composeRecordValues) UnmarshalYAML(n *yaml.Node) error { - wset.rvs = types.RecordValueSet{} - - return eachMap(n, func(k, v *yaml.Node) error { - if isKind(v, yaml.ScalarNode) { - wset.rvs = append(wset.rvs, &types.RecordValue{ - Name: k.Value, - Value: v.Value, - }) - - return nil - } - - if isKind(v, yaml.SequenceNode) { - for i := range v.Content { - if isKind(v, yaml.ScalarNode) { - return nodeErr(n, "expecting scalar node for record value") - } - - wset.rvs = append(wset.rvs, &types.RecordValue{ - Name: k.Value, - Value: v.Content[i].Value, - Place: uint(i), - }) - } - - return nil - } - - return nodeErr(n, "expecting scalar or sequence node for record value") - }) -}