DeforaNetworks/corteza
DeforaNetworks/corteza
3
0
Fork 0
Code

Rework how records are handled

Browse Source
This commit is contained in:
Tomaž Jerman 2020-11-05 20:23:59 +01:00
parent 29e1fa369a
commit 1e2ee1bb00
5 changed files with 236 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
pkg/envoy/resource/compose_record.go
View File

@ -1,36 +1,46 @@
package resource
import (
"fmt"
"github.com/cortezaproject/corteza-server/compose/types"
)
const (
COMPOSE_RECORD_RESOURCE_TYPE = "ComposeRecordSet"
)
type (
ComposeRecordRaw struct {
ID string
Values map[string]string
SysValues map[string]string
RefUser map[string]string
}
ComposeRecordRawSet []*ComposeRecordRaw
crsWalker func(f func(r *ComposeRecordRaw) error) error
ComposeRecord struct {
*base
Res *types.Record
nsRef string
modRef string
userRef map[string]string
// Res *types.Record
Walker crsWalker
NsRef *Ref
ModRef *Ref
ModFields types.ModuleFieldSet
UserRef map[string]string
}
)
func NewComposeRecord(res *types.Record, nsRef, modRef string, userRef map[string]string) *ComposeRecord {
func NewComposeRecordSet(w crsWalker, nsRef, modRef string) *ComposeRecord {
r := &ComposeRecord{base: &base{}}
r.SetResourceType(COMPOSE_RECORD_RESOURCE_TYPE)
r.Res = res
r.nsRef = nsRef
r.modRef = modRef
r.Walker = w
r.AddIdentifier(identifiers(res.ID)...)
r.AddIdentifier(identifiers(modRef)...)
r.AddRef(COMPOSE_NAMESPACE_RESOURCE_TYPE, nsRef)
r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, modRef)
// for _, u := range userRef {
// r.AddRef(USER_RESOURCE_TYPE, u)
// }
r.NsRef = r.AddRef(COMPOSE_NAMESPACE_RESOURCE_TYPE, nsRef)
r.ModRef = r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, modRef)
return r
}
@ -38,9 +48,5 @@ func NewComposeRecord(res *types.Record, nsRef, modRef string, userRef map[strin
func (m *ComposeRecord) SearchQuery() types.RecordFilter {
f := types.RecordFilter{}
if m.Res.ID > 0 {
f.Query = fmt.Sprintf("recordID=%d", m.Res.ID)
}
return f
}

156
pkg/envoy/tmp/compose_record.go Normal file
View File

@ -0,0 +1,156 @@
package tmp
import (
"context"
"errors"
"time"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
"github.com/cortezaproject/corteza-server/store"
)
type (
composeRecordPreproc struct {
es *encoderState
s store.Storer
}
)
func NewComposeRecordPreproc(is *encoderState, s store.Storer) *composeRecordPreproc {
return &composeRecordPreproc{
es: is,
s: s,
}
}
func (p *composeRecordPreproc) Process(ctx context.Context, state *envoy.ExecState) error {
res, is := state.Res.(*resource.ComposeRecord)
if !is {
return nil
}
// Get relate namespace
ns, err := findNamespace(ctx, p.s, state.ParentResources, res.NsRef.Identifiers)
if err != nil {
return err
}
if ns == nil {
return errors.New("@todo couldn't resolve namespace")
}
p.es.Set(res, res.NsRef.ResourceType, ns.ID, res.NsRef.Identifiers.StringSlice()...)
// Get relate namespace
mod, err := findModule(ctx, p.s, state.ParentResources, res.ModRef.Identifiers)
if err != nil {
return err
}
if mod == nil {
return errors.New("@todo couldn't resolve module")
}
p.es.Set(res, res.ModRef.ResourceType, mod.ID, res.ModRef.Identifiers.StringSlice()...)
// @todo existing records
//
// Hookup with labels to determine existing records
ff, err := loadComposeModuleFields(ctx, p.s, mod)
if err != nil {
return err
}
res.ModFields = ff
for _, f := range ff {
switch f.Kind {
case "Record":
refM := f.Options.String("module")
if refM != "" && refM != "0" {
// Make a reference with that module's records
res.AddRef(resource.COMPOSE_RECORD_RESOURCE_TYPE, refM)
}
}
}
return nil
}
func encodeComposeRecord(ctx context.Context, ectx *encodingContext, s store.Storer, state resRefs, res *resource.ComposeRecord) (resRefs, error) {
var err error
rState := make(resRefs)
// Namespace...
nsID := uint64(0)
for _, v := range state[resource.COMPOSE_NAMESPACE_RESOURCE_TYPE] {
nsID = v
break
}
// Module...
modID := uint64(0)
for _, v := range state[resource.COMPOSE_MODULE_RESOURCE_TYPE] {
modID = v
break
}
mod := &types.Module{
NamespaceID: nsID,
ID: modID,
Fields: res.ModFields,
}
return rState, res.Walker(func(r *resource.ComposeRecordRaw) error {
rec := &types.Record{
NamespaceID: nsID,
ModuleID: modID,
}
rec.ID = state[res.ResourceType()][r.ID]
if rec.ID <= 0 {
rec.ID = nextID()
}
rState.Set(resource.COMPOSE_RECORD_RESOURCE_TYPE, rec.ID, r.ID)
for k, v := range r.SysValues {
if v == "" {
continue
}
switch k {
case "createdAt":
// @todo set time
rec.CreatedAt = time.Now()
case "updatedAt":
// @todo set time
rec.UpdatedAt = nil
case "deletedAt":
// @todo set time
rec.DeletedAt = nil
}
}
rvs := make(types.RecordValueSet, 0, len(r.Values))
for k, v := range r.Values {
rv := &types.RecordValue{
RecordID: rec.ID,
Name: k,
Value: v,
Updated: true,
}
rvs = append(rvs, rv)
}
rec.Values = rvSanitizer.Run(mod, rvs)
if !ectx.partial && !ectx.exists {
err = store.CreateComposeRecord(ctx, s, mod, rec)
if err != nil {
return err
}
}
return nil
})
}

13
pkg/envoy/tmp/compose_record_set.go
View File

@ -1,13 +0,0 @@
package tmp
import (
"context"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
"github.com/cortezaproject/corteza-server/store"
)
func encodeComposeRecordSet(ctx context.Context, s store.Storer, rec *resource.ComposeRecordSet, rm resMap) (uint64, error) {
// @todo...
return 0, nil
}

89
pkg/envoy/tmp/compose_record_set_preproc.go
View File

@ -1,89 +0,0 @@
package tmp
import (
"context"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
"github.com/cortezaproject/corteza-server/store"
)
type (
composeRecordSetPreproc struct {
is *importState
s store.Storer
}
)
func NewComposeRecordSetPreproc(is *importState, s store.Storer) *composeRecordSetPreproc {
return &composeRecordSetPreproc{
is: is,
s: s,
}
}
func (p *composeRecordSetPreproc) Process(ctx context.Context, state *envoy.ExecState) error {
// @todo can we/should we have the same pattern as with decoder's CanDecode?
res, is := state.Res.(*resource.ComposeRecordSet)
if !is {
return nil
}
nsID, err := p.namespace(ctx, res, state)
if err != nil {
return err
} else if nsID <= 0 {
// If the namespace doesn't exist, no underlying resource is able to exist.
// @todo generate an error set to show as warnings?
return nil
}
modID, err := p.module(ctx, res, state)
if err != nil {
return err
} else if modID <= 0 {
// If the module doesn't exist, no underlying resource is able to exist.
// @todo generate an error set to show as warnings?
return nil
}
// @todo existing records, related records
return nil
}
func (p *composeRecordSetPreproc) namespace(ctx context.Context, res *resource.ComposeRecordSet, state *envoy.ExecState) (nsID uint64, err error) {
nss := filterComposeNamespaceResources(state.ParentResources)
if len(nss) > 0 {
nsID = p.is.Existint(nss[0])
} else {
ns, idd, err := findMissingComposeNamespace(ctx, p.s, state.MissingDeps)
if err != nil {
return 0, err
}
if ns != nil {
nsID = ns.ID
p.is.AddRefMapping(res, "compose:namespace", nsID, idd.StringSlice()...)
}
}
return nsID, nil
}
func (p *composeRecordSetPreproc) module(ctx context.Context, res *resource.ComposeRecordSet, state *envoy.ExecState) (modID uint64, err error) {
modd := filterComposeModuleResources(state.ParentResources)
if len(modd) > 0 {
modID = p.is.Existint(modd[0])
} else {
mod, idd, err := findMissingComposeModule(ctx, p.s, state.MissingDeps)
if err != nil {
return 0, err
}
if mod != nil {
modID = mod.ID
p.is.AddRefMapping(res, "compose:module", modID, idd.StringSlice()...)
}
}
return modID, nil
}

111
pkg/envoy/yaml/compose_record.go
View File

@ -4,14 +4,15 @@ import (
"fmt"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
"gopkg.in/yaml.v3"
)
type (
composeRecord struct {
res *types.Record `yaml:",inline"`
// res *types.Record `yaml:",inline"`
values map[string]string
sysValues map[string]string
refModule string
refNamespace string
@ -76,13 +77,50 @@ func (wset *composeRecordSet) UnmarshalYAML(n *yaml.Node) error {
func (wset composeRecordSet) MarshalEnvoy() ([]resource.Interface, error) {
nn := make([]resource.Interface, 0, len(wset))
type (
rw struct {
rr resource.ComposeRecordRawSet
nsRef string
modRef string
}
)
// moduleRef to values set
recMap := make(map[string]*rw)
for _, res := range wset {
if tmp, err := res.MarshalEnvoy(); err != nil {
return nil, err
} else {
nn = append(nn, tmp...)
if recMap[res.refModule] == nil {
recMap[res.refModule] = &rw{
rr: make(resource.ComposeRecordRawSet, 0, 10),
nsRef: res.refNamespace,
modRef: res.refModule,
}
}
r := &resource.ComposeRecordRaw{
// @todo change this probably
ID: res.values["id"],
Values: res.values,
RefUser: res.refUser,
SysValues: res.sysValues,
}
recMap[res.refModule].rr = append(recMap[res.refModule].rr, r)
}
for _, w := range recMap {
walker := func(f func(r *resource.ComposeRecordRaw) error) error {
for _, r := range w.rr {
err := f(r)
if err != nil {
return err
}
}
return nil
}
n := resource.NewComposeRecordSet(walker, w.nsRef, w.modRef)
nn = append(nn, n)
}
return nn, nil
@ -100,18 +138,15 @@ func (wset composeRecordSet) setNamespaceRef(ref string) error {
return nil
}
func (wrap composeRecord) MarshalEnvoy() ([]resource.Interface, error) {
return envoy.CollectNodes(
resource.NewComposeRecord(wrap.res, wrap.refNamespace, wrap.refModule, wrap.refUser),
)
}
func (wrap *composeRecord) UnmarshalYAML(n *yaml.Node) (err error) {
if wrap.refUser == nil {
wrap.refUser = make(map[string]string)
}
if wrap.res == nil {
wrap.res = &types.Record{}
if wrap.values == nil {
wrap.values = make(map[string]string)
}
if wrap.sysValues == nil {
wrap.sysValues = make(map[string]string)
}
// @todo enable when records are ready for RBAC
@ -126,20 +161,17 @@ func (wrap *composeRecord) UnmarshalYAML(n *yaml.Node) (err error) {
case "values":
// Use aux structure to decode record values into RVS
aux := composeRecordValues{}
if err := v.Decode(&aux); err != nil {
if err := v.Decode(&wrap.values); err != nil {
return err
}
wrap.res.Values = aux.rvs
return nil
case "createdAt":
return v.Decode(&wrap.res.CreatedAt)
return v.Decode(wrap.sysValues["createdAt"])
case "updatedAt":
return v.Decode(&wrap.res.UpdatedAt)
return v.Decode(wrap.sysValues["updatedAt"])
case "deletedAt":
return v.Decode(&wrap.res.DeletedAt)
return v.Decode(wrap.sysValues["deletedAt"])
case "createdBy":
return v.Decode(wrap.refUser["createdBy"])
case "updatedBy":
@ -154,40 +186,3 @@ func (wrap *composeRecord) UnmarshalYAML(n *yaml.Node) (err error) {
return nil
})
}
// UnmarshalYAML resolves record values definitioons
//
// { <field name>: ... <scalar value>, .... }
// { <field name>: [ <scalar value> ], .... }
func (wset *composeRecordValues) UnmarshalYAML(n *yaml.Node) error {
wset.rvs = types.RecordValueSet{}
return eachMap(n, func(k, v *yaml.Node) error {
if isKind(v, yaml.ScalarNode) {
wset.rvs = append(wset.rvs, &types.RecordValue{
Name: k.Value,
Value: v.Value,
})
return nil
}
if isKind(v, yaml.SequenceNode) {
for i := range v.Content {
if isKind(v, yaml.ScalarNode) {
return nodeErr(n, "expecting scalar node for record value")
}
wset.rvs = append(wset.rvs, &types.RecordValue{
Name: k.Value,
Value: v.Content[i].Value,
Place: uint(i),
})
}
return nil
}
return nodeErr(n, "expecting scalar or sequence node for record value")
})
}