DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Implement base pre-processing & store resource handling

Browse Source
This commit is contained in:
Tomaž Jerman
2020-11-27 11:19:15 +01:00
parent 1f6f4719eb
commit aca237ea4e
11 changed files with 593 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/envoy/tmp/compose_module.go
+60
View File
@@ -0,0 +1,60 @@
package tmp
import (
"context"
"fmt"
"time"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
"github.com/cortezaproject/corteza-server/store"
)
func moduleFilterFromGeneric(gf genericFilter) types.ModuleFilter {
f := types.ModuleFilter{
Name: gf.Name,
Handle: gf.Ref,
}
if gf.ID > 0 {
f.Query = fmt.Sprintf("moduleID=%d", gf.ID)
}
return f
}
func encodeComposeModule(ctx context.Context, s store.Storer, mod *resource.ComposeModule, rm resMap) (uint64, error) {
var (
res = mod.Res
)
res.ID = nextID()
if res.CreatedAt.IsZero() {
res.CreatedAt = time.Now()
}
// Namespace...
// A module can exist under a single namespace, so this is good enough for now
nsID := uint64(0)
for _, v := range rm["compose:namespace"] {
nsID = v
break
}
res.NamespaceID = nsID
// @todo fields
return res.ID, nil
}
func loadComposeModule(ctx context.Context, s store.Storer, f types.ModuleFilter) (*types.Module, error) {
mdd, f, err := store.SearchComposeModules(ctx, s, f)
if err != nil {
return nil, err
}
if len(mdd) > 0 {
return mdd[0], nil
}
return nil, nil
}
pkg/envoy/tmp/compose_module_preproc.go
+90
View File
@@ -0,0 +1,90 @@
package tmp
import (
"context"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
"github.com/cortezaproject/corteza-server/store"
)
type (
composeModulePreproc struct {
is *importState
s store.Storer
}
)
func NewComposeModulePreproc(is *importState, s store.Storer) envoy.Processor {
return &composeModulePreproc{
is: is,
s: s,
}
}
func (p *composeModulePreproc) Process(ctx context.Context, state *envoy.ExecState) error {
// @todo can we/should we have the same pattern as with decoder's CanDecode?
res, is := state.Res.(*resource.ComposeModule)
if !is {
return nil
}
nsID, err := p.namespace(ctx, res, state)
if err != nil {
return err
} else if nsID <= 0 {
// If the namespace doesn't exist, no underlying resource is able to exist.
// @todo generate an error set to show as warnings?
return nil
}
// Check if the current module exits
f := res.SearchQuery()
f.NamespaceID = nsID
mod, err := loadComposeModule(ctx, p.s, f)
if err != nil {
return err
}
if mod != nil {
p.is.Existint(res)
p.is.AddRefMapping(res, res.ResourceType(), mod.ID, res.Identifiers().StringSlice()...)
}
// Go over missing deps and handle those
for _, m := range state.MissingDeps {
switch m.ResourceType {
// @todo change this string to a better thing...
case "compose:module":
f := moduleFilterFromGeneric(GetGenericFilter(m.Identifiers))
f.NamespaceID = nsID
mod, err := loadComposeModule(ctx, p.s, f)
if err != nil {
return err
} else if mod == nil {
continue
}
p.is.AddRefMapping(res, "compose:module", mod.ID, m.Identifiers.StringSlice()...)
}
}
return nil
}
func (p *composeModulePreproc) namespace(ctx context.Context, res *resource.ComposeModule, state *envoy.ExecState) (nsID uint64, err error) {
nss := filterComposeNamespaceResources(state.ParentResources)
if len(nss) > 0 {
nsID = p.is.Existint(nss[0])
} else {
ns, idd, err := findMissingComposeNamespace(ctx, p.s, state.MissingDeps)
if err != nil {
return 0, err
}
if ns != nil {
nsID = ns.ID
p.is.AddRefMapping(res, "compose:namespace", nsID, idd.StringSlice()...)
}
}
return nsID, nil
}
pkg/envoy/tmp/compose_namespace.go
+54
View File
@@ -0,0 +1,54 @@
package tmp
import (
"context"
"fmt"
"time"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
"github.com/cortezaproject/corteza-server/store"
)
func namespaceFilterFromGeneric(gf genericFilter) types.NamespaceFilter {
f := types.NamespaceFilter{
Name: gf.Name,
Slug: gf.Ref,
}
if gf.ID > 0 {
f.Query = fmt.Sprintf("namespaceID=%d", gf.ID)
}
return f
}
func encodeComposeNamespace(ctx context.Context, s store.Storer, ns *resource.ComposeNamespace) (uint64, error) {
var (
res = ns.Res
)
res.ID = nextID()
if res.CreatedAt.IsZero() {
res.CreatedAt = time.Now()
}
// err := store.CreateComposeNamespace(ctx, s, res)
// if err != nil {
// return nil, err
// }
return res.ID, nil
}
func loadComposeNamespace(ctx context.Context, s store.Storer, f types.NamespaceFilter) (*types.Namespace, error) {
nss, f, err := store.SearchComposeNamespaces(ctx, s, f)
if err != nil {
return nil, err
}
if len(nss) > 0 {
return nss[0], nil
}
return nil, nil
}
pkg/envoy/tmp/compose_namespace_preproc.go
+46
View File
@@ -0,0 +1,46 @@
package tmp
import (
"context"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
"github.com/cortezaproject/corteza-server/store"
)
type (
composeNamespacePreproc struct {
is *importState
s store.Storer
}
)
func NewComposeNamespacePreproc(is *importState, s store.Storer) envoy.Processor {
return &composeNamespacePreproc{
is: is,
s: s,
}
}
func (p *composeNamespacePreproc) Process(ctx context.Context, state *envoy.ExecState) error {
// @todo can we/should we have the same pattern as with decoder's CanDecode?
res, is := state.Res.(*resource.ComposeNamespace)
if !is {
return nil
}
// Check if the given namespace exists
// Namespaces are top-level resources, so they don't depend on anything;
// no other checks are needed
ns, err := loadComposeNamespace(ctx, p.s, res.SearchQuery())
if err != nil {
return err
}
if ns != nil {
p.is.AddExisting(res, ns.ID)
}
return nil
}
pkg/envoy/tmp/compose_record_set.go
+13
View File
@@ -0,0 +1,13 @@
package tmp
import (
"context"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
"github.com/cortezaproject/corteza-server/store"
)
func encodeComposeRecordSet(ctx context.Context, s store.Storer, rec *resource.ComposeRecordSet, rm resMap) (uint64, error) {
// @todo...
return 0, nil
}
pkg/envoy/tmp/compose_record_set_preproc.go
+89
View File
@@ -0,0 +1,89 @@
package tmp
import (
"context"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
"github.com/cortezaproject/corteza-server/store"
)
type (
composeRecordSetPreproc struct {
is *importState
s store.Storer
}
)
func NewComposeRecordSetPreproc(is *importState, s store.Storer) *composeRecordSetPreproc {
return &composeRecordSetPreproc{
is: is,
s: s,
}
}
func (p *composeRecordSetPreproc) Process(ctx context.Context, state *envoy.ExecState) error {
// @todo can we/should we have the same pattern as with decoder's CanDecode?
res, is := state.Res.(*resource.ComposeRecordSet)
if !is {
return nil
}
nsID, err := p.namespace(ctx, res, state)
if err != nil {
return err
} else if nsID <= 0 {
// If the namespace doesn't exist, no underlying resource is able to exist.
// @todo generate an error set to show as warnings?
return nil
}
modID, err := p.module(ctx, res, state)
if err != nil {
return err
} else if modID <= 0 {
// If the module doesn't exist, no underlying resource is able to exist.
// @todo generate an error set to show as warnings?
return nil
}
// @todo existing records, related records
return nil
}
func (p *composeRecordSetPreproc) namespace(ctx context.Context, res *resource.ComposeRecordSet, state *envoy.ExecState) (nsID uint64, err error) {
nss := filterComposeNamespaceResources(state.ParentResources)
if len(nss) > 0 {
nsID = p.is.Existint(nss[0])
} else {
ns, idd, err := findMissingComposeNamespace(ctx, p.s, state.MissingDeps)
if err != nil {
return 0, err
}
if ns != nil {
nsID = ns.ID
p.is.AddRefMapping(res, "compose:namespace", nsID, idd.StringSlice()...)
}
}
return nsID, nil
}
func (p *composeRecordSetPreproc) module(ctx context.Context, res *resource.ComposeRecordSet, state *envoy.ExecState) (modID uint64, err error) {
modd := filterComposeModuleResources(state.ParentResources)
if len(modd) > 0 {
modID = p.is.Existint(modd[0])
} else {
mod, idd, err := findMissingComposeModule(ctx, p.s, state.MissingDeps)
if err != nil {
return 0, err
}
if mod != nil {
modID = mod.ID
p.is.AddRefMapping(res, "compose:module", modID, idd.StringSlice()...)
}
}
return modID, nil
}
pkg/envoy/tmp/encoder.go
+64
View File
@@ -0,0 +1,64 @@
package tmp
import (
"context"
"github.com/cortezaproject/corteza-server/compose/service/values"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
"github.com/cortezaproject/corteza-server/store"
)
type (
StoreEncoder struct {
s store.Storer
is *importState
}
)
var (
rvSanitizer = values.Sanitizer()
rvValidator = values.Validator()
)
func NewStoreEncoder(s store.Storer, is *importState) *StoreEncoder {
return &StoreEncoder{
s: s,
is: is,
}
}
func (se *StoreEncoder) Encode(ctx context.Context, ess ...*envoy.ExecState) error {
return store.Tx(ctx, se.s, func(ctx context.Context, s store.Storer) (err error) {
var rID uint64
for _, es := range ess {
// @todo what should we do with existing resources?
// How should we do diffing?
rID = se.is.Existint(es.Res)
if rID <= 0 {
switch res := es.Res.(type) {
case *resource.ComposeNamespace:
rID, err = encodeComposeNamespace(ctx, s, res)
case *resource.ComposeModule:
rID, err = encodeComposeModule(ctx, s, res, se.is.state[res])
case *resource.ComposeRecordSet:
_, err = encodeComposeRecordSet(ctx, s, res, se.is.state[res])
}
if err != nil {
return err
}
}
for _, dr := range es.DepResources {
se.is.AddRefMapping(dr, es.Res.ResourceType(), rID, es.Res.Identifiers().StringSlice()...)
}
}
return nil
})
}
pkg/envoy/tmp/import_state.go
+46
View File
@@ -0,0 +1,46 @@
package tmp
import "github.com/cortezaproject/corteza-server/pkg/envoy/resource"
type (
importState struct {
state resNodeMap
// We'll use this to keep track of the existing resources
existing map[resource.Interface]uint64
}
// ref: internal ID
refMap map[string]uint64
// resource type: refMap
resMap map[string]refMap
resNodeMap map[resource.Interface]resMap
)
func NewImportState() *importState {
return &importState{
state: make(resNodeMap),
existing: make(map[resource.Interface]uint64),
}
}
func (s *importState) AddRefMapping(r resource.Interface, res string, id uint64, refs ...string) {
if s.state[r] == nil {
s.state[r] = make(resMap)
}
if s.state[r][res] == nil {
s.state[r][res] = make(refMap)
}
for _, ref := range refs {
s.state[r][res][ref] = id
}
}
func (s *importState) Existint(r resource.Interface) uint64 {
return s.existing[r]
}
func (s *importState) AddExisting(r resource.Interface, resID uint64) {
s.existing[r] = resID
}
pkg/envoy/tmp/preproc.go
+15
View File
@@ -0,0 +1,15 @@
package tmp
import (
"github.com/cortezaproject/corteza-server/pkg/envoy"
"github.com/cortezaproject/corteza-server/store"
)
func CollectPreproc(is *importState, s store.Storer) []envoy.Processor {
return []envoy.Processor{
NewComposeModulePreproc(is, s),
NewComposeNamespacePreproc(is, s),
NewComposeRecordSetPreproc(is, s),
// ...
}
}
pkg/envoy/tmp/preproc_util.go
+67
View File
@@ -0,0 +1,67 @@
package tmp
import (
"context"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
"github.com/cortezaproject/corteza-server/store"
)
func filterComposeNamespaceResources(rr []resource.Interface) []*resource.ComposeNamespace {
nn := make([]*resource.ComposeNamespace, 0, len(rr))
for _, r := range rr {
switch n := r.(type) {
case *resource.ComposeNamespace:
nn = append(nn, n)
}
}
return nn
}
func filterComposeModuleResources(rr []resource.Interface) []*resource.ComposeModule {
nn := make([]*resource.ComposeModule, 0, len(rr))
for _, r := range rr {
switch n := r.(type) {
case *resource.ComposeModule:
nn = append(nn, n)
}
}
return nn
}
func findMissingComposeNamespace(ctx context.Context, s store.Storer, missing resource.RefSet) (*types.Namespace, resource.Identifiers, error) {
// Check the store if we can find it
nsDd := missing.FilterByResourceType("compose:namespace")
if len(nsDd) > 0 {
nsD := nsDd[0]
ns, err := loadComposeNamespace(ctx, s, namespaceFilterFromGeneric(GetGenericFilter(nsD.Identifiers)))
if err != nil {
return nil, nil, err
}
if ns != nil {
return ns, nsD.Identifiers, nil
}
}
return nil, nil, nil
}
func findMissingComposeModule(ctx context.Context, s store.Storer, missing resource.RefSet) (*types.Module, resource.Identifiers, error) {
// Check the store if we can find it
modDd := missing.FilterByResourceType("compose:module")
if len(modDd) > 0 {
modD := modDd[0]
mod, err := loadComposeModule(ctx, s, moduleFilterFromGeneric(GetGenericFilter(modD.Identifiers)))
if err != nil {
return nil, nil, err
}
if mod != nil {
return mod, modD.Identifiers, nil
}
}
return nil, nil, nil
}
pkg/envoy/tmp/util.go
+49
View File
@@ -0,0 +1,49 @@
package tmp
import (
"regexp"
"strconv"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
"github.com/cortezaproject/corteza-server/pkg/id"
)
type (
genericFilter struct {
ID uint64
Ref string
Name string
}
)
var (
refy = regexp.MustCompile(`^[1-9](\d*)$`)
handy = regexp.MustCompile(`^[A-Za-z][0-9A-Za-z_\-.]*[A-Za-z0-9]$`)
// wrapper around nextID that will aid service testing
nextID = func() uint64 {
return id.Next()
}
)
func GetGenericFilter(ii resource.Identifiers) genericFilter {
f := genericFilter{}
for id := range ii {
if id == "" {
continue
}
if refy.MatchString(id) {
id, err := strconv.ParseUint(id, 10, 64)
if err != nil {
continue
}
f.ID = id
} else if handy.MatchString(id) {
f.Ref = id
} else {
f.Name = id
}
}
return f
}