diff --git a/pkg/envoy/tmp/compose_module.go b/pkg/envoy/tmp/compose_module.go new file mode 100644 index 000000000..c3b039dcd --- /dev/null +++ b/pkg/envoy/tmp/compose_module.go @@ -0,0 +1,60 @@ +package tmp + +import ( + "context" + "fmt" + "time" + + "github.com/cortezaproject/corteza-server/compose/types" + "github.com/cortezaproject/corteza-server/pkg/envoy/resource" + "github.com/cortezaproject/corteza-server/store" +) + +func moduleFilterFromGeneric(gf genericFilter) types.ModuleFilter { + f := types.ModuleFilter{ + Name: gf.Name, + Handle: gf.Ref, + } + if gf.ID > 0 { + f.Query = fmt.Sprintf("moduleID=%d", gf.ID) + } + + return f +} + +func encodeComposeModule(ctx context.Context, s store.Storer, mod *resource.ComposeModule, rm resMap) (uint64, error) { + var ( + res = mod.Res + ) + + res.ID = nextID() + if res.CreatedAt.IsZero() { + res.CreatedAt = time.Now() + } + + // Namespace... + // A module can exist under a single namespace, so this is good enough for now + nsID := uint64(0) + for _, v := range rm["compose:namespace"] { + nsID = v + break + } + res.NamespaceID = nsID + + // @todo fields + + return res.ID, nil +} + +func loadComposeModule(ctx context.Context, s store.Storer, f types.ModuleFilter) (*types.Module, error) { + mdd, f, err := store.SearchComposeModules(ctx, s, f) + if err != nil { + return nil, err + } + + if len(mdd) > 0 { + return mdd[0], nil + } + + return nil, nil +} diff --git a/pkg/envoy/tmp/compose_module_preproc.go b/pkg/envoy/tmp/compose_module_preproc.go new file mode 100644 index 000000000..8ef7ff1a5 --- /dev/null +++ b/pkg/envoy/tmp/compose_module_preproc.go @@ -0,0 +1,90 @@ +package tmp + +import ( + "context" + + "github.com/cortezaproject/corteza-server/pkg/envoy" + "github.com/cortezaproject/corteza-server/pkg/envoy/resource" + "github.com/cortezaproject/corteza-server/store" +) + +type ( + composeModulePreproc struct { + is *importState + s store.Storer + } +) + +func NewComposeModulePreproc(is *importState, s store.Storer) envoy.Processor { + return &composeModulePreproc{ + is: is, + s: s, + } +} + +func (p *composeModulePreproc) Process(ctx context.Context, state *envoy.ExecState) error { + // @todo can we/should we have the same pattern as with decoder's CanDecode? + res, is := state.Res.(*resource.ComposeModule) + if !is { + return nil + } + + nsID, err := p.namespace(ctx, res, state) + if err != nil { + return err + } else if nsID <= 0 { + // If the namespace doesn't exist, no underlying resource is able to exist. + // @todo generate an error set to show as warnings? + return nil + } + + // Check if the current module exits + f := res.SearchQuery() + f.NamespaceID = nsID + mod, err := loadComposeModule(ctx, p.s, f) + if err != nil { + return err + } + if mod != nil { + p.is.Existint(res) + p.is.AddRefMapping(res, res.ResourceType(), mod.ID, res.Identifiers().StringSlice()...) + } + + // Go over missing deps and handle those + for _, m := range state.MissingDeps { + switch m.ResourceType { + // @todo change this string to a better thing... + case "compose:module": + f := moduleFilterFromGeneric(GetGenericFilter(m.Identifiers)) + f.NamespaceID = nsID + mod, err := loadComposeModule(ctx, p.s, f) + if err != nil { + return err + } else if mod == nil { + continue + } + + p.is.AddRefMapping(res, "compose:module", mod.ID, m.Identifiers.StringSlice()...) + } + } + + return nil +} + +func (p *composeModulePreproc) namespace(ctx context.Context, res *resource.ComposeModule, state *envoy.ExecState) (nsID uint64, err error) { + nss := filterComposeNamespaceResources(state.ParentResources) + if len(nss) > 0 { + nsID = p.is.Existint(nss[0]) + } else { + ns, idd, err := findMissingComposeNamespace(ctx, p.s, state.MissingDeps) + if err != nil { + return 0, err + } + if ns != nil { + nsID = ns.ID + p.is.AddRefMapping(res, "compose:namespace", nsID, idd.StringSlice()...) + } + } + + return nsID, nil +} diff --git a/pkg/envoy/tmp/compose_namespace.go b/pkg/envoy/tmp/compose_namespace.go new file mode 100644 index 000000000..663ae7e88 --- /dev/null +++ b/pkg/envoy/tmp/compose_namespace.go @@ -0,0 +1,54 @@ +package tmp + +import ( + "context" + "fmt" + "time" + + "github.com/cortezaproject/corteza-server/compose/types" + "github.com/cortezaproject/corteza-server/pkg/envoy/resource" + "github.com/cortezaproject/corteza-server/store" +) + +func namespaceFilterFromGeneric(gf genericFilter) types.NamespaceFilter { + f := types.NamespaceFilter{ + Name: gf.Name, + Slug: gf.Ref, + } + if gf.ID > 0 { + f.Query = fmt.Sprintf("namespaceID=%d", gf.ID) + } + + return f +} + +func encodeComposeNamespace(ctx context.Context, s store.Storer, ns *resource.ComposeNamespace) (uint64, error) { + var ( + res = ns.Res + ) + + res.ID = nextID() + if res.CreatedAt.IsZero() { + res.CreatedAt = time.Now() + } + + // err := store.CreateComposeNamespace(ctx, s, res) + // if err != nil { + // return nil, err + // } + + return res.ID, nil +} + +func loadComposeNamespace(ctx context.Context, s store.Storer, f types.NamespaceFilter) (*types.Namespace, error) { + nss, f, err := store.SearchComposeNamespaces(ctx, s, f) + if err != nil { + return nil, err + } + + if len(nss) > 0 { + return nss[0], nil + } + + return nil, nil +} diff --git a/pkg/envoy/tmp/compose_namespace_preproc.go b/pkg/envoy/tmp/compose_namespace_preproc.go new file mode 100644 index 000000000..4c2467b30 --- /dev/null +++ b/pkg/envoy/tmp/compose_namespace_preproc.go @@ -0,0 +1,46 @@ +package tmp + +import ( + "context" + + "github.com/cortezaproject/corteza-server/pkg/envoy" + "github.com/cortezaproject/corteza-server/pkg/envoy/resource" + "github.com/cortezaproject/corteza-server/store" +) + +type ( + composeNamespacePreproc struct { + is *importState + s store.Storer + } +) + +func NewComposeNamespacePreproc(is *importState, s store.Storer) envoy.Processor { + return &composeNamespacePreproc{ + is: is, + s: s, + } +} + +func (p *composeNamespacePreproc) Process(ctx context.Context, state *envoy.ExecState) error { + // @todo can we/should we have the same pattern as with decoder's CanDecode? + res, is := state.Res.(*resource.ComposeNamespace) + if !is { + return nil + } + + // Check if the given namespace exists + // Namespaces are top-level resources, so they don't depend on anything; + // no other checks are needed + + ns, err := loadComposeNamespace(ctx, p.s, res.SearchQuery()) + if err != nil { + return err + } + + if ns != nil { + p.is.AddExisting(res, ns.ID) + } + + return nil +} diff --git a/pkg/envoy/tmp/compose_record_set.go b/pkg/envoy/tmp/compose_record_set.go new file mode 100644 index 000000000..124d471de --- /dev/null +++ b/pkg/envoy/tmp/compose_record_set.go @@ -0,0 +1,13 @@ +package tmp + +import ( + "context" + + "github.com/cortezaproject/corteza-server/pkg/envoy/resource" + "github.com/cortezaproject/corteza-server/store" +) + +func encodeComposeRecordSet(ctx context.Context, s store.Storer, rec *resource.ComposeRecordSet, rm resMap) (uint64, error) { + // @todo... + return 0, nil +} diff --git a/pkg/envoy/tmp/compose_record_set_preproc.go b/pkg/envoy/tmp/compose_record_set_preproc.go new file mode 100644 index 000000000..b7de03b29 --- /dev/null +++ b/pkg/envoy/tmp/compose_record_set_preproc.go @@ -0,0 +1,89 @@ +package tmp + +import ( + "context" + + "github.com/cortezaproject/corteza-server/pkg/envoy" + "github.com/cortezaproject/corteza-server/pkg/envoy/resource" + "github.com/cortezaproject/corteza-server/store" +) + +type ( + composeRecordSetPreproc struct { + is *importState + s store.Storer + } +) + +func NewComposeRecordSetPreproc(is *importState, s store.Storer) *composeRecordSetPreproc { + return &composeRecordSetPreproc{ + is: is, + s: s, + } +} + +func (p *composeRecordSetPreproc) Process(ctx context.Context, state *envoy.ExecState) error { + // @todo can we/should we have the same pattern as with decoder's CanDecode? + res, is := state.Res.(*resource.ComposeRecordSet) + if !is { + return nil + } + + nsID, err := p.namespace(ctx, res, state) + if err != nil { + return err + } else if nsID <= 0 { + // If the namespace doesn't exist, no underlying resource is able to exist. + // @todo generate an error set to show as warnings? + return nil + } + + modID, err := p.module(ctx, res, state) + if err != nil { + return err + } else if modID <= 0 { + // If the module doesn't exist, no underlying resource is able to exist. + // @todo generate an error set to show as warnings? + return nil + } + + // @todo existing records, related records + + return nil +} + +func (p *composeRecordSetPreproc) namespace(ctx context.Context, res *resource.ComposeRecordSet, state *envoy.ExecState) (nsID uint64, err error) { + nss := filterComposeNamespaceResources(state.ParentResources) + if len(nss) > 0 { + nsID = p.is.Existint(nss[0]) + } else { + ns, idd, err := findMissingComposeNamespace(ctx, p.s, state.MissingDeps) + if err != nil { + return 0, err + } + if ns != nil { + nsID = ns.ID + p.is.AddRefMapping(res, "compose:namespace", nsID, idd.StringSlice()...) + } + } + + return nsID, nil +} + +func (p *composeRecordSetPreproc) module(ctx context.Context, res *resource.ComposeRecordSet, state *envoy.ExecState) (modID uint64, err error) { + modd := filterComposeModuleResources(state.ParentResources) + if len(modd) > 0 { + modID = p.is.Existint(modd[0]) + } else { + mod, idd, err := findMissingComposeModule(ctx, p.s, state.MissingDeps) + if err != nil { + return 0, err + } + if mod != nil { + modID = mod.ID + p.is.AddRefMapping(res, "compose:module", modID, idd.StringSlice()...) + } + } + + return modID, nil +} diff --git a/pkg/envoy/tmp/encoder.go b/pkg/envoy/tmp/encoder.go new file mode 100644 index 000000000..d2dfe4297 --- /dev/null +++ b/pkg/envoy/tmp/encoder.go @@ -0,0 +1,64 @@ +package tmp + +import ( + "context" + + "github.com/cortezaproject/corteza-server/compose/service/values" + "github.com/cortezaproject/corteza-server/pkg/envoy" + "github.com/cortezaproject/corteza-server/pkg/envoy/resource" + "github.com/cortezaproject/corteza-server/store" +) + +type ( + StoreEncoder struct { + s store.Storer + is *importState + } +) + +var ( + rvSanitizer = values.Sanitizer() + rvValidator = values.Validator() +) + +func NewStoreEncoder(s store.Storer, is *importState) *StoreEncoder { + return &StoreEncoder{ + s: s, + is: is, + } +} + +func (se *StoreEncoder) Encode(ctx context.Context, ess ...*envoy.ExecState) error { + return store.Tx(ctx, se.s, func(ctx context.Context, s store.Storer) (err error) { + var rID uint64 + + for _, es := range ess { + // @todo what should we do with existing resources? + // How should we do diffing? + rID = se.is.Existint(es.Res) + if rID <= 0 { + switch res := es.Res.(type) { + case *resource.ComposeNamespace: + rID, err = encodeComposeNamespace(ctx, s, res) + + case *resource.ComposeModule: + rID, err = encodeComposeModule(ctx, s, res, se.is.state[res]) + + case *resource.ComposeRecordSet: + _, err = encodeComposeRecordSet(ctx, s, res, se.is.state[res]) + } + + if err != nil { + return err + } + } + + for _, dr := range es.DepResources { + se.is.AddRefMapping(dr, es.Res.ResourceType(), rID, es.Res.Identifiers().StringSlice()...) + } + + } + + return nil + }) +} diff --git a/pkg/envoy/tmp/import_state.go b/pkg/envoy/tmp/import_state.go new file mode 100644 index 000000000..46721fd37 --- /dev/null +++ b/pkg/envoy/tmp/import_state.go @@ -0,0 +1,46 @@ +package tmp + +import "github.com/cortezaproject/corteza-server/pkg/envoy/resource" + +type ( + importState struct { + state resNodeMap + // We'll use this to keep track of the existing resources + existing map[resource.Interface]uint64 + } + + // ref: internal ID + refMap map[string]uint64 + // resource type: refMap + resMap map[string]refMap + resNodeMap map[resource.Interface]resMap +) + +func NewImportState() *importState { + return &importState{ + state: make(resNodeMap), + existing: make(map[resource.Interface]uint64), + } +} + +func (s *importState) AddRefMapping(r resource.Interface, res string, id uint64, refs ...string) { + if s.state[r] == nil { + s.state[r] = make(resMap) + } + + if s.state[r][res] == nil { + s.state[r][res] = make(refMap) + } + + for _, ref := range refs { + s.state[r][res][ref] = id + } +} + +func (s *importState) Existint(r resource.Interface) uint64 { + return s.existing[r] +} + +func (s *importState) AddExisting(r resource.Interface, resID uint64) { + s.existing[r] = resID +} diff --git a/pkg/envoy/tmp/preproc.go b/pkg/envoy/tmp/preproc.go new file mode 100644 index 000000000..58cbc9823 --- /dev/null +++ b/pkg/envoy/tmp/preproc.go @@ -0,0 +1,15 @@ +package tmp + +import ( + "github.com/cortezaproject/corteza-server/pkg/envoy" + "github.com/cortezaproject/corteza-server/store" +) + +func CollectPreproc(is *importState, s store.Storer) []envoy.Processor { + return []envoy.Processor{ + NewComposeModulePreproc(is, s), + NewComposeNamespacePreproc(is, s), + NewComposeRecordSetPreproc(is, s), + // ... + } +} diff --git a/pkg/envoy/tmp/preproc_util.go b/pkg/envoy/tmp/preproc_util.go new file mode 100644 index 000000000..6220beeef --- /dev/null +++ b/pkg/envoy/tmp/preproc_util.go @@ -0,0 +1,67 @@ +package tmp + +import ( + "context" + + "github.com/cortezaproject/corteza-server/compose/types" + "github.com/cortezaproject/corteza-server/pkg/envoy/resource" + "github.com/cortezaproject/corteza-server/store" +) + +func filterComposeNamespaceResources(rr []resource.Interface) []*resource.ComposeNamespace { + nn := make([]*resource.ComposeNamespace, 0, len(rr)) + for _, r := range rr { + switch n := r.(type) { + case *resource.ComposeNamespace: + nn = append(nn, n) + } + } + + return nn +} + +func filterComposeModuleResources(rr []resource.Interface) []*resource.ComposeModule { + nn := make([]*resource.ComposeModule, 0, len(rr)) + for _, r := range rr { + switch n := r.(type) { + case *resource.ComposeModule: + nn = append(nn, n) + } + } + + return nn +} + +func findMissingComposeNamespace(ctx context.Context, s store.Storer, missing resource.RefSet) (*types.Namespace, resource.Identifiers, error) { + // Check the store if we can find it + nsDd := missing.FilterByResourceType("compose:namespace") + if len(nsDd) > 0 { + nsD := nsDd[0] + ns, err := loadComposeNamespace(ctx, s, namespaceFilterFromGeneric(GetGenericFilter(nsD.Identifiers))) + if err != nil { + return nil, nil, err + } + if ns != nil { + return ns, nsD.Identifiers, nil + } + } + + return nil, nil, nil +} + +func findMissingComposeModule(ctx context.Context, s store.Storer, missing resource.RefSet) (*types.Module, resource.Identifiers, error) { + // Check the store if we can find it + modDd := missing.FilterByResourceType("compose:module") + if len(modDd) > 0 { + modD := modDd[0] + mod, err := loadComposeModule(ctx, s, moduleFilterFromGeneric(GetGenericFilter(modD.Identifiers))) + if err != nil { + return nil, nil, err + } + if mod != nil { + return mod, modD.Identifiers, nil + } + } + + return nil, nil, nil +} diff --git a/pkg/envoy/tmp/util.go b/pkg/envoy/tmp/util.go new file mode 100644 index 000000000..786f80293 --- /dev/null +++ b/pkg/envoy/tmp/util.go @@ -0,0 +1,49 @@ +package tmp + +import ( + "regexp" + "strconv" + + "github.com/cortezaproject/corteza-server/pkg/envoy/resource" + "github.com/cortezaproject/corteza-server/pkg/id" +) + +type ( + genericFilter struct { + ID uint64 + Ref string + Name string + } +) + +var ( + refy = regexp.MustCompile(`^[1-9](\d*)$`) + handy = regexp.MustCompile(`^[A-Za-z][0-9A-Za-z_\-.]*[A-Za-z0-9]$`) + + // wrapper around nextID that will aid service testing + nextID = func() uint64 { + return id.Next() + } +) + +func GetGenericFilter(ii resource.Identifiers) genericFilter { + f := genericFilter{} + for id := range ii { + if id == "" { + continue + } + if refy.MatchString(id) { + id, err := strconv.ParseUint(id, 10, 64) + if err != nil { + continue + } + f.ID = id + } else if handy.MatchString(id) { + f.Ref = id + } else { + f.Name = id + } + } + + return f +}