DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Improve error handling & recovery

Browse Source
This commit is contained in:
Tomaž Jerman
2023-03-17 10:58:46 +01:00
parent a0e25b457f
commit 2f015a26f6
29 changed files with 1498 additions and 960 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/codegen/assets/templates/gocode/envoy/rbac_rules_parse.go.tpl
+5
View File
@@ -41,6 +41,11 @@ func ParseRule(res string) (string, *Ref, []*Ref, error) {
}
}
// @todo this is to support res. tr. resources also.
// Split it into a separate function and remove this.
if !strings.HasPrefix(resourceType, "corteza::") {
resourceType = "corteza::" + resourceType
}
// make the resource provide the slice of parent resources we should nest under
switch resourceType {
server/codegen/assets/templates/gocode/envoy/store_decode.go.tpl
+100 -78
View File
@@ -10,6 +10,7 @@ import (
"github.com/cortezaproject/corteza/server/pkg/dal"
"github.com/cortezaproject/corteza/server/pkg/envoyx"
"github.com/cortezaproject/corteza/server/store"
"github.com/pkg/errors"
{{- range .imports }}
"{{ . }}"
@@ -64,27 +65,34 @@ func (d StoreDecoder) decode(ctx context.Context, s store.Storer, dl dal.FullSer
// Get all requested scopes
scopedNodes := make(envoyx.NodeSet, len(p.Filter))
{{ if eq .componentIdent "compose" }}
for i, a := range wrappedFilters {
if a.f.Scope.ResourceType == "" {
continue
}
err = func() (err error) {
for i, a := range wrappedFilters {
if a.f.Scope.ResourceType == "" {
continue
}
// For now the scope can only point to namespace so this will do
var nn envoyx.NodeSet
nn, err = d.decodeNamespace(ctx, s, dl, d.makeNamespaceFilter(nil, nil, envoyx.ResourceFilter{Identifiers: a.f.Scope.Identifiers}))
if err != nil {
return
}
if len(nn) > 1 {
err = fmt.Errorf("ambiguous scope %v", a.f.Scope)
return
}
if len(nn) == 0 {
err = fmt.Errorf("invalid scope: resource not found %v", a.f)
return
}
// For now the scope can only point to namespace so this will do
var nn envoyx.NodeSet
nn, err = d.decodeNamespace(ctx, s, dl, d.makeNamespaceFilter(nil, nil, envoyx.ResourceFilter{Identifiers: a.f.Scope.Identifiers}))
if err != nil {
return
}
if len(nn) > 1 {
err = fmt.Errorf("ambiguous scope %v: matches multiple resources", a.f.Scope)
return
}
if len(nn) == 0 {
err = fmt.Errorf("invalid scope %v: resource not found", a.f)
return
}
scopedNodes[i] = nn[0]
scopedNodes[i] = nn[0]
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to decode node scopes")
return
}
{{ else }}
// @note skipping scope logic since it's currently only supported within
@@ -97,72 +105,86 @@ func (d StoreDecoder) decode(ctx context.Context, s store.Storer, dl dal.FullSer
// lives easier.
refNodes := make([]map[string]*envoyx.Node, len(p.Filter))
refRefs := make([]map[string]envoyx.Ref, len(p.Filter))
for i, a := range wrappedFilters {
if len(a.f.Refs) == 0 {
continue
err = func() (err error) {
for i, a := range wrappedFilters {
if len(a.f.Refs) == 0 {
continue
}
auxr := make(map[string]*envoyx.Node, len(a.f.Refs))
auxa := make(map[string]envoyx.Ref)
for field, ref := range a.f.Refs {
f := ref.ResourceFilter()
aux, err := d.decode(ctx, s, dl, envoyx.DecodeParams{
Type: envoyx.DecodeTypeStore,
Filter: f,
})
if err != nil {
return err
}
// @todo consider changing this.
// Currently it's required because the .decode may return some
// nested nodes as well.
// Consider a flag or a new function.
aux = envoyx.NodesForResourceType(ref.ResourceType, aux...)
if len(aux) == 0 {
return fmt.Errorf("invalid reference %v", ref)
}
if len(aux) > 1 {
return fmt.Errorf("ambiguous reference: too many resources returned %v", a.f)
}
auxr[field] = aux[0]
auxa[field] = aux[0].ToRef()
}
refNodes[i] = auxr
refRefs[i] = auxa
}
auxr := make(map[string]*envoyx.Node, len(a.f.Refs))
auxa := make(map[string]envoyx.Ref)
for field, ref := range a.f.Refs {
f := ref.ResourceFilter()
aux, err := d.decode(ctx, s, dl, envoyx.DecodeParams{
Type: envoyx.DecodeTypeStore,
Filter: f,
})
if err != nil {
return nil, err
}
// @todo consider changing this.
// Currently it's required because the .decode may return some
// nested nodes as well.
// Consider a flag or a new function.
aux = envoyx.NodesForResourceType(ref.ResourceType, aux...)
if len(aux) == 0 {
return nil, fmt.Errorf("invalid reference %v", ref)
}
if len(aux) > 1 {
return nil, fmt.Errorf("ambiguous reference: too many resources returned %v", a.f)
}
auxr[field] = aux[0]
auxa[field] = aux[0].ToRef()
}
refNodes[i] = auxr
refRefs[i] = auxa
return
}()
if err != nil {
err = errors.Wrap(err, "failed to decode node references")
return
}
var aux envoyx.NodeSet
for i, wf := range wrappedFilters {
switch wf.rt {
{{ range .resources -}}
{{- if .envoy.omit}}{{continue}}{{ end -}}
err = func() (err error) {
var aux envoyx.NodeSet
for i, wf := range wrappedFilters {
switch wf.rt {
{{ range .resources -}}
{{- if .envoy.omit}}{{continue}}{{ end -}}
case types.{{.expIdent}}ResourceType:
aux, err = d.decode{{.expIdent}}(ctx, s, dl, d.make{{.expIdent}}Filter(scopedNodes[i], refNodes[i], wf.f))
if err != nil {
return
}
for _, a := range aux {
a.Identifiers = a.Identifiers.Merge(wf.f.Identifiers)
a.References = envoyx.MergeRefs(a.References, refRefs[i])
}
out = append(out, aux...)
case types.{{.expIdent}}ResourceType:
aux, err = d.decode{{.expIdent}}(ctx, s, dl, d.make{{.expIdent}}Filter(scopedNodes[i], refNodes[i], wf.f))
if err != nil {
return
}
for _, a := range aux {
a.Identifiers = a.Identifiers.Merge(wf.f.Identifiers)
a.References = envoyx.MergeRefs(a.References, refRefs[i])
}
out = append(out, aux...)
{{ end }}
default:
aux, err = d.extendDecoder(ctx, s, dl, wf.rt, refNodes[i], wf.f)
if err!= nil {
return
{{ end }}
default:
aux, err = d.extendDecoder(ctx, s, dl, wf.rt, refNodes[i], wf.f)
if err!= nil {
return
}
for _, a := range aux {
a.Identifiers = a.Identifiers.Merge(wf.f.Identifiers)
a.References = envoyx.MergeRefs(a.References, refRefs[i])
}
out = append(out, aux...)
}
for _, a := range aux {
a.Identifiers = a.Identifiers.Merge(wf.f.Identifiers)
a.References = envoyx.MergeRefs(a.References, refRefs[i])
}
out = append(out, aux...)
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to decode filters")
return
}
return
server/codegen/assets/templates/gocode/envoy/store_encode.go.tpl
+67 -43
View File
@@ -11,6 +11,7 @@ import (
"github.com/cortezaproject/corteza/server/pkg/expr"
"github.com/cortezaproject/corteza/server/pkg/id"
"github.com/cortezaproject/corteza/server/store"
"github.com/pkg/errors"
{{- range .imports }}
"{{ . }}"
{{- end }}
@@ -24,6 +25,11 @@ type (
StoreEncoder struct{}
)
const (
paramsKeyStorer = "storer"
paramsKeyDAL = "dal"
)
{{ $rootRes := .resources }}
@@ -116,6 +122,7 @@ func (e StoreEncoder) prepare{{.expIdent}}(ctx context.Context, p envoyx.EncodeP
existing := make(map[int]types.{{.expIdent}}, len(nn))
err = e.matchup{{.expIdent}}s(ctx, s, existing, nn)
if err != nil {
err = errors.Wrap(err, "failed to matchup existing {{.expIdent}}s")
return
}
@@ -144,7 +151,7 @@ func (e StoreEncoder) prepare{{.expIdent}}(ctx context.Context, p envoyx.EncodeP
// In the future, we can pass down the tree and re-do the deps like that
switch n.Config.MergeAlg {
case envoyx.OnConflictPanic:
err = fmt.Errorf("resource already exists")
err = fmt.Errorf("resource %v already exists, n.Identifiers.Slice")
return
case envoyx.OnConflictReplace:
@@ -201,23 +208,30 @@ func (e StoreEncoder) encode{{.expIdent}}s(ctx context.Context, p envoyx.EncodeP
func (e StoreEncoder) encode{{.expIdent}}(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
// Grab dependency references
var auxID uint64
for fieldLabel, ref := range n.References {
rn := tree.ParentForRef(n, ref)
if rn == nil {
err = fmt.Errorf("missing node for ref %v", ref)
return
}
err = func() (err error) {
for fieldLabel, ref := range n.References {
rn := tree.ParentForRef(n, ref)
if rn == nil {
err = fmt.Errorf("parent reference %v not found", ref)
return
}
auxID = rn.Resource.GetID()
if auxID == 0 {
err = fmt.Errorf("related resource doesn't provide an ID")
return
}
auxID = rn.Resource.GetID()
if auxID == 0 {
err = fmt.Errorf("parent reference does not provide an identifier")
return
}
err = n.Resource.SetValue(fieldLabel, 0, auxID)
if err != nil {
return
err = n.Resource.SetValue(fieldLabel, 0, auxID)
if err != nil {
return
}
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to set dependency references")
return
}
{{ if .envoy.store.sanitizeBeforeSave }}
@@ -229,6 +243,7 @@ func (e StoreEncoder) encode{{.expIdent}}(ctx context.Context, p envoyx.EncodePa
// Flush to the DB
err = store.Upsert{{.store.expIdent}}(ctx, s, n.Resource.(*types.{{.expIdent}}))
if err != nil {
err = errors.Wrap(err, "failed to upsert {{.expIdent}}")
return
}
@@ -238,47 +253,56 @@ func (e StoreEncoder) encode{{.expIdent}}(ctx context.Context, p envoyx.EncodePa
//
// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
// the root function calls?
{{/*
{{/*
@note this setup will not duplicate encode calls since we only take the
most specific parent resource.
most specific parent resource.
*/}}
{{ $extendedEncoder := .envoy.store.extendedEncoder}}
{{ if $extendedEncoder }}
nested := make(envoyx.NodeSet, 0, 10)
{{ end }}
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
nn = envoyx.OmitPlaceholderNodes(nn...)
err = func() (err error) {
for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
nn = envoyx.OmitPlaceholderNodes(nn...)
switch rt {
{{- range $cmp := $rootRes }}
{{ if $cmp.envoy.omit }}
{{continue}}
{{ end }}
{{ if not $cmp.parents }}
{{continue}}
{{ end }}
{{ $p := index $cmp.parents (sub (len $cmp.parents) 1)}}
{{ if not (eq $p.handle $a.ident) }}
switch rt {
{{- range $cmp := $rootRes }}
{{ if $cmp.envoy.omit }}
{{continue}}
{{ end }}
{{ if not $cmp.parents }}
{{continue}}
{{ end }}
case types.{{$cmp.expIdent}}ResourceType:
err = e.encode{{$cmp.expIdent}}s(ctx, p, s, nn, tree)
if err != nil {
return
}
{{ if $extendedEncoder }}
nested = append(nested, nn...)
{{ end }}
{{ $p := index $cmp.parents (sub (len $cmp.parents) 1)}}
{{ if not (eq $p.handle $a.ident) }}
{{continue}}
{{ end }}
{{- end }}
case types.{{$cmp.expIdent}}ResourceType:
err = e.encode{{$cmp.expIdent}}s(ctx, p, s, nn, tree)
if err != nil {
return
}
{{ if $extendedEncoder }}
nested = append(nested, nn...)
{{ end }}
{{- end }}
}
}
return
}()
if err != nil {
err = errors.Wrap(err, "failed to encode nested resources")
return
}
{{ if .envoy.store.extendedEncoder }}
err = e.encode{{.expIdent}}Extend(ctx, p, s, n, nested, tree)
if err != nil {
err = errors.Wrap(err, "post encode logic failed with errors")
return
}
{{ end }}
@@ -286,6 +310,7 @@ func (e StoreEncoder) encode{{.expIdent}}(ctx context.Context, p envoyx.EncodePa
{{ if .envoy.store.extendedSubResources }}
err = e.encode{{.expIdent}}ExtendSubResources(ctx, p, s, n, tree)
if err != nil {
err = errors.Wrap(err, "failed to encode extended sub resources")
return
}
{{ end }}
@@ -349,15 +374,15 @@ func (e StoreEncoder) matchup{{.expIdent}}s(ctx context.Context, s store.Storer,
// // // // // // // // // // // // // // // // // // // // // // // // //
func (e *StoreEncoder) grabStorer(p envoyx.EncodeParams) (s store.Storer, err error) {
auxs, ok := p.Params["storer"]
auxs, ok := p.Params[paramsKeyStorer]
if !ok {
err = fmt.Errorf("storer not defined")
err = errors.Errorf("store encoder expects a store conforming to store.Storer interface")
return
}
s, ok = auxs.(store.Storer)
if !ok {
err = fmt.Errorf("invalid storer provided")
err = errors.Errorf("store encoder expects a store conforming to store.Storer interface")
return
}
@@ -378,6 +403,5 @@ func (e *StoreEncoder) runEvals(ctx context.Context, existing bool, n *envoyx.No
}
n.Evaluated.Skip, err = n.Config.SkipIfEval.Test(ctx, aux.(*expr.Vars))
return
}
server/codegen/assets/templates/gocode/envoy/yaml_decode.go.tpl
+15 -8
View File
@@ -6,7 +6,6 @@ import (
"strings"
"context"
"io"
"fmt"
"os"
systemTypes "github.com/cortezaproject/corteza/server/system/types"
@@ -16,6 +15,7 @@ import (
"golang.org/x/text/language"
"github.com/spf13/cast"
"gopkg.in/yaml.v3"
"github.com/pkg/errors"
{{- range .imports }}
"{{ . }}"
@@ -37,6 +37,10 @@ type (
}
)
const (
paramsKeyStream = "stream"
)
func (d YamlDecoder) CanFile(f *os.File) (ok bool) {
// @todo improve/expand
return d.canExt(f.Name())
@@ -65,6 +69,7 @@ func (d YamlDecoder) Decode(ctx context.Context, p envoyx.DecodeParams) (out env
doc := &auxYamlDoc{}
err = yaml.NewDecoder(r).Decode(doc)
if err != nil {
err = errors.Wrap(err, "{{$cmpIdent}} yaml decoder: failed to decode document")
return
}
@@ -96,13 +101,15 @@ func (d *auxYamlDoc) UnmarshalYAML(n *yaml.Node) (err error) {
if y7s.IsMapping(v) {
aux, err = d.unmarshal{{.expIdent}}Map(dctx, v)
d.nodes = append(d.nodes, aux...)
return err
}
{{- end }}
if y7s.IsSeq(v) {
aux, err = d.unmarshal{{.expIdent}}Seq(dctx, v)
d.nodes = append(d.nodes, aux...)
}
if err != nil {
err = errors.Wrap(err, "failed to unmarshal {{.ident}}")
}
return err
{{ end }}
@@ -112,14 +119,14 @@ func (d *auxYamlDoc) UnmarshalYAML(n *yaml.Node) (err error) {
aux, err = unmarshalAllowNode(v)
d.nodes = append(d.nodes, aux...)
if err != nil {
return err
err = errors.Wrap(err, "failed to unmarshal node: RBAC allow")
}
case "deny":
aux, err = unmarshalDenyNode(v)
d.nodes = append(d.nodes, aux...)
if err != nil {
return err
err = errors.Wrap(err, "failed to unmarshal node: RBAC deny")
}
// Resource translation nodes
@@ -127,7 +134,7 @@ func (d *auxYamlDoc) UnmarshalYAML(n *yaml.Node) (err error) {
aux, err = unmarshalLocaleNode(v)
d.nodes = append(d.nodes, aux...)
if err != nil {
return err
err = errors.Wrap(err, "failed to unmarshal node: locale")
}
{{ end }}
@@ -136,7 +143,7 @@ func (d *auxYamlDoc) UnmarshalYAML(n *yaml.Node) (err error) {
aux, err = d.unmarshalYAML(kv, v)
d.nodes = append(d.nodes, aux...)
if err != nil {
return err
err = errors.Wrap(err, "failed to unmarshal node")
}
}
return nil
@@ -783,7 +790,7 @@ func (d *auxYamlDoc) decodeEnvoyConfig(n *yaml.Node) (out envoyx.EnvoyConfig) {
// // // // // // // // // // // // // // // // // // // // // // // // //
func (d YamlDecoder) getReader(ctx context.Context, p envoyx.DecodeParams) (r io.Reader, err error) {
aux, ok := p.Params["stream"]
aux, ok := p.Params[paramsKeyStream]
if ok {
r, ok = aux.(io.Reader)
if ok {
@@ -792,7 +799,7 @@ func (d YamlDecoder) getReader(ctx context.Context, p envoyx.DecodeParams) (r io
}
// @todo consider adding support for managing files from a location
err = fmt.Errorf("YAML decoder expects a stream conforming to io.Reader interface")
err = errors.Errorf("YAML decoder expects a stream conforming to io.Reader interface")
return
}
server/codegen/assets/templates/gocode/envoy/yaml_encode.go.tpl
+7 -5
View File
@@ -4,14 +4,13 @@ package {{ .package }}
import (
"context"
"fmt"
"io"
"time"
"github.com/cortezaproject/corteza/server/pkg/envoyx"
"github.com/cortezaproject/corteza/server/pkg/y7s"
"gopkg.in/yaml.v3"
"github.com/pkg/errors"
{{- range .imports }}
"{{ . }}"
{{- end }}
@@ -23,6 +22,10 @@ type (
YamlEncoder struct{}
)
const (
paramsKeyWriter = "writer"
)
// Encode encodes the given Corteza resources into some YAML supported format
//
// Encoding should not do any additional processing apart from matching with
@@ -258,7 +261,6 @@ func (e YamlEncoder) encodeTimestamp(p envoyx.EncodeParams, t time.Time) (any, e
func (e YamlEncoder) encodeTimestampNil(p envoyx.EncodeParams, t *time.Time) (any, error) {
if t == nil { return nil, nil }
// @todo timestamp encoding format
return e.encodeTimestamp(p, *t)
}
@@ -280,7 +282,7 @@ func (e YamlEncoder) encodeRef(p envoyx.EncodeParams, id uint64, field string, n
// // // // // // // // // // // // // // // // // // // // // // // // //
func (e YamlEncoder) getWriter(p envoyx.EncodeParams) (out io.Writer, err error) {
aux, ok := p.Params["writer"]
aux, ok := p.Params[paramsKeyWriter]
if ok {
out, ok = aux.(io.Writer)
if ok {
@@ -289,6 +291,6 @@ func (e YamlEncoder) getWriter(p envoyx.EncodeParams) (out io.Writer, err error)
}
// @todo consider adding support for managing files from a location
err = fmt.Errorf("YAML encoder expects a writer conforming to io.Writer interface")
err = errors.Errorf("YAML encoder expects a writer conforming to io.Writer interface")
return
}