diff --git a/server/automation/envoy/store_encode.gen.go b/server/automation/envoy/store_encode.gen.go index b315fe845..52473b843 100644 --- a/server/automation/envoy/store_encode.gen.go +++ b/server/automation/envoy/store_encode.gen.go @@ -48,6 +48,8 @@ func (e StoreEncoder) Prepare(ctx context.Context, p envoyx.EncodeParams, rt str case types.TriggerResourceType: return e.prepareTrigger(ctx, p, s, nn) + default: + return e.prepare(ctx, p, s, rt, nn) } return diff --git a/server/automation/envoy/yaml_decode.gen.go b/server/automation/envoy/yaml_decode.gen.go index 1fefcd186..855f0b132 100644 --- a/server/automation/envoy/yaml_decode.gen.go +++ b/server/automation/envoy/yaml_decode.gen.go @@ -175,25 +175,6 @@ func (d *auxYamlDoc) unmarshalExtendedTriggersSeq(dctx documentContext, n *yaml. return } -// unmarshalTriggersExtendedMap unmarshals Triggers when provided as a mapping node -// -// When map encoded, the map key is used as a preset identifier. -// The identifier is passed to the node function as a meta node -func (d *auxYamlDoc) unmarshalExtendedTriggersMap(dctx documentContext, n *yaml.Node) (out envoyx.NodeSet, err error) { - var aux envoyx.NodeSet - err = y7s.EachMap(n, func(k, n *yaml.Node) error { - aux, err = d.unmarshalTriggersExtendedNode(dctx, n, k) - if err != nil { - return err - } - out = append(out, aux...) - - return nil - }) - - return -} - // unmarshalWorkflowNode is a cookie-cutter function to unmarshal // the yaml node into the corresponding Corteza type & Node func (d *auxYamlDoc) unmarshalWorkflowNode(dctx documentContext, n *yaml.Node, meta ...*yaml.Node) (out envoyx.NodeSet, err error) { @@ -359,13 +340,13 @@ func (d *auxYamlDoc) unmarshalWorkflowNode(dctx documentContext, n *yaml.Node, m // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil switch strings.ToLower(k.Value) { case "triggers": - default: if y7s.IsSeq(n) { nestedNodes, err = d.unmarshalExtendedTriggersSeq(dctx, n) if err != nil { @@ -395,17 +376,24 @@ func (d *auxYamlDoc) unmarshalWorkflowNode(dctx documentContext, n *yaml.Node, m Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, @@ -597,6 +585,7 @@ func (d *auxYamlDoc) unmarshalTriggerNode(dctx documentContext, n *yaml.Node, me // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil @@ -624,17 +613,24 @@ func (d *auxYamlDoc) unmarshalTriggerNode(dctx documentContext, n *yaml.Node, me Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, diff --git a/server/codegen/assets/templates/gocode/envoy/store_encode.go.tpl b/server/codegen/assets/templates/gocode/envoy/store_encode.go.tpl index 4c9b52d2c..875f8e42d 100644 --- a/server/codegen/assets/templates/gocode/envoy/store_encode.go.tpl +++ b/server/codegen/assets/templates/gocode/envoy/store_encode.go.tpl @@ -53,6 +53,8 @@ func (e StoreEncoder) Prepare(ctx context.Context, p envoyx.EncodeParams, rt str case types.{{.expIdent}}ResourceType: return e.prepare{{.expIdent}}(ctx, p, s, nn) {{ end -}} + default: + return e.prepare(ctx, p, s, rt, nn) } return @@ -228,12 +230,16 @@ func (e StoreEncoder) encode{{.expIdent}}(ctx context.Context, p envoyx.EncodePa @note this setup will not duplicate encode calls since we only take the most specific parent resource. */}} + {{ $extendedEncoder := .envoy.store.extendedEncoder}} + {{ if $extendedEncoder }} + nested := make(envoyx.NodeSet, 0, 10) + {{ end }} for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) { nn = envoyx.OmitPlaceholderNodes(nn...) switch rt { {{- range $cmp := $rootRes }} - {{ if or ($cmp.envoy.omit) (not $cmp.envoy.use) }} + {{ if $cmp.envoy.omit }} {{continue}} {{ end }} {{ if not $cmp.parents }} @@ -250,12 +256,28 @@ func (e StoreEncoder) encode{{.expIdent}}(ctx context.Context, p envoyx.EncodePa if err != nil { return } + {{ if $extendedEncoder }} + nested = append(nested, nn...) + {{ end }} {{- end }} } } - return + {{ if .envoy.store.extendedEncoder }} + err = e.encode{{.expIdent}}Extend(ctx, p, s, n, nested, tree) + if err != nil { + return + } + {{ end }} + + {{ if .envoy.store.extendedSubResources }} + err = e.encode{{.expIdent}}ExtendSubResources(ctx, p, s, n, tree) + if err != nil { + return + } + {{ end }} + return } // matchup{{.expIdent}}s returns an index with indicates what resources already exist diff --git a/server/codegen/assets/templates/gocode/envoy/util_rbac.go.tpl b/server/codegen/assets/templates/gocode/envoy/util_rbac.go.tpl index 9fd3de20f..06d361650 100644 --- a/server/codegen/assets/templates/gocode/envoy/util_rbac.go.tpl +++ b/server/codegen/assets/templates/gocode/envoy/util_rbac.go.tpl @@ -43,7 +43,7 @@ func SplitResourceIdentifier(ref string) (out map[string]Ref) { } {{- range $cmp := $rootCmp.resources }} - {{ if or ($cmp.envoy.omit) (not $cmp.envoy.use) }} + {{ if $cmp.envoy.omit }} {{continue}} {{ end }} diff --git a/server/codegen/assets/templates/gocode/envoy/utils.go.tpl b/server/codegen/assets/templates/gocode/envoy/utils.go.tpl index 9978f0cce..5503fddc5 100644 --- a/server/codegen/assets/templates/gocode/envoy/utils.go.tpl +++ b/server/codegen/assets/templates/gocode/envoy/utils.go.tpl @@ -22,11 +22,13 @@ var ( "{{ .fqrt }}": true, {{- end }} {{- end }} + "corteza::compose:record-datasource": true, } // superNeedyResources is the second level of filtering in case the first // pass removes everything superNeedyResources = map[string]bool{ "corteza::compose:module-field": true, + "corteza::compose:record-datasource": true, } ) diff --git a/server/codegen/assets/templates/gocode/envoy/yaml_decode.go.tpl b/server/codegen/assets/templates/gocode/envoy/yaml_decode.go.tpl index 2ee40b1b9..b50a629f2 100644 --- a/server/codegen/assets/templates/gocode/envoy/yaml_decode.go.tpl +++ b/server/codegen/assets/templates/gocode/envoy/yaml_decode.go.tpl @@ -191,7 +191,7 @@ func (d *auxYamlDoc) unmarshalExtended{{.expIdent}}Seq(dctx documentContext, n * return } - +{{if not .supportMappedInput}}{{continue}}{{end}} // unmarshal{{.expIdent}}ExtendedMap unmarshals {{.expIdent}} when provided as a mapping node // // When map encoded, the map key is used as a preset identifier. @@ -417,6 +417,7 @@ func (d *auxYamlDoc) unmarshal{{ .expIdent }}Node(dctx documentContext, n *yaml. // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil @@ -432,7 +433,7 @@ func (d *auxYamlDoc) unmarshal{{ .expIdent }}Node(dctx documentContext, n *yaml. */}} {{ $a := . }} {{- range $cmp := $rootRes }} - {{ if or ($cmp.envoy.omit) (not $cmp.envoy.use) }} + {{ if $cmp.envoy.omit }} {{continue}} {{ end }} @@ -468,7 +469,6 @@ func (d *auxYamlDoc) unmarshal{{ .expIdent }}Node(dctx documentContext, n *yaml. case {{ range $i, $l := $identKeys -}} "{{ $l }}"{{if not (eq $i (sub (len $identKeys) 1))}},{{end}} {{- end}}: - default: if y7s.IsSeq(n) { nestedNodes, err = d.unmarshalExtended{{.expIdent}}Seq(dctx, n) if err != nil { @@ -504,17 +504,30 @@ func (d *auxYamlDoc) unmarshal{{ .expIdent }}Node(dctx documentContext, n *yaml. Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + {{ if .envoy.yaml.extendedResourcePostProcess}} + auxNestedNodes, err = d.postProcessNested{{.expIdent}}Nodes(auxNestedNodes) + if err!= nil { + return + } + {{ end }} + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, diff --git a/server/codegen/assets/templates/gocode/envoy/yaml_encode.go.tpl b/server/codegen/assets/templates/gocode/envoy/yaml_encode.go.tpl index 1d90a80cb..06f8dcd8a 100644 --- a/server/codegen/assets/templates/gocode/envoy/yaml_encode.go.tpl +++ b/server/codegen/assets/templates/gocode/envoy/yaml_encode.go.tpl @@ -152,7 +152,7 @@ func (e YamlEncoder) encode{{.expIdent}}(ctx context.Context, p envoyx.EncodePar _ = aux {{ $a := . }} {{- range $cmp := $rootRes }} - {{ if or ($cmp.envoy.omit) (not $cmp.envoy.use) }}{{continue}}{{ end }} + {{ if $cmp.envoy.omit }}{{continue}}{{ end }} {{if not $cmp.parents}}{{continue}}{{end}} {{/* diff --git a/server/codegen/schema/resource.cue b/server/codegen/schema/resource.cue index 8d5413168..b4e4827f4 100644 --- a/server/codegen/schema/resource.cue +++ b/server/codegen/schema/resource.cue @@ -177,6 +177,7 @@ import ( // referencing this resource identKeys: [...string] | *([identKeyLabel]+identKeyAlias) + extendedResourcePostProcess: bool | *false extendedResourceDecoders: [...{ ident: string expIdent: string @@ -189,6 +190,10 @@ import ( // store decode/encode configs store: { + // enable or disable custom logic after the resource is imported + extendedEncoder: bool | *false + extendedSubResources: bool | *false + // enable or disable additional custom processing for determining // resource references extendedRefDecoder: bool | *false diff --git a/server/compose/envoy/record_datasource.go b/server/compose/envoy/record_datasource.go new file mode 100644 index 000000000..835f1bf2e --- /dev/null +++ b/server/compose/envoy/record_datasource.go @@ -0,0 +1,73 @@ +package envoy + +import ( + "github.com/cortezaproject/corteza/server/pkg/envoyx" + "github.com/spf13/cast" +) + +type ( + // RecordDatasource provides a mechanism for you to access large + // record datasets optimally + RecordDatasource struct { + mapping datasourceMapping + provider envoyx.Provider + + // Reusable buffer for reading records + rowCache map[string]string + + // Index to map from ref to ID + // @todo we might need to flush these to the disc in case a huge dataset is passed in + refToID map[string]uint64 + } +) + +func (rd *RecordDatasource) SetProvider(s envoyx.Provider) bool { + if rd.mapping.SourceIdent != s.Ident() { + return false + } + + rd.provider = s + return true +} + +func (rd *RecordDatasource) Next(out map[string]string) (ident string, more bool, err error) { + if rd.rowCache == nil { + rd.rowCache = make(map[string]string) + } + + more, err = rd.provider.Next(rd.rowCache) + if err != nil || !more { + return + } + + rd.applyMapping(rd.rowCache, out) + + ident = out[rd.mapping.KeyField] + + return +} + +func (rd *RecordDatasource) Reset() (err error) { + return rd.provider.Reset() +} + +func (rd *RecordDatasource) applyMapping(in, out map[string]string) { + for _, m := range rd.mapping.Mapping.m { + if m.Skip { + continue + } + + // @todo expand when needed (expressions and such) + out[m.Field] = in[m.Column] + } +} + +func (rd *RecordDatasource) ResolveRef(ref any) (out uint64, err error) { + r, err := cast.ToStringE(ref) + if err != nil { + return + } + + out = rd.refToID[r] + return +} diff --git a/server/compose/envoy/record_getter.go b/server/compose/envoy/record_getter.go new file mode 100644 index 000000000..625d1f60a --- /dev/null +++ b/server/compose/envoy/record_getter.go @@ -0,0 +1,102 @@ +package envoy + +import ( + "context" + + "github.com/cortezaproject/corteza/server/compose/dalutils" + "github.com/cortezaproject/corteza/server/compose/types" + "github.com/cortezaproject/corteza/server/pkg/dal" + "github.com/cortezaproject/corteza/server/pkg/envoyx" + "github.com/spf13/cast" +) + +type ( + // recordGetter is a utility struct to resolve record references from + // different parts of the system such as the dep graph and the database + recordGetter struct { + // Bits to get data from the dep graph + relDatasource *RecordDatasource + + // Bits to get data from the database + dalSvc dal.FullService + relModule types.Module + baseFilter types.RecordFilter + } +) + +func makeRecordGetter(dalSvc dal.FullService, tt envoyx.Traverser, n *envoyx.Node, modRef, dsRef envoyx.Ref) (g *recordGetter) { + g = &recordGetter{ + dalSvc: dalSvc, + } + + // Resolve from dep graph + auxDs := tt.ParentForRef(n, dsRef) + if auxDs != nil { + g.relDatasource = auxDs.Datasource.(*RecordDatasource) + } + + // Resolve from the database + mod := g.getRefMod(tt, n, modRef) + if mod != nil { + g.baseFilter = types.RecordFilter{ + ModuleID: mod.ID, + NamespaceID: mod.NamespaceID, + } + } + + return +} + +// resolve resolves the provided reference into a record ID; 0 if can't be resolved +func (g *recordGetter) resolve(ctx context.Context, ref any) (out uint64, err error) { + // Try to get from datasource + if g.relDatasource != nil { + out, err = g.getDS(ref) + if err != nil { + return + } + } + if out > 0 { + return + } + + // Fallback to the store + out, err = g.getDB(ctx, ref) + if err != nil { + return + } + if out > 0 { + return + } + return +} + +func (g *recordGetter) getDS(ref any) (out uint64, err error) { + return g.relDatasource.ResolveRef(ref) +} + +func (g *recordGetter) getDB(ctx context.Context, ref any) (out uint64, err error) { + // @note the old version only resolved IDs so that's what we're doing here also + // @todo consider expanding this + id, err := cast.ToUint64E(ref) + if err != nil { + return 0, nil + } + + aux, err := dalutils.ComposeRecordsFind(ctx, g.dalSvc, &g.relModule, id) + if err != nil { + return + } + + out = aux.ID + return +} + +func (g *recordGetter) getRefMod(tt envoyx.Traverser, n *envoyx.Node, ref envoyx.Ref) (refMod *types.Module) { + aux := tt.ParentForRef(n, ref) + if aux == nil { + return nil + } + + return aux.Resource.(*types.Module) +} diff --git a/server/compose/envoy/store_encode.gen.go b/server/compose/envoy/store_encode.gen.go index 172d5b021..0545a155c 100644 --- a/server/compose/envoy/store_encode.gen.go +++ b/server/compose/envoy/store_encode.gen.go @@ -54,6 +54,8 @@ func (e StoreEncoder) Prepare(ctx context.Context, p envoyx.EncodeParams, rt str case types.PageResourceType: return e.preparePage(ctx, p, s, nn) + default: + return e.prepare(ctx, p, s, rt, nn) } return @@ -406,14 +408,34 @@ func (e StoreEncoder) encodeModule(ctx context.Context, p envoyx.EncodeParams, s // @todo how can we remove the OmitPlaceholderNodes call the same way we did for // the root function calls? + nested := make(envoyx.NodeSet, 0, 10) + for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) { nn = envoyx.OmitPlaceholderNodes(nn...) switch rt { + case types.ModuleFieldResourceType: + err = e.encodeModuleFields(ctx, p, s, nn, tree) + if err != nil { + return + } + + nested = append(nested, nn...) + } } + err = e.encodeModuleExtend(ctx, p, s, n, nested, tree) + if err != nil { + return + } + + err = e.encodeModuleExtendSubResources(ctx, p, s, n, tree) + if err != nil { + return + } + return } @@ -771,6 +793,24 @@ func (e StoreEncoder) encodeNamespace(ctx context.Context, p envoyx.EncodeParams switch rt { + case types.ChartResourceType: + err = e.encodeCharts(ctx, p, s, nn, tree) + if err != nil { + return + } + + case types.ModuleResourceType: + err = e.encodeModules(ctx, p, s, nn, tree) + if err != nil { + return + } + + case types.PageResourceType: + err = e.encodePages(ctx, p, s, nn, tree) + if err != nil { + return + } + } } diff --git a/server/compose/envoy/store_encode.go b/server/compose/envoy/store_encode.go index 0fcd494fc..8eb4d5f09 100644 --- a/server/compose/envoy/store_encode.go +++ b/server/compose/envoy/store_encode.go @@ -1,7 +1,14 @@ package envoy import ( + "context" + "fmt" + + "github.com/cortezaproject/corteza/server/compose/service" "github.com/cortezaproject/corteza/server/compose/types" + "github.com/cortezaproject/corteza/server/pkg/dal" + "github.com/cortezaproject/corteza/server/pkg/envoyx" + "github.com/cortezaproject/corteza/server/store" ) func (e StoreEncoder) setChartDefaults(res *types.Chart) (err error) { @@ -43,3 +50,77 @@ func (e StoreEncoder) setPageDefaults(res *types.Page) (err error) { func (e StoreEncoder) validatePage(*types.Page) (err error) { return } + +func (e StoreEncoder) setRecordDefaults(*types.Record) (err error) { + return +} + +func (e StoreEncoder) validateRecord(*types.Record) (err error) { + return +} + +func (e StoreEncoder) prepare(ctx context.Context, p envoyx.EncodeParams, s store.Storer, rt string, nn envoyx.NodeSet) (err error) { + switch rt { + case ComposeRecordDatasourceAuxType: + return e.prepareRecordDatasource(ctx, p, s, nn) + } + + return +} + +func (e StoreEncoder) encodeModuleExtend(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, nested envoyx.NodeSet, tree envoyx.Traverser) (err error) { + + // Push fields under mod + mod := n.Resource.(*types.Module) + for _, n := range nested { + if n.ResourceType != types.ModuleFieldResourceType { + continue + } + + mod.Fields = append(mod.Fields, n.Resource.(*types.ModuleField)) + } + + // Register to DAL + dl, err := e.grabDal(p) + if err != nil { + return + } + + nsNode := tree.ParentForRef(n, n.References["NamespaceID"]) + ns := nsNode.Resource.(*types.Namespace) + + // @todo get connection and things from there + model, err := service.ModuleToModel(ns, mod, "compose_record") + if err != nil { + return err + } + + return dl.ReplaceModel(ctx, model) +} + +func (e StoreEncoder) encodeModuleExtendSubResources(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) { + cc := tree.ChildrenForResourceType(n, ComposeRecordDatasourceAuxType) + + dl, err := e.grabDal(p) + if err != nil { + return + } + + return e.encodeRecordDatasources(ctx, p, s, dl, cc, tree) +} + +func (e *StoreEncoder) grabDal(p envoyx.EncodeParams) (dl dal.FullService, err error) { + auxs, ok := p.Params["dal"] + if !ok { + err = fmt.Errorf("dal not defined") + return + } + + dl, ok = auxs.(dal.FullService) + if !ok { + err = fmt.Errorf("invalid dal provided") + return + } + + return +} diff --git a/server/compose/envoy/store_encode_datasource.go b/server/compose/envoy/store_encode_datasource.go new file mode 100644 index 000000000..f2636bee7 --- /dev/null +++ b/server/compose/envoy/store_encode_datasource.go @@ -0,0 +1,208 @@ +package envoy + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/cortezaproject/corteza/server/compose/dalutils" + "github.com/cortezaproject/corteza/server/compose/types" + "github.com/cortezaproject/corteza/server/pkg/dal" + "github.com/cortezaproject/corteza/server/pkg/envoyx" + "github.com/cortezaproject/corteza/server/pkg/id" + "github.com/cortezaproject/corteza/server/store" +) + +func (e StoreEncoder) prepareRecordDatasource(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) { + // @todo match existing records; for now use just the ID like V1 + + for _, n := range nn { + if n.Datasource == nil { + panic("unexpected state: cannot call prepareRecordDatasource with nodes without a defined Datasource") + } + + ds, ok := n.Datasource.(*RecordDatasource) + if !ok { + panic("unexpected datasource type: node expecting type of RecordDatasource") + } + + err = e.prepareRecords(ctx, p, s, ds) + if err != nil { + return + } + err = ds.Reset() + if err != nil { + return + } + } + + return +} + +func (e StoreEncoder) prepareRecords(ctx context.Context, p envoyx.EncodeParams, s store.Storer, ds *RecordDatasource) (err error) { + var ( + aux = make(map[string]string) + more bool + ident string + rec types.Record + ) + + ds.refToID = make(map[string]uint64) + + for { + ident, more, err = ds.Next(aux) + if err != nil || !more { + return + } + + ds.refToID[ident] = id.Next() + + rec, err = e.auxToRecord(aux) + if err != nil { + return err + } + + // @note defaults and validation will have to happen again when encoding + // since we won't persist it. + // Consider supporting updating the datasource's data. + err = e.setRecordDefaults(&rec) + if err != nil { + return err + } + + err = e.validateRecord(&rec) + if err != nil { + return err + } + } +} + +func (e StoreEncoder) encodeRecordDatasources(ctx context.Context, p envoyx.EncodeParams, s store.Storer, dl dal.FullService, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) { + for _, n := range nn { + err = e.encodeRecordDatasource(ctx, p, s, dl, n, tree) + if err != nil { + return + } + } + + return +} + +func (e StoreEncoder) encodeRecordDatasource(ctx context.Context, p envoyx.EncodeParams, s store.Storer, dl dal.FullService, n *envoyx.Node, tree envoyx.Traverser) (err error) { + var ( + auxRec = make(map[string]string) + more bool + ident string + rec types.Record + + nsNode *envoyx.Node + ns *types.Namespace + modNode *envoyx.Node + mod *types.Module + + // This was already validated so we can blindly cast + ds = n.Datasource.(*RecordDatasource) + ) + + // Get the parent namespace + nsNode = tree.ParentForRef(n, n.References["NamespaceID"]) + if nsNode == nil { + err = fmt.Errorf("cannot encode record datasource: missing reference for NamespaceID") + return + } + ns = nsNode.Resource.(*types.Namespace) + + // Get the parent module + modNode = tree.ParentForRef(n, n.References["ModuleID"]) + if modNode == nil { + err = fmt.Errorf("cannot encode record datasource: missing reference for ModuleID") + return + } + mod = modNode.Resource.(*types.Module) + + // Prepare getters for reference fields + // @todo user refs + var ( + modIndex = make(map[string]envoyx.Ref) + dsIndex = make(map[string]envoyx.Ref) + ) + // - first pass collects all the references + for k, v := range n.References { + if k == "NamespaceID" || k == "ModuleID" { + continue + } + + // Encoded as . + pp := strings.Split(k, ".") + f := pp[0] + kind := pp[1] + + switch kind { + case "module": + modIndex[f] = v + case "datasource": + dsIndex[f] = v + } + } + + // - second pass makes the getters + getters := make(map[string]*recordGetter) + for k := range modIndex { + aux := makeRecordGetter(dl, tree, n, modIndex[k], dsIndex[k]) + getters[k] = aux + } + + // Iterate and encode + // + // @todo utilize batching + for { + ident, more, err = ds.Next(auxRec) + if err != nil || !more { + return + } + + rec, err = e.auxToRecord(auxRec) + if err != nil { + return err + } + + rec.NamespaceID = ns.ID + rec.ModuleID = mod.ID + + // @todo temp + rec.CreatedAt = time.Now() + + // Values and refs + rec.ID = ds.refToID[ident] + for i, v := range rec.Values { + if getters[v.Name] == nil { + continue + } + + v.Ref, err = getters[v.Name].resolve(ctx, v.Value) + if err != nil { + return err + } + + rec.Values[i] = v + } + + // Save it + err = dalutils.ComposeRecordCreate(ctx, dl, mod, &rec) + if err != nil { + return + } + } +} + +func (e StoreEncoder) auxToRecord(aux map[string]string) (out types.Record, err error) { + for k, v := range aux { + err = out.SetValue(k, 0, v) + if err != nil { + return + } + } + + return +} diff --git a/server/compose/envoy/yaml_decode.gen.go b/server/compose/envoy/yaml_decode.gen.go index 24050e22c..ffec37557 100644 --- a/server/compose/envoy/yaml_decode.gen.go +++ b/server/compose/envoy/yaml_decode.gen.go @@ -361,6 +361,7 @@ func (d *auxYamlDoc) unmarshalChartNode(dctx documentContext, n *yaml.Node, meta // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil @@ -388,17 +389,24 @@ func (d *auxYamlDoc) unmarshalChartNode(dctx documentContext, n *yaml.Node, meta Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, @@ -473,6 +481,22 @@ func (d *auxYamlDoc) unmarshalModuleMap(dctx documentContext, n *yaml.Node) (out return } +// unmarshalSourceExtendedSeq unmarshals Source when provided as a sequence node +func (d *auxYamlDoc) unmarshalExtendedSourceSeq(dctx documentContext, n *yaml.Node) (out envoyx.NodeSet, err error) { + var aux envoyx.NodeSet + err = y7s.EachSeq(n, func(n *yaml.Node) error { + aux, err = d.unmarshalSourceExtendedNode(dctx, n) + if err != nil { + return err + } + out = append(out, aux...) + + return nil + }) + + return +} + // unmarshalModuleNode is a cookie-cutter function to unmarshal // the yaml node into the corresponding Corteza type & Node func (d *auxYamlDoc) unmarshalModuleNode(dctx documentContext, n *yaml.Node, meta ...*yaml.Node) (out envoyx.NodeSet, err error) { @@ -615,11 +639,34 @@ func (d *auxYamlDoc) unmarshalModuleNode(dctx documentContext, n *yaml.Node, met // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil switch strings.ToLower(k.Value) { + case "modulefield", "module_fields", "modulefields", "fields": + if y7s.IsSeq(n) { + nestedNodes, err = d.unmarshalModuleFieldSeq(dctx, n) + if err != nil { + return err + } + } else { + nestedNodes, err = d.unmarshalModuleFieldMap(dctx, n) + if err != nil { + return err + } + } + break + + case "source", "datasource": + if y7s.IsSeq(n) { + nestedNodes, err = d.unmarshalExtendedSourceSeq(dctx, n) + if err != nil { + return err + } + } + break } // Iterate nested nodes and update their reference to the current resource @@ -642,17 +689,29 @@ func (d *auxYamlDoc) unmarshalModuleNode(dctx documentContext, n *yaml.Node, met Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + auxNestedNodes, err = d.postProcessNestedModuleNodes(auxNestedNodes) + if err != nil { + return + } + + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, @@ -802,6 +861,25 @@ func (d *auxYamlDoc) unmarshalModuleFieldNode(dctx documentContext, n *yaml.Node break + case "options": + + // Handle custom node decoder + // + // The decoder may update the passed resource with arbitrary values + // as well as provide additional references and identifiers for the node. + var ( + auxRefs map[string]envoyx.Ref + auxIdents envoyx.Identifiers + ) + auxRefs, auxIdents, err = unmarshalModuleFieldOptionsNode(r, n) + if err != nil { + return err + } + refs = envoyx.MergeRefs(refs, auxRefs) + ii = ii.Merge(auxIdents) + + break + // Handle RBAC rules case "allow": auxOut, err = unmarshalAllowNode(n) @@ -862,6 +940,7 @@ func (d *auxYamlDoc) unmarshalModuleFieldNode(dctx documentContext, n *yaml.Node // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil @@ -889,17 +968,24 @@ func (d *auxYamlDoc) unmarshalModuleFieldNode(dctx documentContext, n *yaml.Node Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, @@ -1096,11 +1182,68 @@ func (d *auxYamlDoc) unmarshalNamespaceNode(dctx documentContext, n *yaml.Node, // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil switch strings.ToLower(k.Value) { + case "chart", "charts", "chrt": + if y7s.IsSeq(n) { + nestedNodes, err = d.unmarshalChartSeq(dctx, n) + if err != nil { + return err + } + } else { + nestedNodes, err = d.unmarshalChartMap(dctx, n) + if err != nil { + return err + } + } + break + + case "module", "modules", "mod": + if y7s.IsSeq(n) { + nestedNodes, err = d.unmarshalModuleSeq(dctx, n) + if err != nil { + return err + } + } else { + nestedNodes, err = d.unmarshalModuleMap(dctx, n) + if err != nil { + return err + } + } + break + + case "modulefield", "module_fields", "modulefields", "fields": + if y7s.IsSeq(n) { + nestedNodes, err = d.unmarshalModuleFieldSeq(dctx, n) + if err != nil { + return err + } + } else { + nestedNodes, err = d.unmarshalModuleFieldMap(dctx, n) + if err != nil { + return err + } + } + break + + case "page", "pages", "pg": + if y7s.IsSeq(n) { + nestedNodes, err = d.unmarshalPageSeq(dctx, n) + if err != nil { + return err + } + } else { + nestedNodes, err = d.unmarshalPageMap(dctx, n) + if err != nil { + return err + } + } + break + } // Iterate nested nodes and update their reference to the current resource @@ -1123,17 +1266,24 @@ func (d *auxYamlDoc) unmarshalNamespaceNode(dctx documentContext, n *yaml.Node, Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, @@ -1369,6 +1519,7 @@ func (d *auxYamlDoc) unmarshalPageNode(dctx documentContext, n *yaml.Node, meta // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil @@ -1396,17 +1547,24 @@ func (d *auxYamlDoc) unmarshalPageNode(dctx documentContext, n *yaml.Node, meta Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, diff --git a/server/compose/envoy/yaml_decode.go b/server/compose/envoy/yaml_decode.go index 0ea431d19..bf302a468 100644 --- a/server/compose/envoy/yaml_decode.go +++ b/server/compose/envoy/yaml_decode.go @@ -10,6 +10,30 @@ import ( "gopkg.in/yaml.v3" ) +type ( + mapEntry struct { + Column string + Field string + Skip bool + } + + fieldMapping struct { + m map[string]mapEntry + } + + datasourceMapping struct { + SourceIdent string `yaml:"source"` + KeyField string `yaml:"key"` + References map[string]string + Scope map[string]string + Mapping fieldMapping + } +) + +const ( + ComposeRecordDatasourceAuxType = "corteza::compose:record-datasource" +) + func unmarshalChartConfigNode(r *types.Chart, n *yaml.Node) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) { err = y7s.EachMap(n, func(k, v *yaml.Node) error { if k.Value != "reports" { @@ -61,6 +85,177 @@ func unmarshalChartConfigReportNode(r *types.Chart, n *yaml.Node, index int) (re return } +func unmarshalModuleFieldOptionsNode(r *types.ModuleField, n *yaml.Node) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) { + refs = make(map[string]envoyx.Ref) + + err = y7s.EachMap(n, func(k, v *yaml.Node) error { + switch strings.ToLower(k.Value) { + case "module", "module_id", "moduleid": + var aux any + err = y7s.DecodeScalar(v, "moduleID", &aux) + if err != nil { + return err + } + refs["Options.ModuleID"] = envoyx.Ref{ + ResourceType: types.ModuleResourceType, + Identifiers: envoyx.MakeIdentifiers(aux), + } + default: + return nil + } + return nil + }) + + return +} + func (d *auxYamlDoc) unmarshalYAML(k string, n *yaml.Node) (out envoyx.NodeSet, err error) { return } + +func (d *auxYamlDoc) postProcessNestedModuleNodes(nn envoyx.NodeSet) (out envoyx.NodeSet, err error) { + // Get all references from all module fields + refs := make(map[string]envoyx.Ref) + for _, n := range nn { + if n.ResourceType != types.ModuleFieldResourceType { + continue + } + + if len(n.References) == 0 { + continue + } + + r, ok := n.References["Options.ModuleID"] + if !ok { + continue + } + + refs[fmt.Sprintf("%s.module", n.Identifiers.FriendlyIdentifier())] = r + refs[fmt.Sprintf("%s.datasource", n.Identifiers.FriendlyIdentifier())] = envoyx.Ref{ + ResourceType: ComposeRecordDatasourceAuxType, + Identifiers: r.Identifiers, + Scope: r.Scope, + Optional: true, + } + } + + // Update datasources with references + + for _, n := range nn { + if n.Datasource == nil { + continue + } + + n.References = envoyx.MergeRefs(n.References, refs) + } + out = nn + return +} + +func (d *auxYamlDoc) unmarshalSourceExtendedNode(dctx documentContext, n *yaml.Node, meta ...*yaml.Node) (out envoyx.NodeSet, err error) { + var r datasourceMapping + + // @todo we're omitting errors because there will be a bunch due to invalid + // resource field types. This might be a bit unstable as other errors may + // also get ignored. + // + // A potential fix would be to firstly unmarshal into an any, check errors + // and then unmarshal into the resource while omitting errors. + n.Decode(&r) + + // @todo for now we only support record datasources; extend when needed + auxN := &envoyx.Node{ + Datasource: &RecordDatasource{ + mapping: r, + }, + + ResourceType: ComposeRecordDatasourceAuxType, + Identifiers: envoyx.MakeIdentifiers(r.SourceIdent), + } + auxN.References, auxN.Scope = d.procMappingRefs(r.References) + out = append(out, auxN) + + return +} + +// UnmarshalYAML is used to get the yaml parsed into a series of nodes so +// we can easily pass it down +func (d *fieldMapping) UnmarshalYAML(n *yaml.Node) (err error) { + d.m = make(map[string]mapEntry) + if y7s.IsSeq(n) { + err = y7s.EachSeq(n, func(n *yaml.Node) error { + a, err := d.unmarshalMappingNode(n) + d.m[a.Column] = a + return err + }) + } else { + err = y7s.EachMap(n, func(k, n *yaml.Node) error { + a, err := d.unmarshalMappingNode(n) + if a.Column == "" { + err = y7s.DecodeScalar(k, "fieldMapping column", &a.Column) + if err != nil { + return err + } + } + + d.m[a.Column] = a + return err + }) + } + + return +} + +func (d *fieldMapping) unmarshalMappingNode(n *yaml.Node) (out mapEntry, err error) { + if y7s.IsKind(n, yaml.ScalarNode) { + err = y7s.DecodeScalar(n, "Column", &out.Column) + if err != nil { + return + } + err = y7s.DecodeScalar(n, "Field", &out.Field) + return + } + + // @todo we're omitting errors because there will be a bunch due to invalid + // resource field types. This might be a bit unstable as other errors may + // also get ignored. + // + // A potential fix would be to firstly unmarshal into an any, check errors + // and then unmarshal into the resource while omitting errors. + n.Decode(&out) + + err = y7s.EachMap(n, func(k, v *yaml.Node) error { + switch strings.ToLower(k.Value) { + case "skip": + if v.Value == "/" { + out.Skip = true + } + } + return nil + }) + + return +} + +func (d *auxYamlDoc) procMappingRefs(in map[string]string) (out map[string]envoyx.Ref, scope envoyx.Scope) { + out = make(map[string]envoyx.Ref) + + scope = envoyx.Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: envoyx.MakeIdentifiers(in["namespace"]), + } + + out["NamespaceID"] = envoyx.Ref{ + ResourceType: types.NamespaceResourceType, + Identifiers: envoyx.MakeIdentifiers(in["namespace"]), + Scope: scope, + } + + out["ModuleID"] = envoyx.Ref{ + ResourceType: types.ModuleResourceType, + Identifiers: envoyx.MakeIdentifiers(in["module"]), + Scope: scope, + } + + return +} diff --git a/server/compose/envoy/yaml_encode.gen.go b/server/compose/envoy/yaml_encode.gen.go index 16761ae03..767a50d67 100644 --- a/server/compose/envoy/yaml_encode.gen.go +++ b/server/compose/envoy/yaml_encode.gen.go @@ -238,6 +238,17 @@ func (e YamlEncoder) encodeModule(ctx context.Context, p envoyx.EncodeParams, no var aux *yaml.Node _ = aux + aux, err = e.encodeModuleFields(ctx, p, tt.ChildrenForResourceType(node, types.ModuleFieldResourceType), tt) + if err != nil { + return + } + out, err = y7s.AddMap(out, + "moduleField", aux, + ) + if err != nil { + return + } + return } @@ -374,6 +385,39 @@ func (e YamlEncoder) encodeNamespace(ctx context.Context, p envoyx.EncodeParams, var aux *yaml.Node _ = aux + aux, err = e.encodeCharts(ctx, p, tt.ChildrenForResourceType(node, types.ChartResourceType), tt) + if err != nil { + return + } + out, err = y7s.AddMap(out, + "chart", aux, + ) + if err != nil { + return + } + + aux, err = e.encodeModules(ctx, p, tt.ChildrenForResourceType(node, types.ModuleResourceType), tt) + if err != nil { + return + } + out, err = y7s.AddMap(out, + "module", aux, + ) + if err != nil { + return + } + + aux, err = e.encodePages(ctx, p, tt.ChildrenForResourceType(node, types.PageResourceType), tt) + if err != nil { + return + } + out, err = y7s.AddMap(out, + "page", aux, + ) + if err != nil { + return + } + return } diff --git a/server/compose/module.cue b/server/compose/module.cue index 8f962f91d..aea6fc359 100644 --- a/server/compose/module.cue +++ b/server/compose/module.cue @@ -89,8 +89,18 @@ module: { supportMappedInput: true mappedField: "Handle" identKeyAlias: ["modules", "mod"] + + extendedResourcePostProcess: true + extendedResourceDecoders: [{ + ident: "source" + expIdent: "Source" + identKeys: ["source", "datasource"] + supportMappedInput: false + }] } store: { + extendedEncoder: true + extendedSubResources: true extendedFilterBuilder: true extendedDecoder: true } diff --git a/server/compose/module_field.cue b/server/compose/module_field.cue index efac0c7f1..a3ad850d1 100644 --- a/server/compose/module_field.cue +++ b/server/compose/module_field.cue @@ -39,6 +39,11 @@ moduleField: { dal: { type: "JSON", defaultEmptyObject: true } omitSetter: true omitGetter: true + envoy: { + yaml: { + customDecoder: true + } + } } name: { sortable: true diff --git a/server/compose/page.cue b/server/compose/page.cue index 6f37a6fb7..ee0d53b83 100644 --- a/server/compose/page.cue +++ b/server/compose/page.cue @@ -92,7 +92,7 @@ page: { filter: { struct: { - page_id: { goType: "uint64", ident: "pageID", storeIdent: "id" } + page_id: { goType: "[]uint64", ident: "pageID", storeIdent: "id" } namespace_id: { goType: "uint64", ident: "namespaceID", storeIdent: "rel_namespace" } parent_id: { goType: "uint64", ident: "parentID" } module_id: { goType: "uint64", ident: "moduleID", storeIdent: "rel_module" } diff --git a/server/compose/record.cue b/server/compose/record.cue index 47b034060..f83d7be2b 100644 --- a/server/compose/record.cue +++ b/server/compose/record.cue @@ -13,6 +13,9 @@ record: { model: { ident: "compose_record" + defaultSetter: true + defaultGetter: true + attributes: { id: schema.IdField revision: { @@ -71,9 +74,6 @@ record: { omit: true } - defaultGetter: true - defaultSetter: true - filter: { struct: { module_id: { goType: "uint64" } diff --git a/server/compose/types/getters_setters.gen.go b/server/compose/types/getters_setters.gen.go index 04e55c249..bbd0526c0 100644 --- a/server/compose/types/getters_setters.gen.go +++ b/server/compose/types/getters_setters.gen.go @@ -351,6 +351,9 @@ func (r *Record) GetValue(name string, pos uint) (any, error) { case "updatedBy", "UpdatedBy": return r.UpdatedBy, nil + default: + return r.getValue(name, pos) + } return nil, nil } @@ -382,6 +385,9 @@ func (r *Record) SetValue(name string, pos uint, value any) (err error) { case "updatedBy", "UpdatedBy": return cast2.Uint64(value, &r.UpdatedBy) + default: + return r.setValue(name, pos, value) + } return nil } diff --git a/server/pkg/envoyx/csv/decoder.go b/server/pkg/envoyx/csv/decoder.go index 9cf372b7b..0622b7597 100644 --- a/server/pkg/envoyx/csv/decoder.go +++ b/server/pkg/envoyx/csv/decoder.go @@ -57,7 +57,7 @@ func Decoder(r io.Reader, ident string) (out *decoder, err error) { } r, err = out.flushTemp(r) - defer out.src.Seek(0, 0) + defer out.Reset() if err != nil { return } @@ -111,12 +111,12 @@ func (d *decoder) Reset() error { // Next returns the field: value mapping for the next row func (d *decoder) Next(out map[string]string) (more bool, err error) { - if !d.skipHead { + if d.skipHead { _, err = d.reader.Read() if err != nil { return } - d.skipHead = true + d.skipHead = false } aux, err := d.reader.Read() diff --git a/server/pkg/envoyx/datasource.go b/server/pkg/envoyx/datasource.go new file mode 100644 index 000000000..70d255420 --- /dev/null +++ b/server/pkg/envoyx/datasource.go @@ -0,0 +1,29 @@ +package envoyx + +type ( + Provider interface { + Next(out map[string]string) (more bool, err error) + Reset() error + Ident() string + } + + Datasource interface { + Next(out map[string]string) (ident string, more bool, err error) + Reset() error + SetProvider(Provider) bool + } +) + +func SetDecoderSources(nn NodeSet, dd ...Provider) { + for _, n := range nn { + if n.Datasource == nil { + continue + } + + for _, d := range dd { + if n.Datasource.SetProvider(d) { + break + } + } + } +} diff --git a/server/pkg/envoyx/envoy.go b/server/pkg/envoyx/envoy.go index f0e3b1373..31ae0c5fc 100644 --- a/server/pkg/envoyx/envoy.go +++ b/server/pkg/envoyx/envoy.go @@ -137,6 +137,8 @@ func (svc *service) Decode(ctx context.Context, p DecodeParams) (nn NodeSet, err return svc.decodeUri(ctx, p) case DecodeTypeStore: return svc.decodeStore(ctx, p) + default: + err = fmt.Errorf("unsupported decoder type %s", p.Type) } return @@ -156,7 +158,8 @@ func (svc *service) Encode(ctx context.Context, p EncodeParams, dg *depGraph) (e return svc.encodeStore(ctx, dg, p) case EncodeTypeIo: return svc.encodeIo(ctx, dg, p) - + default: + err = fmt.Errorf("unsupported encoder type %s", p.Type) } return } diff --git a/server/pkg/envoyx/node.go b/server/pkg/envoyx/node.go index 563521113..c5cd396a7 100644 --- a/server/pkg/envoyx/node.go +++ b/server/pkg/envoyx/node.go @@ -9,7 +9,8 @@ import ( type ( // Node is a wrapper around a Corteza resource for use within Envoy Node struct { - Resource resource + Resource resource + Datasource Datasource ResourceType string Identifiers Identifiers @@ -57,6 +58,9 @@ type ( ResourceType string Identifiers Identifiers Scope Scope + // @todo consider replacing with something that indicates + // it can't be fetched from the DB + Optional bool } ) diff --git a/server/pkg/envoyx/util_rbac.gen.go b/server/pkg/envoyx/util_rbac.gen.go index 87e4a11f2..a4edd8c8d 100644 --- a/server/pkg/envoyx/util_rbac.gen.go +++ b/server/pkg/envoyx/util_rbac.gen.go @@ -275,6 +275,11 @@ func SplitResourceIdentifier(ref string) (out map[string]Ref) { return } + out["0"] = Ref{ + ResourceType: "corteza::compose:namespace", + Identifiers: MakeIdentifiers(gRef(pp, 0)), + } + if gRef(pp, 1) == "" { return } @@ -289,6 +294,11 @@ func SplitResourceIdentifier(ref string) (out map[string]Ref) { return } + out["0"] = Ref{ + ResourceType: "corteza::compose:namespace", + Identifiers: MakeIdentifiers(gRef(pp, 0)), + } + if gRef(pp, 1) == "" { return } @@ -303,10 +313,20 @@ func SplitResourceIdentifier(ref string) (out map[string]Ref) { return } + out["0"] = Ref{ + ResourceType: "corteza::compose:namespace", + Identifiers: MakeIdentifiers(gRef(pp, 0)), + } + if gRef(pp, 1) == "" { return } + out["1"] = Ref{ + ResourceType: "corteza::compose:module", + Identifiers: MakeIdentifiers(gRef(pp, 1)), + } + if gRef(pp, 2) == "" { return } @@ -331,6 +351,11 @@ func SplitResourceIdentifier(ref string) (out map[string]Ref) { return } + out["0"] = Ref{ + ResourceType: "corteza::compose:namespace", + Identifiers: MakeIdentifiers(gRef(pp, 0)), + } + if gRef(pp, 1) == "" { return } @@ -345,10 +370,20 @@ func SplitResourceIdentifier(ref string) (out map[string]Ref) { return } + out["0"] = Ref{ + ResourceType: "corteza::compose:namespace", + Identifiers: MakeIdentifiers(gRef(pp, 0)), + } + if gRef(pp, 1) == "" { return } + out["1"] = Ref{ + ResourceType: "corteza::compose:module", + Identifiers: MakeIdentifiers(gRef(pp, 1)), + } + if gRef(pp, 2) == "" { return } diff --git a/server/pkg/envoyx/utils.gen.go b/server/pkg/envoyx/utils.gen.go index f66e903d3..79a5b37bc 100644 --- a/server/pkg/envoyx/utils.gen.go +++ b/server/pkg/envoyx/utils.gen.go @@ -25,12 +25,14 @@ var ( "corteza::federation:exposed-module": true, "corteza::federation:module-mapping": true, - "corteza::federation:shared-module": true, + "corteza::federation:shared-module": true, + "corteza::compose:record-datasource": true, } // superNeedyResources is the second level of filtering in case the first // pass removes everything superNeedyResources = map[string]bool{ - "corteza::compose:module-field": true, + "corteza::compose:module-field": true, + "corteza::compose:record-datasource": true, } ) diff --git a/server/store/adapters/rdbms/filters.gen.go b/server/store/adapters/rdbms/filters.gen.go index a48addd29..fbe985e40 100644 --- a/server/store/adapters/rdbms/filters.gen.go +++ b/server/store/adapters/rdbms/filters.gen.go @@ -203,8 +203,8 @@ func ApigwFilterFilter(d drivers.Dialect, f systemType.ApigwFilterFilter) (ee [] ee = append(ee, expr) } - if f.ApigwFilterID > 0 { - ee = append(ee, goqu.C("id").Eq(f.ApigwFilterID)) + if len(f.ApigwFilterID) > 0 { + ee = append(ee, goqu.C("id").In(f.ApigwFilterID)) } if f.RouteID > 0 { @@ -231,8 +231,8 @@ func ApigwRouteFilter(d drivers.Dialect, f systemType.ApigwRouteFilter) (ee []go ee = append(ee, expr) } - if len(f.ApigwrouteID) > 0 { - ee = append(ee, goqu.C("id").In(f.ApigwrouteID)) + if len(f.ApigwRouteID) > 0 { + ee = append(ee, goqu.C("id").In(f.ApigwRouteID)) } if val := strings.TrimSpace(f.Route); len(val) > 0 { @@ -651,8 +651,8 @@ func ComposePageFilter(d drivers.Dialect, f composeType.PageFilter) (ee []goqu.E ee = append(ee, expr) } - if f.PageID > 0 { - ee = append(ee, goqu.C("id").Eq(f.PageID)) + if len(f.PageID) > 0 { + ee = append(ee, goqu.C("id").In(f.PageID)) } if val := strings.TrimSpace(f.Handle); len(val) > 0 { @@ -723,8 +723,8 @@ func DalConnectionFilter(d drivers.Dialect, f systemType.DalConnectionFilter) (e ee = append(ee, expr) } - if len(f.ConnectionID) > 0 { - ee = append(ee, goqu.C("id").In(f.ConnectionID)) + if len(f.DalConnectionID) > 0 { + ee = append(ee, goqu.C("id").In(f.DalConnectionID)) } if val := strings.TrimSpace(f.Handle); len(val) > 0 { @@ -751,8 +751,8 @@ func DalSensitivityLevelFilter(d drivers.Dialect, f systemType.DalSensitivityLev ee = append(ee, expr) } - if len(f.SensitivityLevelID) > 0 { - ee = append(ee, goqu.C("id").In(f.SensitivityLevelID)) + if len(f.DalSensitivityLevelID) > 0 { + ee = append(ee, goqu.C("id").In(f.DalSensitivityLevelID)) } if val := strings.TrimSpace(f.Handle); len(val) > 0 { @@ -1002,8 +1002,8 @@ func QueueFilter(d drivers.Dialect, f systemType.QueueFilter) (ee []goqu.Express ee = append(ee, expr) } - if f.QueueID > 0 { - ee = append(ee, goqu.C("id").Eq(f.QueueID)) + if len(f.QueueID) > 0 { + ee = append(ee, goqu.C("id").In(f.QueueID)) } if f.Query != "" { diff --git a/server/system/apigw_filter.cue b/server/system/apigw_filter.cue index c4af838d9..3f59b929b 100644 --- a/server/system/apigw_filter.cue +++ b/server/system/apigw_filter.cue @@ -69,7 +69,7 @@ apigw_filter: { filter: { struct: { - apigw_filter_id: {goType: "uint64", ident: "apigwFilterID", storeIdent: "id"} + apigw_filter_id: {goType: "[]uint64", ident: "apigwFilterID", storeIdent: "id"} route_id: {goType: "uint64", ident: "routeID", storeIdent: "rel_route"} deleted: {goType: "filter.State", storeIdent: "deleted_at"} disabled: {goType: "filter.State", storeIdent: "enabled"} diff --git a/server/system/apigw_route.cue b/server/system/apigw_route.cue index 245ec05e5..35ac31972 100644 --- a/server/system/apigw_route.cue +++ b/server/system/apigw_route.cue @@ -73,7 +73,7 @@ apigw_route: { filter: { struct: { - apigw_route_id: { goType: "[]uint64", ident: "apigwrouteID", storeIdent: "id" } + apigw_route_id: { goType: "[]uint64", ident: "apigwRouteID", storeIdent: "id" } route: {goType: "string", storeIdent: "id"} endpoint: {goType: "string"} method: {goType: "string"} diff --git a/server/system/dal_connection.cue b/server/system/dal_connection.cue index 67c7ca611..0f06a3908 100644 --- a/server/system/dal_connection.cue +++ b/server/system/dal_connection.cue @@ -42,7 +42,7 @@ dal_connection: { filter: { struct: { - dal_connection_id: {goType: "[]uint64", ident: "connectionID", storeIdent: "id"} + dal_connection_id: {goType: "[]uint64", ident: "dalConnectionID", storeIdent: "id"} handle: {goType: "string"} type: {goType: "string"} diff --git a/server/system/dal_sensitivity_level.cue b/server/system/dal_sensitivity_level.cue index 4ca1be6d6..2f1c7068c 100644 --- a/server/system/dal_sensitivity_level.cue +++ b/server/system/dal_sensitivity_level.cue @@ -36,7 +36,7 @@ dal_sensitivity_level: { filter: { struct: { - dal_sensitivity_level_id: {goType: "[]uint64", ident: "sensitivityLevelID", storeIdent: "id"} + dal_sensitivity_level_id: {goType: "[]uint64", ident: "dalSensitivityLevelID", storeIdent: "id"} handle: { goType: "string" } deleted: {goType: "filter.State", storeIdent: "deleted_at"} diff --git a/server/system/envoy/store_encode.gen.go b/server/system/envoy/store_encode.gen.go index cd4945edf..f8b45efc2 100644 --- a/server/system/envoy/store_encode.gen.go +++ b/server/system/envoy/store_encode.gen.go @@ -69,6 +69,8 @@ func (e StoreEncoder) Prepare(ctx context.Context, p envoyx.EncodeParams, rt str return e.prepareDalConnection(ctx, p, s, nn) case types.DalSensitivityLevelResourceType: return e.prepareDalSensitivityLevel(ctx, p, s, nn) + default: + return e.prepare(ctx, p, s, rt, nn) } return diff --git a/server/system/envoy/yaml_decode.gen.go b/server/system/envoy/yaml_decode.gen.go index 0b6757196..620424ab9 100644 --- a/server/system/envoy/yaml_decode.gen.go +++ b/server/system/envoy/yaml_decode.gen.go @@ -370,6 +370,7 @@ func (d *auxYamlDoc) unmarshalApplicationNode(dctx documentContext, n *yaml.Node // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil @@ -397,17 +398,24 @@ func (d *auxYamlDoc) unmarshalApplicationNode(dctx documentContext, n *yaml.Node Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, @@ -622,6 +630,7 @@ func (d *auxYamlDoc) unmarshalApigwRouteNode(dctx documentContext, n *yaml.Node, // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil @@ -649,17 +658,24 @@ func (d *auxYamlDoc) unmarshalApigwRouteNode(dctx documentContext, n *yaml.Node, Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, @@ -838,6 +854,7 @@ func (d *auxYamlDoc) unmarshalApigwFilterNode(dctx documentContext, n *yaml.Node // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil @@ -865,17 +882,24 @@ func (d *auxYamlDoc) unmarshalApigwFilterNode(dctx documentContext, n *yaml.Node Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, @@ -1091,6 +1115,7 @@ func (d *auxYamlDoc) unmarshalAuthClientNode(dctx documentContext, n *yaml.Node, // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil @@ -1118,17 +1143,24 @@ func (d *auxYamlDoc) unmarshalAuthClientNode(dctx documentContext, n *yaml.Node, Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, @@ -1351,6 +1383,7 @@ func (d *auxYamlDoc) unmarshalQueueNode(dctx documentContext, n *yaml.Node, meta // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil @@ -1378,17 +1411,24 @@ func (d *auxYamlDoc) unmarshalQueueNode(dctx documentContext, n *yaml.Node, meta Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, @@ -1615,6 +1655,7 @@ func (d *auxYamlDoc) unmarshalReportNode(dctx documentContext, n *yaml.Node, met // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil @@ -1642,17 +1683,24 @@ func (d *auxYamlDoc) unmarshalReportNode(dctx documentContext, n *yaml.Node, met Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, @@ -1825,6 +1873,7 @@ func (d *auxYamlDoc) unmarshalRoleNode(dctx documentContext, n *yaml.Node, meta // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil @@ -1852,17 +1901,24 @@ func (d *auxYamlDoc) unmarshalRoleNode(dctx documentContext, n *yaml.Node, meta Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, @@ -2048,6 +2104,7 @@ func (d *auxYamlDoc) unmarshalTemplateNode(dctx documentContext, n *yaml.Node, m // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil @@ -2075,17 +2132,24 @@ func (d *auxYamlDoc) unmarshalTemplateNode(dctx documentContext, n *yaml.Node, m Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, @@ -2258,6 +2322,7 @@ func (d *auxYamlDoc) unmarshalUserNode(dctx documentContext, n *yaml.Node, meta // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil @@ -2285,17 +2350,24 @@ func (d *auxYamlDoc) unmarshalUserNode(dctx documentContext, n *yaml.Node, meta Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, @@ -2507,6 +2579,7 @@ func (d *auxYamlDoc) unmarshalDalConnectionNode(dctx documentContext, n *yaml.No // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil @@ -2534,17 +2607,24 @@ func (d *auxYamlDoc) unmarshalDalConnectionNode(dctx documentContext, n *yaml.No Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, @@ -2739,6 +2819,7 @@ func (d *auxYamlDoc) unmarshalDalSensitivityLevelNode(dctx documentContext, n *y // This operation is done in the second pass of the document so we have // the complete context of the current resource; such as the identifier, // references, and scope. + var auxNestedNodes envoyx.NodeSet err = y7s.EachMap(n, func(k, n *yaml.Node) error { nestedNodes = nil @@ -2766,17 +2847,24 @@ func (d *auxYamlDoc) unmarshalDalSensitivityLevelNode(dctx documentContext, n *y Scope: scope, } + for f, ref := range a.References { + ref.Scope = scope + a.References[f] = ref + } + for f, ref := range refs { a.References[f] = ref } } - auxOut = append(auxOut, nestedNodes...) + auxNestedNodes = append(auxNestedNodes, nestedNodes...) return nil }) if err != nil { return } + out = append(out, auxNestedNodes...) + a := &envoyx.Node{ Resource: r, diff --git a/server/system/queue.cue b/server/system/queue.cue index b41316d29..6dfad3d7f 100644 --- a/server/system/queue.cue +++ b/server/system/queue.cue @@ -48,7 +48,7 @@ queue: { filter: { struct: { - queue_id: {goType: "uint64", ident: "queueID", storeIdent: "id"} + queue_id: {goType: "[]uint64", ident: "queueID", storeIdent: "id"} query: {goType: "string"} deleted: {goType: "filter.State", storeIdent: "deleted_at"} } diff --git a/server/system/types/user.go b/server/system/types/user.go index 71349fadb..02ac490ea 100644 --- a/server/system/types/user.go +++ b/server/system/types/user.go @@ -3,6 +3,7 @@ package types import ( "database/sql/driver" "encoding/json" + "fmt" "time" "github.com/cortezaproject/corteza/server/pkg/sql" @@ -117,9 +118,9 @@ const ( SystemUser UserKind = "sys" ) -// func (u User) String() string { -// return fmt.Sprintf("%d", u.ID) -// } +func (u User) String() string { + return fmt.Sprintf("%d", u.ID) +} func (u *User) Valid() bool { return u.ID > 0 && u.SuspendedAt == nil && u.DeletedAt == nil