Add missing bits from V1, slight touchup

This commit is contained in:
Tomaž Jerman
2023-03-17 10:58:46 +01:00
parent a8b60c6525
commit 2e9ba97dda
37 changed files with 1602 additions and 72 deletions
+8 -1
View File
@@ -12,7 +12,13 @@ chart: {
model: {
ident: "compose_chart"
attributes: {
id: schema.IdField
id: schema.IdField & {
envoy: {
yaml: {
identKeyEncode: "chartID"
}
}
}
handle: schema.HandleField
namespace_id: {
ident: "namespaceID",
@@ -37,6 +43,7 @@ chart: {
envoy: {
yaml: {
customDecoder: true
customEncoder: true
}
}
}
+60 -6
View File
@@ -2,6 +2,7 @@ package envoy
import (
"context"
"strings"
"github.com/cortezaproject/corteza/server/pkg/dal"
"github.com/cortezaproject/corteza/server/pkg/envoyx"
@@ -43,7 +44,7 @@ func (rd *RecordDatasource) SetProvider(s envoyx.Provider) bool {
return true
}
func (rd *RecordDatasource) Next(ctx context.Context, out map[string]string) (ident string, more bool, err error) {
func (rd *RecordDatasource) Next(ctx context.Context, out map[string]string) (ident []string, more bool, err error) {
if rd.rowCache == nil {
rd.rowCache = make(map[string]string)
}
@@ -55,7 +56,9 @@ func (rd *RecordDatasource) Next(ctx context.Context, out map[string]string) (id
rd.applyMapping(rd.rowCache, out)
ident = out[rd.mapping.KeyField]
for _, k := range rd.mapping.KeyField {
ident = append(ident, out[k])
}
return
}
@@ -66,32 +69,83 @@ func (rd *RecordDatasource) Reset(ctx context.Context) (err error) {
func (rd *RecordDatasource) applyMapping(in, out map[string]string) {
if len(rd.mapping.Mapping.m) == 0 {
if !rd.mapping.Defaultable {
return
}
for k, v := range in {
out[k] = v
}
return
}
if rd.mapping.Defaultable {
rd.applyMappingWithDefaults(in, out)
} else {
rd.applyMappingWoDefaults(in, out)
}
}
func (rd *RecordDatasource) applyMappingWithDefaults(in, out map[string]string) {
maps := make(map[string]mapEntry)
for k, v := range rd.mapping.Mapping.m {
maps[k] = v
}
for k, v := range in {
if m, ok := maps[k]; ok {
if m.Skip {
continue
}
out[m.Field] = v
} else {
out[k] = v
}
}
}
func (rd *RecordDatasource) applyMappingWoDefaults(in, out map[string]string) {
for _, m := range rd.mapping.Mapping.m {
if m.Skip {
continue
}
// @todo expand when needed (expressions and such)
out[m.Field] = in[m.Column]
}
}
func (rd *RecordDatasource) ResolveRef(ref any) (out uint64, err error) {
r, err := cast.ToStringE(ref)
func (rd *RecordDatasource) ResolveRef(ref ...any) (out uint64, err error) {
idents, err := cast.ToStringSliceE(ref)
if err != nil {
return
}
out = rd.refToID[r]
for i, ident := range idents {
idents[i] = strings.Replace(ident, "-", "_", -1)
}
out = rd.refToID[strings.Join(idents, "-")]
return
}
func (rd *RecordDatasource) ResolveRefS(ref ...string) (out uint64, err error) {
aux := make([]any, len(ref))
for i, r := range ref {
aux[i] = r
}
return rd.ResolveRef(aux...)
}
// @todo this should be replaced by some smarter structure
func (rd *RecordDatasource) AddRef(id uint64, idents ...string) {
for i, ident := range idents {
idents[i] = strings.Replace(ident, "-", "_", -1)
}
rd.refToID[strings.Join(idents, "-")] = id
}
func (ar auxRecord) SetValue(name string, pos uint, value any) (err error) {
ar[name] = cast.ToString(value)
return
+4
View File
@@ -395,6 +395,8 @@ func (d StoreDecoder) decodeModuleField(ctx context.Context, s store.Storer, dl
},
}
refs = envoyx.MergeRefs(refs, d.decodeModuleFieldRefs(r))
var scope envoyx.Scope
scope = envoyx.Scope{
@@ -524,6 +526,8 @@ func (d StoreDecoder) decodePage(ctx context.Context, s store.Storer, dl dal.Ful
},
}
refs = envoyx.MergeRefs(refs, d.decodePageRefs(r))
var scope envoyx.Scope
scope = envoyx.Scope{
+70 -14
View File
@@ -87,38 +87,94 @@ func (d StoreDecoder) makeModuleFieldFilter(scope *envoyx.Node, refs map[string]
}
func (d StoreDecoder) extendedModuleDecoder(ctx context.Context, s store.Storer, dl dal.FullService, f types.ModuleFilter, base envoyx.NodeSet) (out envoyx.NodeSet, err error) {
var ff types.ModuleFieldSet
var ff envoyx.NodeSet
for _, b := range base {
mod := b.Resource.(*types.Module)
ff, _, err = store.SearchComposeModuleFields(ctx, s, types.ModuleFieldFilter{ModuleID: []uint64{b.Resource.GetID()}})
// Get all of the related module fields, append them to the output and
// the original module (so other code can have access to the related fields)
ff, err = d.decodeModuleField(ctx, s, dl, types.ModuleFieldFilter{ModuleID: []uint64{mod.ID}})
if err != nil {
return
}
// No need to assign them under the module since we're working with nodes now
for _, f := range ff {
out = append(out, &envoyx.Node{
Resource: f,
ResourceType: types.ModuleFieldResourceType,
Identifiers: envoyx.MakeIdentifiers(f.ID, f.Name),
References: envoyx.MergeRefs(b.References, map[string]envoyx.Ref{
"ModuleID": b.ToRef(),
}),
Scope: b.Scope,
f.Scope = b.Scope
f.References = envoyx.MergeRefs(b.References, map[string]envoyx.Ref{
"ModuleID": b.ToRef(),
})
mod.Fields = append(mod.Fields, f)
mod.Fields = append(mod.Fields, f.Resource.(*types.ModuleField))
}
out = append(out, ff...)
}
return
}
func (d StoreDecoder) decodeChartRefs(c *types.Chart) (refs map[string]envoyx.Ref) {
refs = make(map[string]envoyx.Ref, len(c.Config.Reports))
for i, r := range c.Config.Reports {
if r.ModuleID == 0 {
continue
}
refs[fmt.Sprintf("Config.Reports.%d.ModuleID", i)] = envoyx.Ref{
ResourceType: types.ModuleResourceType,
Identifiers: envoyx.MakeIdentifiers(r.ModuleID),
}
}
return
}
func (d StoreDecoder) decodeModuleFieldRefs(c *types.ModuleField) (refs map[string]envoyx.Ref) {
refs = make(map[string]envoyx.Ref, 1)
id := c.Options.UInt64("moduleID")
if id == 0 {
return
}
refs["Options.ModuleID"] = envoyx.Ref{
ResourceType: types.ModuleResourceType,
Identifiers: envoyx.MakeIdentifiers(id),
}
return
}
func (d StoreDecoder) decodePageRefs(p *types.Page) (refs map[string]envoyx.Ref) {
refs = make(map[string]envoyx.Ref, len(p.Blocks)/2)
for index, b := range p.Blocks {
switch b.Kind {
case "RecordList":
refs = envoyx.MergeRefs(refs, getPageBlockRecordListRefs(b, index))
case "Automation":
refs = envoyx.MergeRefs(refs, getPageBlockAutomationRefs(b, index))
case "RecordOrganizer":
refs = envoyx.MergeRefs(refs, getPageBlockRecordOrganizerRefs(b, index))
case "Chart":
refs = envoyx.MergeRefs(refs, getPageBlockChartRefs(b, index))
case "Calendar":
refs = envoyx.MergeRefs(refs, getPageBlockCalendarRefs(b, index))
case "Metric":
refs = envoyx.MergeRefs(refs, getPageBlockMetricRefs(b, index))
case "Comment":
refs = envoyx.MergeRefs(refs, getPageBlockCommentRefs(b, index))
}
}
// @todo
return
}
+52
View File
@@ -3,6 +3,7 @@ package envoy
import (
"context"
"fmt"
"time"
"github.com/cortezaproject/corteza/server/compose/service"
"github.com/cortezaproject/corteza/server/compose/types"
@@ -12,6 +13,9 @@ import (
)
func (e StoreEncoder) setChartDefaults(res *types.Chart) (err error) {
if res.CreatedAt.IsZero() {
res.CreatedAt = time.Now()
}
return
}
@@ -20,6 +24,9 @@ func (e StoreEncoder) validateChart(*types.Chart) (err error) {
}
func (e StoreEncoder) setModuleDefaults(res *types.Module) (err error) {
if res.CreatedAt.IsZero() {
res.CreatedAt = time.Now()
}
return
}
@@ -28,6 +35,25 @@ func (e StoreEncoder) validateModule(*types.Module) (err error) {
}
func (e StoreEncoder) setModuleFieldDefaults(res *types.ModuleField) (err error) {
if res.CreatedAt.IsZero() {
res.CreatedAt = time.Now()
}
// Update validator ID
maxValidatorID := uint64(0)
for _, v := range res.Expressions.Validators {
if v.ValidatorID > maxValidatorID {
maxValidatorID = v.ValidatorID
}
}
for _, v := range res.Expressions.Validators {
if v.ValidatorID == 0 {
v.ValidatorID = maxValidatorID + 1
maxValidatorID++
}
}
return
}
@@ -36,6 +62,9 @@ func (e StoreEncoder) validateModuleField(*types.ModuleField) (err error) {
}
func (e StoreEncoder) setNamespaceDefaults(res *types.Namespace) (err error) {
if res.CreatedAt.IsZero() {
res.CreatedAt = time.Now()
}
return
}
@@ -44,6 +73,29 @@ func (e StoreEncoder) validateNamespace(*types.Namespace) (err error) {
}
func (e StoreEncoder) setPageDefaults(res *types.Page) (err error) {
if res.CreatedAt.IsZero() {
res.CreatedAt = time.Now()
}
if res.Title == "" {
res.Title = res.Handle
}
// Update pageblock ID
maxPageBlockID := uint64(0)
for _, b := range res.Blocks {
if b.BlockID > maxPageBlockID {
maxPageBlockID = b.BlockID
}
}
for _, b := range res.Blocks {
if b.BlockID == 0 {
b.BlockID = maxPageBlockID + 1
maxPageBlockID++
}
}
return
}
@@ -44,7 +44,7 @@ func (e StoreEncoder) prepareRecords(ctx context.Context, p envoyx.EncodeParams,
var (
aux = make(map[string]string)
more bool
ident string
ident []string
rec types.Record
)
@@ -56,7 +56,7 @@ func (e StoreEncoder) prepareRecords(ctx context.Context, p envoyx.EncodeParams,
return
}
ds.refToID[ident] = id.Next()
ds.AddRef(id.Next(), ident...)
rec, err = e.auxToRecord(aux)
if err != nil {
@@ -93,7 +93,7 @@ func (e StoreEncoder) encodeRecordDatasource(ctx context.Context, p envoyx.Encod
var (
auxRec = make(map[string]string)
more bool
ident string
ident []string
rec types.Record
nsNode *envoyx.Node
@@ -173,7 +173,11 @@ func (e StoreEncoder) encodeRecordDatasource(ctx context.Context, p envoyx.Encod
rec.CreatedAt = time.Now()
// Values and refs
rec.ID = ds.refToID[ident]
rec.ID, err = ds.ResolveRefS(ident...)
if err != nil {
return err
}
for i, v := range rec.Values {
if getters[v.Name] == nil {
continue
+171 -4
View File
@@ -231,6 +231,7 @@ func (d *auxYamlDoc) unmarshalChartNode(dctx documentContext, n *yaml.Node, meta
auxOut envoyx.NodeSet
nestedNodes envoyx.NodeSet
scope envoyx.Scope
envoyConfig envoyx.NodeConfig
rbacNodes envoyx.NodeSet
)
_ = auxOut
@@ -252,7 +253,7 @@ func (d *auxYamlDoc) unmarshalChartNode(dctx documentContext, n *yaml.Node, meta
auxRefs map[string]envoyx.Ref
auxIdents envoyx.Identifiers
)
auxRefs, auxIdents, err = unmarshalChartConfigNode(r, n)
auxRefs, auxIdents, err = d.unmarshalChartConfigNode(r, n)
if err != nil {
return err
}
@@ -317,6 +318,8 @@ func (d *auxYamlDoc) unmarshalChartNode(dctx documentContext, n *yaml.Node, meta
}
rbacNodes = append(rbacNodes, auxOut...)
auxOut = nil
case "(envoy)":
envoyConfig = d.decodeEnvoyConfig(n)
}
return nil
@@ -415,6 +418,8 @@ func (d *auxYamlDoc) unmarshalChartNode(dctx documentContext, n *yaml.Node, meta
References: refs,
Scope: scope,
Config: envoyConfig,
}
// Update RBAC resource nodes with references regarding the resource
for _, rn := range rbacNodes {
@@ -528,6 +533,7 @@ func (d *auxYamlDoc) unmarshalModuleNode(dctx documentContext, n *yaml.Node, met
auxOut envoyx.NodeSet
nestedNodes envoyx.NodeSet
scope envoyx.Scope
envoyConfig envoyx.NodeConfig
rbacNodes envoyx.NodeSet
)
_ = auxOut
@@ -595,6 +601,8 @@ func (d *auxYamlDoc) unmarshalModuleNode(dctx documentContext, n *yaml.Node, met
}
rbacNodes = append(rbacNodes, auxOut...)
auxOut = nil
case "(envoy)":
envoyConfig = d.decodeEnvoyConfig(n)
}
return nil
@@ -659,7 +667,7 @@ func (d *auxYamlDoc) unmarshalModuleNode(dctx documentContext, n *yaml.Node, met
}
break
case "source", "datasource":
case "source", "datasource", "records":
if y7s.IsSeq(n) {
nestedNodes, err = d.unmarshalExtendedSourceSeq(dctx, n)
if err != nil {
@@ -720,6 +728,8 @@ func (d *auxYamlDoc) unmarshalModuleNode(dctx documentContext, n *yaml.Node, met
References: refs,
Scope: scope,
Config: envoyConfig,
}
// Update RBAC resource nodes with references regarding the resource
for _, rn := range rbacNodes {
@@ -817,6 +827,7 @@ func (d *auxYamlDoc) unmarshalModuleFieldNode(dctx documentContext, n *yaml.Node
auxOut envoyx.NodeSet
nestedNodes envoyx.NodeSet
scope envoyx.Scope
envoyConfig envoyx.NodeConfig
rbacNodes envoyx.NodeSet
)
_ = auxOut
@@ -828,6 +839,44 @@ func (d *auxYamlDoc) unmarshalModuleFieldNode(dctx documentContext, n *yaml.Node
switch strings.ToLower(k.Value) {
case "defaultvalue":
// Handle custom node decoder
//
// The decoder may update the passed resource with arbitrary values
// as well as provide additional references and identifiers for the node.
var (
auxRefs map[string]envoyx.Ref
auxIdents envoyx.Identifiers
)
auxRefs, auxIdents, err = d.unmarshalModuleFieldDefaultValueNode(r, n)
if err != nil {
return err
}
refs = envoyx.MergeRefs(refs, auxRefs)
ii = ii.Merge(auxIdents)
break
case "expressions":
// Handle custom node decoder
//
// The decoder may update the passed resource with arbitrary values
// as well as provide additional references and identifiers for the node.
var (
auxRefs map[string]envoyx.Ref
auxIdents envoyx.Identifiers
)
auxRefs, auxIdents, err = d.unmarshalModuleFieldExpressionsNode(r, n)
if err != nil {
return err
}
refs = envoyx.MergeRefs(refs, auxRefs)
ii = ii.Merge(auxIdents)
break
case "id":
// Handle identifiers
err = y7s.DecodeScalar(n, "id", &auxNodeValue)
@@ -871,7 +920,7 @@ func (d *auxYamlDoc) unmarshalModuleFieldNode(dctx documentContext, n *yaml.Node
auxRefs map[string]envoyx.Ref
auxIdents envoyx.Identifiers
)
auxRefs, auxIdents, err = unmarshalModuleFieldOptionsNode(r, n)
auxRefs, auxIdents, err = d.unmarshalModuleFieldOptionsNode(r, n)
if err != nil {
return err
}
@@ -896,6 +945,8 @@ func (d *auxYamlDoc) unmarshalModuleFieldNode(dctx documentContext, n *yaml.Node
}
rbacNodes = append(rbacNodes, auxOut...)
auxOut = nil
case "(envoy)":
envoyConfig = d.decodeEnvoyConfig(n)
}
return nil
@@ -994,6 +1045,8 @@ func (d *auxYamlDoc) unmarshalModuleFieldNode(dctx documentContext, n *yaml.Node
References: refs,
Scope: scope,
Config: envoyConfig,
}
// Update RBAC resource nodes with references regarding the resource
for _, rn := range rbacNodes {
@@ -1091,6 +1144,7 @@ func (d *auxYamlDoc) unmarshalNamespaceNode(dctx documentContext, n *yaml.Node,
auxOut envoyx.NodeSet
nestedNodes envoyx.NodeSet
scope envoyx.Scope
envoyConfig envoyx.NodeConfig
rbacNodes envoyx.NodeSet
)
_ = auxOut
@@ -1138,6 +1192,8 @@ func (d *auxYamlDoc) unmarshalNamespaceNode(dctx documentContext, n *yaml.Node,
}
rbacNodes = append(rbacNodes, auxOut...)
auxOut = nil
case "(envoy)":
envoyConfig = d.decodeEnvoyConfig(n)
}
return nil
@@ -1292,6 +1348,8 @@ func (d *auxYamlDoc) unmarshalNamespaceNode(dctx documentContext, n *yaml.Node,
References: refs,
Scope: scope,
Config: envoyConfig,
}
// Update RBAC resource nodes with references regarding the resource
for _, rn := range rbacNodes {
@@ -1358,6 +1416,41 @@ func (d *auxYamlDoc) unmarshalPageMap(dctx documentContext, n *yaml.Node) (out e
return
}
// unmarshalPagesExtendedSeq unmarshals Pages when provided as a sequence node
func (d *auxYamlDoc) unmarshalExtendedPagesSeq(dctx documentContext, n *yaml.Node) (out envoyx.NodeSet, err error) {
var aux envoyx.NodeSet
err = y7s.EachSeq(n, func(n *yaml.Node) error {
aux, err = d.unmarshalPagesExtendedNode(dctx, n)
if err != nil {
return err
}
out = append(out, aux...)
return nil
})
return
}
// unmarshalPagesExtendedMap unmarshals Pages when provided as a mapping node
//
// When map encoded, the map key is used as a preset identifier.
// The identifier is passed to the node function as a meta node
func (d *auxYamlDoc) unmarshalExtendedPagesMap(dctx documentContext, n *yaml.Node) (out envoyx.NodeSet, err error) {
var aux envoyx.NodeSet
err = y7s.EachMap(n, func(k, n *yaml.Node) error {
aux, err = d.unmarshalPagesExtendedNode(dctx, n, k)
if err != nil {
return err
}
out = append(out, aux...)
return nil
})
return
}
// unmarshalPageNode is a cookie-cutter function to unmarshal
// the yaml node into the corresponding Corteza type & Node
func (d *auxYamlDoc) unmarshalPageNode(dctx documentContext, n *yaml.Node, meta ...*yaml.Node) (out envoyx.NodeSet, err error) {
@@ -1389,6 +1482,7 @@ func (d *auxYamlDoc) unmarshalPageNode(dctx documentContext, n *yaml.Node, meta
auxOut envoyx.NodeSet
nestedNodes envoyx.NodeSet
scope envoyx.Scope
envoyConfig envoyx.NodeConfig
rbacNodes envoyx.NodeSet
)
_ = auxOut
@@ -1400,6 +1494,25 @@ func (d *auxYamlDoc) unmarshalPageNode(dctx documentContext, n *yaml.Node, meta
switch strings.ToLower(k.Value) {
case "blocks":
// Handle custom node decoder
//
// The decoder may update the passed resource with arbitrary values
// as well as provide additional references and identifiers for the node.
var (
auxRefs map[string]envoyx.Ref
auxIdents envoyx.Identifiers
)
auxRefs, auxIdents, err = d.unmarshalPageBlocksNode(r, n)
if err != nil {
return err
}
refs = envoyx.MergeRefs(refs, auxRefs)
ii = ii.Merge(auxIdents)
break
case "handle":
// Handle identifiers
err = y7s.DecodeScalar(n, "handle", &auxNodeValue)
@@ -1446,7 +1559,14 @@ func (d *auxYamlDoc) unmarshalPageNode(dctx documentContext, n *yaml.Node, meta
break
case "selfid":
case "selfid", "parent":
// Handle field alias
//
// @todo consider adding an is empty check before overwriting
err = y7s.DecodeScalar(n, "selfID", &r.SelfID)
if err != nil {
return err
}
// Handle references
err = y7s.DecodeScalar(n, "selfID", &auxNodeValue)
if err != nil {
@@ -1459,6 +1579,17 @@ func (d *auxYamlDoc) unmarshalPageNode(dctx documentContext, n *yaml.Node, meta
break
case "weight", "order":
// Handle field alias
//
// @todo consider adding an is empty check before overwriting
err = y7s.DecodeScalar(n, "weight", &r.Weight)
if err != nil {
return err
}
break
// Handle RBAC rules
case "allow":
auxOut, err = unmarshalAllowNode(n)
@@ -1475,6 +1606,8 @@ func (d *auxYamlDoc) unmarshalPageNode(dctx documentContext, n *yaml.Node, meta
}
rbacNodes = append(rbacNodes, auxOut...)
auxOut = nil
case "(envoy)":
envoyConfig = d.decodeEnvoyConfig(n)
}
return nil
@@ -1525,6 +1658,19 @@ func (d *auxYamlDoc) unmarshalPageNode(dctx documentContext, n *yaml.Node, meta
switch strings.ToLower(k.Value) {
case "children", "pages":
if y7s.IsSeq(n) {
nestedNodes, err = d.unmarshalExtendedPagesSeq(dctx, n)
if err != nil {
return err
}
} else {
nestedNodes, err = d.unmarshalExtendedPagesMap(dctx, n)
if err != nil {
return err
}
}
break
}
// Iterate nested nodes and update their reference to the current resource
@@ -1573,6 +1719,8 @@ func (d *auxYamlDoc) unmarshalPageNode(dctx documentContext, n *yaml.Node, meta
References: refs,
Scope: scope,
Config: envoyConfig,
}
// Update RBAC resource nodes with references regarding the resource
for _, rn := range rbacNodes {
@@ -1716,6 +1864,25 @@ func unmarshalLocaleNode(n *yaml.Node) (out envoyx.NodeSet, err error) {
})
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Envoy config unmarshal logic
// // // // // // // // // // // // // // // // // // // // // // // // //
func (d *auxYamlDoc) decodeEnvoyConfig(n *yaml.Node) (out envoyx.NodeConfig) {
y7s.EachMap(n, func(k, v *yaml.Node) (err error) {
switch strings.ToLower(k.Value) {
case "skipif", "skip":
return y7s.DecodeScalar(v, "decode skip if", &out.SkipIf)
case "onexisting", "mergealg":
out.MergeAlg = envoyx.CastMergeAlg(v.Value)
}
return nil
})
return
}
// // // // // // // // // // // // // // // // // // // // // // // // //
// Utilities
// // // // // // // // // // // // // // // // // // // // // // // // //
+270 -7
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"strings"
automationTypes "github.com/cortezaproject/corteza/server/automation/types"
"github.com/cortezaproject/corteza/server/compose/types"
"github.com/cortezaproject/corteza/server/pkg/envoyx"
"github.com/cortezaproject/corteza/server/pkg/y7s"
@@ -22,10 +23,16 @@ type (
}
datasourceMapping struct {
SourceIdent string `yaml:"source"`
KeyField string `yaml:"key"`
SourceIdent string `yaml:"source"`
KeyField []string `yaml:"key"`
References map[string]string
Scope map[string]string
// Defaultable indicates wether the mapping should keep the values where
// the ident is not explicitly mapped.
//
// When true, the value is assigned to the given identifier.
Defaultable bool `yaml:"defaultable"`
Mapping fieldMapping
}
)
@@ -34,7 +41,7 @@ const (
ComposeRecordDatasourceAuxType = "corteza::compose:record-datasource"
)
func unmarshalChartConfigNode(r *types.Chart, n *yaml.Node) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
func (d *auxYamlDoc) unmarshalChartConfigNode(r *types.Chart, n *yaml.Node) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
err = y7s.EachMap(n, func(k, v *yaml.Node) error {
if k.Value != "reports" {
return nil
@@ -49,7 +56,7 @@ func unmarshalChartConfigNode(r *types.Chart, n *yaml.Node) (refs map[string]env
err = y7s.EachSeq(v, func(c *yaml.Node) error {
i++
auxRefs, auxIdents, err = unmarshalChartConfigReportNode(r, c, i)
auxRefs, auxIdents, err = d.unmarshalChartConfigReportNode(r, c, i)
refs = envoyx.MergeRefs(refs, auxRefs)
idents = idents.Merge(auxIdents)
return err
@@ -58,7 +65,7 @@ func unmarshalChartConfigNode(r *types.Chart, n *yaml.Node) (refs map[string]env
return err
}
} else {
refs, idents, err = unmarshalChartConfigReportNode(r, v, 0)
refs, idents, err = d.unmarshalChartConfigReportNode(r, v, 0)
return err
}
return nil
@@ -67,7 +74,7 @@ func unmarshalChartConfigNode(r *types.Chart, n *yaml.Node) (refs map[string]env
return
}
func unmarshalChartConfigReportNode(r *types.Chart, n *yaml.Node, index int) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
func (d *auxYamlDoc) unmarshalChartConfigReportNode(r *types.Chart, n *yaml.Node, index int) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
err = y7s.EachMap(n, func(k, v *yaml.Node) error {
switch strings.ToLower(k.Value) {
case "module", "mod", "moduleid", "module_id":
@@ -85,7 +92,153 @@ func unmarshalChartConfigReportNode(r *types.Chart, n *yaml.Node, index int) (re
return
}
func unmarshalModuleFieldOptionsNode(r *types.ModuleField, n *yaml.Node) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
func (d *auxYamlDoc) unmarshalPageBlocksNode(r *types.Page, n *yaml.Node) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
refs = map[string]envoyx.Ref{}
for index, b := range r.Blocks {
switch b.Kind {
case "RecordList":
refs = envoyx.MergeRefs(refs, getPageBlockRecordListRefs(b, index))
case "Automation":
refs = envoyx.MergeRefs(refs, getPageBlockAutomationRefs(b, index))
case "RecordOrganizer":
refs = envoyx.MergeRefs(refs, getPageBlockRecordOrganizerRefs(b, index))
case "Chart":
refs = envoyx.MergeRefs(refs, getPageBlockChartRefs(b, index))
case "Calendar":
refs = envoyx.MergeRefs(refs, getPageBlockCalendarRefs(b, index))
case "Metric":
refs = envoyx.MergeRefs(refs, getPageBlockMetricRefs(b, index))
case "Comment":
refs = envoyx.MergeRefs(refs, getPageBlockCommentRefs(b, index))
}
}
return
}
func getPageBlockRecordListRefs(b types.PageBlock, index int) (refs map[string]envoyx.Ref) {
refs = make(map[string]envoyx.Ref)
id := optString(b.Options, "module", "moduleID")
if id == "" || id == "0" {
return
}
refs[fmt.Sprintf("Blocks.%d.Options.ModuleID", index)] = envoyx.Ref{
ResourceType: types.ModuleResourceType,
Identifiers: envoyx.MakeIdentifiers(id),
}
return
}
func getPageBlockChartRefs(b types.PageBlock, index int) (refs map[string]envoyx.Ref) {
refs = make(map[string]envoyx.Ref)
id := optString(b.Options, "chart", "chartID")
if id == "" || id == "0" {
return
}
refs[fmt.Sprintf("Blocks.%d.Options.ChartID", index)] = envoyx.Ref{
ResourceType: types.ChartResourceType,
Identifiers: envoyx.MakeIdentifiers(id),
}
return
}
func getPageBlockCalendarRefs(b types.PageBlock, index int) (refs map[string]envoyx.Ref) {
refs = make(map[string]envoyx.Ref)
ff, _ := b.Options["feeds"].([]interface{})
for j, f := range ff {
feed, _ := f.(map[string]interface{})
opt, _ := (feed["options"]).(map[string]interface{})
id := optString(opt, "module", "moduleID")
if id == "" || id == "0" {
return
}
refs[fmt.Sprintf("Blocks.%d.Options.feeds.%d.ModuleID", index, j)] = envoyx.Ref{
ResourceType: types.ChartResourceType,
Identifiers: envoyx.MakeIdentifiers(id),
}
}
return
}
func getPageBlockMetricRefs(b types.PageBlock, index int) (refs map[string]envoyx.Ref) {
refs = make(map[string]envoyx.Ref)
mm, _ := b.Options["metrics"].([]interface{})
for j, m := range mm {
mops, _ := m.(map[string]interface{})
id := optString(mops, "module", "moduleID")
if id == "" || id == "0" {
return
}
refs[fmt.Sprintf("Blocks.%d.Options.metrics.%d.ModuleID", index, j)] = envoyx.Ref{
ResourceType: types.ChartResourceType,
Identifiers: envoyx.MakeIdentifiers(id),
}
}
return
}
func getPageBlockCommentRefs(b types.PageBlock, index int) (refs map[string]envoyx.Ref) {
// Same difference
return getPageBlockRecordListRefs(b, index)
}
func getPageBlockRecordOrganizerRefs(b types.PageBlock, index int) (refs map[string]envoyx.Ref) {
// Same difference
return getPageBlockRecordListRefs(b, index)
}
func getPageBlockAutomationRefs(b types.PageBlock, index int) (refs map[string]envoyx.Ref) {
refs = make(map[string]envoyx.Ref)
bb, _ := b.Options["buttons"].([]interface{})
for _, b := range bb {
button, _ := b.(map[string]interface{})
id := optString(button, "workflow", "workflowID")
if id == "" || id == "0" {
return
}
refs[fmt.Sprintf("Blocks.%d.Options.WorkflowID", index)] = envoyx.Ref{
ResourceType: automationTypes.WorkflowResourceType,
Identifiers: envoyx.MakeIdentifiers(id),
}
}
return
}
func optString(opt map[string]interface{}, kk ...string) string {
for _, k := range kk {
if vr, has := opt[k]; has {
v, _ := vr.(string)
return v
}
}
return ""
}
func (d *auxYamlDoc) unmarshalModuleFieldOptionsNode(r *types.ModuleField, n *yaml.Node) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
refs = make(map[string]envoyx.Ref)
err = y7s.EachMap(n, func(k, v *yaml.Node) error {
@@ -109,6 +262,83 @@ func unmarshalModuleFieldOptionsNode(r *types.ModuleField, n *yaml.Node) (refs m
return
}
func (d *auxYamlDoc) unmarshalModuleFieldDefaultValueNode(r *types.ModuleField, n *yaml.Node) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
var rvs = types.RecordValueSet{}
switch n.Kind {
case yaml.ScalarNode:
rvs = rvs.Set(&types.RecordValue{Value: n.Value})
case yaml.SequenceNode:
_ = y7s.EachSeq(n, func(v *yaml.Node) error {
rvs = rvs.Set(&types.RecordValue{Value: n.Value, Place: uint(len(rvs))})
return nil
})
}
r.DefaultValue = rvs
return
}
func (d *auxYamlDoc) unmarshalModuleFieldExpressionsNode(r *types.ModuleField, n *yaml.Node) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
err = y7s.EachMap(n, func(k *yaml.Node, v *yaml.Node) error {
switch k.Value {
case "sanitizer":
var aux string
err = y7s.DecodeScalar(v, "sanitizer", &aux)
if err != nil {
return err
}
r.Expressions.Sanitizers = append(r.Expressions.Sanitizers, aux)
case "sanitizers":
return y7s.EachSeq(v, func(san *yaml.Node) error {
r.Expressions.Sanitizers = append(r.Expressions.Sanitizers, san.Value)
return nil
})
case "validator":
var aux types.ModuleFieldValidator
err = v.Decode(&aux)
if err != nil {
return err
}
r.Expressions.Validators = append(r.Expressions.Validators, aux)
case "validators":
return y7s.Each(v, func(k *yaml.Node, v *yaml.Node) error {
var aux types.ModuleFieldValidator
if y7s.IsKind(v, yaml.MappingNode) {
err = v.Decode(&aux)
if err != nil {
return err
}
} else {
aux.Test = k.Value
aux.Error = v.Value
}
r.Expressions.Validators = append(r.Expressions.Validators, aux)
return nil
})
case "formatter":
r.Expressions.Formatters = append(r.Expressions.Formatters, v.Value)
return nil
case "formatters":
return y7s.EachSeq(v, func(san *yaml.Node) error {
r.Expressions.Formatters = append(r.Expressions.Formatters, san.Value)
return nil
})
}
return nil
})
return
}
func (d *auxYamlDoc) unmarshalYAML(k string, n *yaml.Node) (out envoyx.NodeSet, err error) {
return
}
@@ -152,6 +382,10 @@ func (d *auxYamlDoc) postProcessNestedModuleNodes(nn envoyx.NodeSet) (out envoyx
return
}
func (d *auxYamlDoc) unmarshalPagesExtendedNode(dctx documentContext, n *yaml.Node, meta ...*yaml.Node) (out envoyx.NodeSet, err error) {
return d.unmarshalPageNode(dctx, n, meta...)
}
func (d *auxYamlDoc) unmarshalSourceExtendedNode(dctx documentContext, n *yaml.Node, meta ...*yaml.Node) (out envoyx.NodeSet, err error) {
var r datasourceMapping
@@ -163,6 +397,35 @@ func (d *auxYamlDoc) unmarshalSourceExtendedNode(dctx documentContext, n *yaml.N
// and then unmarshal into the resource while omitting errors.
n.Decode(&r)
err = y7s.EachMap(n, func(k, n *yaml.Node) error {
var auxNodeValue any
_ = auxNodeValue
switch strings.ToLower(k.Value) {
case "origin", "from":
err = y7s.DecodeScalar(n, "origin", &r.SourceIdent)
if err != nil {
return err
}
case "key", "index", "pk":
if !y7s.IsKind(n, yaml.SequenceNode) {
r.KeyField = []string{n.Value}
} else {
r.KeyField = make([]string, 0, 3)
y7s.EachSeq(n, func(n *yaml.Node) error {
r.KeyField = append(r.KeyField, n.Value)
return nil
})
}
case "map":
return n.Decode(&r.Mapping)
}
return nil
})
// @todo for now we only support record datasources; extend when needed
auxN := &envoyx.Node{
Datasource: &RecordDatasource{
+24 -8
View File
@@ -133,7 +133,10 @@ func (e YamlEncoder) encodeChart(ctx context.Context, p envoyx.EncodeParams, nod
res := node.Resource.(*types.Chart)
// Pre-compute some map values so we can omit error checking when encoding yaml nodes
auxConfig, err := e.encodeChartConfigC(ctx, p, tt, node, res, res.Config)
if err != nil {
return
}
auxCreatedAt, err := e.encodeTimestamp(p, res.CreatedAt)
if err != nil {
return
@@ -153,11 +156,11 @@ func (e YamlEncoder) encodeChart(ctx context.Context, p envoyx.EncodeParams, nod
}
out, err = y7s.AddMap(out,
"config", res.Config,
"config", auxConfig,
"createdAt", auxCreatedAt,
"deletedAt", auxDeletedAt,
"handle", res.Handle,
"id", res.ID,
"chartID", res.ID,
"name", res.Name,
"namespaceID", auxNamespaceID,
"updatedAt", auxUpdatedAt,
@@ -208,6 +211,10 @@ func (e YamlEncoder) encodeModule(ctx context.Context, p envoyx.EncodeParams, no
if err != nil {
return
}
auxFields, err := e.encodeModuleFieldsC(ctx, p, tt, node, res, res.Fields)
if err != nil {
return
}
auxNamespaceID, err := e.encodeRef(p, res.NamespaceID, "NamespaceID", node, tt)
if err != nil {
@@ -219,12 +226,12 @@ func (e YamlEncoder) encodeModule(ctx context.Context, p envoyx.EncodeParams, no
}
out, err = y7s.AddMap(out,
"config", e.encodeModuleConfig(p, res.Config),
"config", res.Config,
"createdAt", auxCreatedAt,
"deletedAt", auxDeletedAt,
"fields", res.Fields,
"fields", auxFields,
"handle", res.Handle,
"id", res.ID,
"moduleID", res.ID,
"meta", res.Meta,
"name", res.Name,
"namespaceID", auxNamespaceID,
@@ -294,6 +301,11 @@ func (e YamlEncoder) encodeModuleField(ctx context.Context, p envoyx.EncodeParam
return
}
auxOptions, err := e.encodeModuleFieldOptionsC(ctx, p, tt, node, res, res.Options)
if err != nil {
return
}
auxUpdatedAt, err := e.encodeTimestampNil(p, res.UpdatedAt)
if err != nil {
return
@@ -311,7 +323,7 @@ func (e YamlEncoder) encodeModuleField(ctx context.Context, p envoyx.EncodeParam
"moduleID", auxModuleID,
"multi", res.Multi,
"name", res.Name,
"options", res.Options,
"options", auxOptions,
"place", res.Place,
"required", res.Required,
"updatedAt", auxUpdatedAt,
@@ -447,6 +459,10 @@ func (e YamlEncoder) encodePage(ctx context.Context, p envoyx.EncodeParams, node
res := node.Resource.(*types.Page)
// Pre-compute some map values so we can omit error checking when encoding yaml nodes
auxBlocks, err := e.encodePageBlocksC(ctx, p, tt, node, res, res.Blocks)
if err != nil {
return
}
auxCreatedAt, err := e.encodeTimestamp(p, res.CreatedAt)
if err != nil {
@@ -476,7 +492,7 @@ func (e YamlEncoder) encodePage(ctx context.Context, p envoyx.EncodeParams, node
}
out, err = y7s.AddMap(out,
"blocks", res.Blocks,
"blocks", auxBlocks,
"children", res.Children,
"config", res.Config,
"createdAt", auxCreatedAt,
+232 -3
View File
@@ -1,13 +1,242 @@
package envoy
import (
"context"
"fmt"
"github.com/cortezaproject/corteza/server/compose/types"
"github.com/cortezaproject/corteza/server/pkg/envoyx"
"github.com/cortezaproject/corteza/server/pkg/y7s"
)
func (e YamlEncoder) encodeModuleConfig(p envoyx.EncodeParams, cfg types.ModuleConfig) any {
func (e YamlEncoder) encodeChartConfigC(ctx context.Context, p envoyx.EncodeParams, tt envoyx.Traverser, n *envoyx.Node, chart *types.Chart, cfg types.ChartConfig) (_ any, err error) {
// @todo...
reports, _ := y7s.MakeSeq()
return nil
for i, r := range cfg.Reports {
modRef, ok := n.References[fmt.Sprintf("Config.Reports.%d.ModuleID", i)]
if !ok {
continue
}
mNode := tt.ParentForRef(n, modRef)
if mNode == nil {
err = fmt.Errorf("module for ref not found")
return
}
r, err := y7s.MakeMap(
"filter", r.Filter,
"module", mNode.Identifiers.FriendlyIdentifier(),
"metrics", r.Metrics,
"dimensions", r.Dimensions,
"yAxis", r.YAxis,
)
if err != nil {
return nil, err
}
reports, err = y7s.AddSeq(reports, r)
if err != nil {
return nil, err
}
}
return reports, nil
}
func (e YamlEncoder) encodeModuleFieldsC(ctx context.Context, p envoyx.EncodeParams, tt envoyx.Traverser, n *envoyx.Node, mod *types.Module, fields types.ModuleFieldSet) (_ any, err error) {
fn := tt.ChildrenForResourceType(n, types.ModuleFieldResourceType)
out, err := e.encodeModuleFields(ctx, p, fn, tt)
return out, err
}
func (e YamlEncoder) encodeModuleFieldOptionsC(ctx context.Context, p envoyx.EncodeParams, tt envoyx.Traverser, n *envoyx.Node, f *types.ModuleField, opt types.ModuleFieldOptions) (_ any, err error) {
if opt == nil {
opt = make(types.ModuleFieldOptions)
}
switch f.Kind {
case "Record":
mNode := tt.ParentForRef(n, n.References["Options.ModuleID"])
if mNode == nil {
err = fmt.Errorf("module for ref not found")
return
}
opt["module"] = mNode.Identifiers.FriendlyIdentifier()
delete(opt, "moduleID")
case "User":
aux := make([]string, 0, 2)
for i := range opt.Strings("roles") {
rNode := tt.ParentForRef(n, n.References[fmt.Sprintf("Options.RoleID.%d", i)])
if rNode == nil {
err = fmt.Errorf("role for ref not found")
return
}
aux = append(aux, rNode.Identifiers.FriendlyIdentifier())
}
opt["roles"] = aux
delete(opt, "role")
delete(opt, "roleID")
}
nopt, _ := y7s.MakeMap()
for k, v := range opt {
nopt, err = y7s.AddMap(nopt, k, v)
if err != nil {
return nil, err
}
}
return nopt, nil
}
func (e YamlEncoder) encodePageBlocksC(ctx context.Context, p envoyx.EncodeParams, tt envoyx.Traverser, n *envoyx.Node, pg *types.Page, bb types.PageBlocks) (_ any, err error) {
out, _ := y7s.MakeSeq()
var aux any
for i, b := range pg.Blocks {
aux, err = e.encodePageBlockC(ctx, p, tt, n, pg, i, b)
if err != nil {
return
}
out, err = y7s.AddSeq(out, aux)
if err != nil {
return
}
}
return out, nil
}
func (e YamlEncoder) encodePageBlockC(ctx context.Context, p envoyx.EncodeParams, tt envoyx.Traverser, n *envoyx.Node, pg *types.Page, index int, b types.PageBlock) (_ any, err error) {
switch b.Kind {
case "RecordList":
b = e.cleanupPageblockRecordList(b)
node := tt.ParentForRef(n, n.References[fmt.Sprintf("Blocks.%d.Options.ModuleID", index)])
if node == nil {
err = fmt.Errorf("module for ref not found")
return
}
b.Options["module"] = node.Identifiers.FriendlyIdentifier()
delete(b.Options, "moduleID")
break
case "RecordOrganizer":
node := tt.ParentForRef(n, n.References[fmt.Sprintf("Blocks.%d.Options.ModuleID", index)])
if node == nil {
err = fmt.Errorf("module for ref not found")
return
}
b.Options["module"] = node.Identifiers.FriendlyIdentifier()
delete(b.Options, "moduleID")
break
case "Chart":
node := tt.ParentForRef(n, n.References[fmt.Sprintf("Blocks.%d.Options.ChartID", index)])
if node == nil {
err = fmt.Errorf("chart for ref not found")
return
}
b.Options["chart"] = node.Identifiers.FriendlyIdentifier()
delete(b.Options, "chartID")
break
case "Calendar":
ff, _ := b.Options["feeds"].([]interface{})
for i, f := range ff {
feed, _ := f.(map[string]interface{})
fOpts, _ := (feed["options"]).(map[string]interface{})
node := tt.ParentForRef(n, n.References[fmt.Sprintf("Blocks.%d.Options.feeds.%d.ModuleID", index, i)])
if node == nil {
err = fmt.Errorf("module for ref not found")
return
}
fOpts["module"] = node.Identifiers.FriendlyIdentifier()
delete(fOpts, "moduleID")
}
break
case "Automation":
bb, _ := b.Options["buttons"].([]interface{})
for i, b := range bb {
button, _ := b.(map[string]interface{})
if _, has := button["workflowID"]; !has {
continue
}
node := tt.ParentForRef(n, n.References[fmt.Sprintf("Blocks.%d.Options.buttons.%d.WorkflowID", index, i)])
if node == nil {
err = fmt.Errorf("chart for ref not found")
return
}
button["workflow"] = node.Identifiers.FriendlyIdentifier()
delete(button, "workflowID")
i++
}
break
case "Metric":
mm, _ := b.Options["metrics"].([]interface{})
for i, m := range mm {
node := tt.ParentForRef(n, n.References[fmt.Sprintf("Blocks.%d.Options.metrics.%d.ModuleID", index, i)])
if node == nil {
err = fmt.Errorf("chart for ref not found")
return
}
mops, _ := m.(map[string]interface{})
mops["module"] = node.Identifiers.FriendlyIdentifier()
delete(mops, "moduleID")
}
break
case "Comment":
node := tt.ParentForRef(n, n.References[fmt.Sprintf("Blocks.%d.Options.ModuleID", index)])
if node == nil {
err = fmt.Errorf("module for ref not found")
return
}
b.Options["module"] = node.Identifiers.FriendlyIdentifier()
delete(b.Options, "moduleID")
break
}
return
}
func (e YamlEncoder) cleanupPageblockRecordList(b types.PageBlock) (_ types.PageBlock) {
rawFF, has := b.Options["fields"]
if !has {
return
}
ff, ok := rawFF.([]interface{})
if !ok {
return
}
retFF := make([]interface{}, 0, len(ff))
for _, rawF := range ff {
switch c := rawF.(type) {
case string:
retFF = append(retFF, map[string]interface{}{"name": c})
case map[string]interface{}, map[string]string:
retFF = append(retFF, c)
default:
retFF = append(retFF, rawF)
}
}
b.Options["fields"] = retFF
return b
}
+14 -7
View File
@@ -13,7 +13,13 @@ module: {
model: {
ident: "compose_module"
attributes: {
id: schema.IdField
id: schema.IdField & {
envoy: {
yaml: {
identKeyEncode: "moduleID"
}
}
}
namespace_id: {
ident: "namespaceID",
goType: "uint64",
@@ -42,17 +48,17 @@ module: {
dal: { type: "JSON", defaultEmptyObject: true }
omitSetter: true
omitGetter: true
envoy: {
yaml: {
customEncoder: true
}
}
}
fields: {
goType: "types.ModuleFieldSet",
store: false
omitSetter: true
omitGetter: true
envoy: {
yaml: {
customEncoder: true
}
}
}
created_at: schema.SortableTimestampNowField
updated_at: schema.SortableTimestampNilField
@@ -94,7 +100,8 @@ module: {
extendedResourceDecoders: [{
ident: "source"
expIdent: "Source"
identKeys: ["source", "datasource"]
// @deprecated records is what the old version used
identKeys: ["source", "datasource", "records"]
supportMappedInput: false
}]
}
+12
View File
@@ -42,6 +42,7 @@ moduleField: {
envoy: {
yaml: {
customDecoder: true
customEncoder: true
}
}
}
@@ -78,12 +79,22 @@ moduleField: {
dal: { type: "JSON", defaultEmptyObject: true }
omitSetter: true
omitGetter: true
envoy: {
yaml: {
customDecoder: true
}
}
}
expressions: {
goType: "types.ModuleFieldExpr"
dal: { type: "JSON", defaultEmptyObject: true }
omitSetter: true
omitGetter: true
envoy: {
yaml: {
customDecoder: true
}
}
}
created_at: schema.SortableTimestampNowField
updated_at: schema.SortableTimestampNilField
@@ -127,6 +138,7 @@ moduleField: {
store: {
handleField: ""
customFilterBuilder: true
extendedRefDecoder: true
}
}
+23
View File
@@ -28,6 +28,9 @@ page: {
store: {
filterRefField: "ParentID"
}
yaml: {
identKeyAlias: ["parent"]
}
}
}
module_id: {
@@ -54,6 +57,12 @@ page: {
dal: { type: "JSON", defaultEmptyObject: true }
omitSetter: true
omitGetter: true
envoy: {
yaml: {
customDecoder: true
customEncoder: true
}
}
}
children: {
goType: "types.PageSet", store: false
@@ -67,6 +76,11 @@ page: {
weight: {
goType: "int", sortable: true
dal: { type: "Number", default: 0, meta: { "rdbms:type": "integer" } }
envoy: {
yaml: {
identKeyAlias: ["order"]
}
}
}
description: {
goType: "string"
@@ -113,9 +127,18 @@ page: {
supportMappedInput: true
mappedField: "Handle"
identKeyAlias: ["pages", "pg"]
extendedResourceDecoders: [{
ident: "pages"
expIdent: "Pages"
identKeys: ["children", "pages"]
supportMappedInput: true
mappedField: "Handle"
}]
}
store: {
extendedFilterBuilder: true
extendedRefDecoder: true
}
}