POC importing record datasources

server/automation/envoy/store_encode.gen.go

@@ -48,6 +48,8 @@ func (e StoreEncoder) Prepare(ctx context.Context, p envoyx.EncodeParams, rt str
 
 	case types.TriggerResourceType:
 		return e.prepareTrigger(ctx, p, s, nn)
+	default:
+		return e.prepare(ctx, p, s, rt, nn)
 	}
 
 	return

server/automation/envoy/yaml_decode.gen.go

@@ -175,25 +175,6 @@ func (d *auxYamlDoc) unmarshalExtendedTriggersSeq(dctx documentContext, n *yaml.
 	return
 }
 
-// unmarshalTriggersExtendedMap unmarshals Triggers when provided as a mapping node
-//
-// When map encoded, the map key is used as a preset identifier.
-// The identifier is passed to the node function as a meta node
-func (d *auxYamlDoc) unmarshalExtendedTriggersMap(dctx documentContext, n *yaml.Node) (out envoyx.NodeSet, err error) {
-	var aux envoyx.NodeSet
-	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
-		aux, err = d.unmarshalTriggersExtendedNode(dctx, n, k)
-		if err != nil {
-			return err
-		}
-		out = append(out, aux...)
-
-		return nil
-	})
-
-	return
-}
-
 // unmarshalWorkflowNode is a cookie-cutter function to unmarshal
 // the yaml node into the corresponding Corteza type & Node
 func (d *auxYamlDoc) unmarshalWorkflowNode(dctx documentContext, n *yaml.Node, meta ...*yaml.Node) (out envoyx.NodeSet, err error) {
@@ -359,13 +340,13 @@ func (d *auxYamlDoc) unmarshalWorkflowNode(dctx documentContext, n *yaml.Node, m
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
 		switch strings.ToLower(k.Value) {
 
 		case "triggers":
-		default:
 			if y7s.IsSeq(n) {
 				nestedNodes, err = d.unmarshalExtendedTriggersSeq(dctx, n)
 				if err != nil {
@@ -395,17 +376,24 @@ func (d *auxYamlDoc) unmarshalWorkflowNode(dctx documentContext, n *yaml.Node, m
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 
@@ -597,6 +585,7 @@ func (d *auxYamlDoc) unmarshalTriggerNode(dctx documentContext, n *yaml.Node, me
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
@@ -624,17 +613,24 @@ func (d *auxYamlDoc) unmarshalTriggerNode(dctx documentContext, n *yaml.Node, me
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 

server/codegen/assets/templates/gocode/envoy/store_encode.go.tpl

@@ -53,6 +53,8 @@ func (e StoreEncoder) Prepare(ctx context.Context, p envoyx.EncodeParams, rt str
 	case types.{{.expIdent}}ResourceType:
 		return e.prepare{{.expIdent}}(ctx, p, s, nn)
 {{ end -}}
+	default:
+		return e.prepare(ctx, p, s, rt, nn)
 	}
 
 	return
@@ -228,12 +230,16 @@ func (e StoreEncoder) encode{{.expIdent}}(ctx context.Context, p envoyx.EncodePa
 		@note this setup will not duplicate encode calls since we only take the
 		      most specific parent resource.
 	*/}}
+	{{ $extendedEncoder := .envoy.store.extendedEncoder}}
+	{{ if $extendedEncoder }}
+	nested := make(envoyx.NodeSet, 0, 10)
+	{{ end }}
 	for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
 		nn = envoyx.OmitPlaceholderNodes(nn...)
 
 		switch rt {
 		{{- range $cmp := $rootRes }}
-			{{ if or ($cmp.envoy.omit) (not $cmp.envoy.use) }}
+			{{ if $cmp.envoy.omit }}
 				{{continue}}
 			{{ end }}
 			{{ if not $cmp.parents }}
@@ -250,12 +256,28 @@ func (e StoreEncoder) encode{{.expIdent}}(ctx context.Context, p envoyx.EncodePa
 				if err != nil {
 					return
 				}
+				{{ if $extendedEncoder }}
+				nested = append(nested, nn...)
+				{{ end }}
 
 		{{- end }}
 		}
 	}
 
-	return 
+	{{ if .envoy.store.extendedEncoder }}
+	err = e.encode{{.expIdent}}Extend(ctx, p, s, n, nested, tree)
+	if err != nil {
+		return
+	}
+	{{ end }}
+
+	{{ if .envoy.store.extendedSubResources }}
+	err = e.encode{{.expIdent}}ExtendSubResources(ctx, p, s, n, tree)
+	if err != nil {
+		return
+	}
+	{{ end }}
+	return
 }
 
 // matchup{{.expIdent}}s returns an index with indicates what resources already exist

server/codegen/assets/templates/gocode/envoy/util_rbac.go.tpl

@@ -43,7 +43,7 @@ func SplitResourceIdentifier(ref string) (out map[string]Ref) {
       }
 
       {{- range $cmp := $rootCmp.resources }}
-        {{ if or ($cmp.envoy.omit) (not $cmp.envoy.use) }}
+        {{ if $cmp.envoy.omit }}
           {{continue}}
         {{ end }}
 

server/codegen/assets/templates/gocode/envoy/utils.go.tpl

@@ -22,11 +22,13 @@ var (
   "{{ .fqrt }}": true,
 {{- end }}
 {{- end }}
+    "corteza::compose:record-datasource": true,
   }
 
   // superNeedyResources is the second level of filtering in case the first
   // pass removes everything
   superNeedyResources = map[string]bool{
     "corteza::compose:module-field": true,
+    "corteza::compose:record-datasource": true,
   }
 )

server/codegen/assets/templates/gocode/envoy/yaml_decode.go.tpl

@@ -191,7 +191,7 @@ func (d *auxYamlDoc) unmarshalExtended{{.expIdent}}Seq(dctx documentContext, n *
 
 	return
 }
-
+{{if not .supportMappedInput}}{{continue}}{{end}}
 // unmarshal{{.expIdent}}ExtendedMap unmarshals {{.expIdent}} when provided as a mapping node
 //
 // When map encoded, the map key is used as a preset identifier.
@@ -417,6 +417,7 @@ func (d *auxYamlDoc) unmarshal{{ .expIdent }}Node(dctx documentContext, n *yaml.
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
@@ -432,7 +433,7 @@ func (d *auxYamlDoc) unmarshal{{ .expIdent }}Node(dctx documentContext, n *yaml.
 	*/}}
 	{{ $a := . }}
 	{{- range $cmp := $rootRes }}
-		{{ if or ($cmp.envoy.omit) (not $cmp.envoy.use) }}
+		{{ if $cmp.envoy.omit }}
 			{{continue}}
 		{{ end }}
 
@@ -468,7 +469,6 @@ func (d *auxYamlDoc) unmarshal{{ .expIdent }}Node(dctx documentContext, n *yaml.
 		case {{ range $i, $l := $identKeys -}}
 		"{{ $l }}"{{if not (eq $i (sub (len $identKeys) 1))}},{{end}}
 		{{- end}}:
-		default:
 			if y7s.IsSeq(n) {
 				nestedNodes, err = d.unmarshalExtended{{.expIdent}}Seq(dctx, n)
 				if err != nil {
@@ -504,17 +504,30 @@ func (d *auxYamlDoc) unmarshal{{ .expIdent }}Node(dctx documentContext, n *yaml.
 				Scope: scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	{{ if .envoy.yaml.extendedResourcePostProcess}}
+	auxNestedNodes, err = d.postProcessNested{{.expIdent}}Nodes(auxNestedNodes)
+	if err!= nil {
+		return
+	}
+	{{ end }}
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 

server/codegen/assets/templates/gocode/envoy/yaml_encode.go.tpl

@@ -152,7 +152,7 @@ func (e YamlEncoder) encode{{.expIdent}}(ctx context.Context, p envoyx.EncodePar
 	_ = aux
 	{{ $a := . }}
 	{{- range $cmp := $rootRes }}
-		{{ if or ($cmp.envoy.omit) (not $cmp.envoy.use) }}{{continue}}{{ end }}
+		{{ if $cmp.envoy.omit }}{{continue}}{{ end }}
 		{{if not $cmp.parents}}{{continue}}{{end}}
 
 		{{/*

server/codegen/schema/resource.cue

@@ -177,6 +177,7 @@ import (
 		// referencing this resource
 		identKeys: [...string] | *([identKeyLabel]+identKeyAlias)
 
+		extendedResourcePostProcess: bool | *false
 		extendedResourceDecoders: [...{
 			ident: string
 			expIdent: string
@@ -189,6 +190,10 @@ import (
 
 	// store decode/encode configs
 	store: {
+		// enable or disable custom logic after the resource is imported
+		extendedEncoder: bool | *false
+		extendedSubResources: bool | *false
+
 		// enable or disable additional custom processing for determining 
 		// resource references
 		extendedRefDecoder: bool | *false

server/compose/envoy/record_datasource.go

@@ -0,0 +1,73 @@
+package envoy
+
+import (
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/spf13/cast"
+)
+
+type (
+	// RecordDatasource provides a mechanism for you to access large
+	// record datasets optimally
+	RecordDatasource struct {
+		mapping  datasourceMapping
+		provider envoyx.Provider
+
+		// Reusable buffer for reading records
+		rowCache map[string]string
+
+		// Index to map from ref to ID
+		// @todo we might need to flush these to the disc in case a huge dataset is passed in
+		refToID map[string]uint64
+	}
+)
+
+func (rd *RecordDatasource) SetProvider(s envoyx.Provider) bool {
+	if rd.mapping.SourceIdent != s.Ident() {
+		return false
+	}
+
+	rd.provider = s
+	return true
+}
+
+func (rd *RecordDatasource) Next(out map[string]string) (ident string, more bool, err error) {
+	if rd.rowCache == nil {
+		rd.rowCache = make(map[string]string)
+	}
+
+	more, err = rd.provider.Next(rd.rowCache)
+	if err != nil || !more {
+		return
+	}
+
+	rd.applyMapping(rd.rowCache, out)
+
+	ident = out[rd.mapping.KeyField]
+
+	return
+}
+
+func (rd *RecordDatasource) Reset() (err error) {
+	return rd.provider.Reset()
+}
+
+func (rd *RecordDatasource) applyMapping(in, out map[string]string) {
+	for _, m := range rd.mapping.Mapping.m {
+		if m.Skip {
+			continue
+		}
+
+		// @todo expand when needed (expressions and such)
+		out[m.Field] = in[m.Column]
+	}
+}
+
+func (rd *RecordDatasource) ResolveRef(ref any) (out uint64, err error) {
+	r, err := cast.ToStringE(ref)
+	if err != nil {
+		return
+	}
+
+	out = rd.refToID[r]
+	return
+}

server/compose/envoy/record_getter.go

@@ -0,0 +1,102 @@
+package envoy
+
+import (
+	"context"
+
+	"github.com/cortezaproject/corteza/server/compose/dalutils"
+	"github.com/cortezaproject/corteza/server/compose/types"
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/spf13/cast"
+)
+
+type (
+	// recordGetter is a utility struct to resolve record references from
+	// different parts of the system such as the dep graph and the database
+	recordGetter struct {
+		// Bits to get data from the dep graph
+		relDatasource *RecordDatasource
+
+		// Bits to get data from the database
+		dalSvc     dal.FullService
+		relModule  types.Module
+		baseFilter types.RecordFilter
+	}
+)
+
+func makeRecordGetter(dalSvc dal.FullService, tt envoyx.Traverser, n *envoyx.Node, modRef, dsRef envoyx.Ref) (g *recordGetter) {
+	g = &recordGetter{
+		dalSvc: dalSvc,
+	}
+
+	// Resolve from dep graph
+	auxDs := tt.ParentForRef(n, dsRef)
+	if auxDs != nil {
+		g.relDatasource = auxDs.Datasource.(*RecordDatasource)
+	}
+
+	// Resolve from the database
+	mod := g.getRefMod(tt, n, modRef)
+	if mod != nil {
+		g.baseFilter = types.RecordFilter{
+			ModuleID:    mod.ID,
+			NamespaceID: mod.NamespaceID,
+		}
+	}
+
+	return
+}
+
+// resolve resolves the provided reference into a record ID; 0 if can't be resolved
+func (g *recordGetter) resolve(ctx context.Context, ref any) (out uint64, err error) {
+	// Try to get from datasource
+	if g.relDatasource != nil {
+		out, err = g.getDS(ref)
+		if err != nil {
+			return
+		}
+	}
+	if out > 0 {
+		return
+	}
+
+	// Fallback to the store
+	out, err = g.getDB(ctx, ref)
+	if err != nil {
+		return
+	}
+	if out > 0 {
+		return
+	}
+	return
+}
+
+func (g *recordGetter) getDS(ref any) (out uint64, err error) {
+	return g.relDatasource.ResolveRef(ref)
+}
+
+func (g *recordGetter) getDB(ctx context.Context, ref any) (out uint64, err error) {
+	// @note the old version only resolved IDs so that's what we're doing here also
+	// @todo consider expanding this
+	id, err := cast.ToUint64E(ref)
+	if err != nil {
+		return 0, nil
+	}
+
+	aux, err := dalutils.ComposeRecordsFind(ctx, g.dalSvc, &g.relModule, id)
+	if err != nil {
+		return
+	}
+
+	out = aux.ID
+	return
+}
+
+func (g *recordGetter) getRefMod(tt envoyx.Traverser, n *envoyx.Node, ref envoyx.Ref) (refMod *types.Module) {
+	aux := tt.ParentForRef(n, ref)
+	if aux == nil {
+		return nil
+	}
+
+	return aux.Resource.(*types.Module)
+}

server/compose/envoy/store_encode.gen.go

@@ -54,6 +54,8 @@ func (e StoreEncoder) Prepare(ctx context.Context, p envoyx.EncodeParams, rt str
 	case types.PageResourceType:
 		return e.preparePage(ctx, p, s, nn)
 
+	default:
+		return e.prepare(ctx, p, s, rt, nn)
 	}
 
 	return
@@ -406,14 +408,34 @@ func (e StoreEncoder) encodeModule(ctx context.Context, p envoyx.EncodeParams, s
 	// @todo how can we remove the OmitPlaceholderNodes call the same way we did for
 	//       the root function calls?
 
+	nested := make(envoyx.NodeSet, 0, 10)
+
 	for rt, nn := range envoyx.NodesByResourceType(tree.Children(n)...) {
 		nn = envoyx.OmitPlaceholderNodes(nn...)
 
 		switch rt {
 
+		case types.ModuleFieldResourceType:
+			err = e.encodeModuleFields(ctx, p, s, nn, tree)
+			if err != nil {
+				return
+			}
+
+			nested = append(nested, nn...)
+
 		}
 	}
 
+	err = e.encodeModuleExtend(ctx, p, s, n, nested, tree)
+	if err != nil {
+		return
+	}
+
+	err = e.encodeModuleExtendSubResources(ctx, p, s, n, tree)
+	if err != nil {
+		return
+	}
+
 	return
 }
 
@@ -771,6 +793,24 @@ func (e StoreEncoder) encodeNamespace(ctx context.Context, p envoyx.EncodeParams
 
 		switch rt {
 
+		case types.ChartResourceType:
+			err = e.encodeCharts(ctx, p, s, nn, tree)
+			if err != nil {
+				return
+			}
+
+		case types.ModuleResourceType:
+			err = e.encodeModules(ctx, p, s, nn, tree)
+			if err != nil {
+				return
+			}
+
+		case types.PageResourceType:
+			err = e.encodePages(ctx, p, s, nn, tree)
+			if err != nil {
+				return
+			}
+
 		}
 	}
 

server/compose/envoy/store_encode.go

@@ -1,7 +1,14 @@
 package envoy
 
 import (
+	"context"
+	"fmt"
+
+	"github.com/cortezaproject/corteza/server/compose/service"
 	"github.com/cortezaproject/corteza/server/compose/types"
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/store"
 )
 
 func (e StoreEncoder) setChartDefaults(res *types.Chart) (err error) {
@@ -43,3 +50,77 @@ func (e StoreEncoder) setPageDefaults(res *types.Page) (err error) {
 func (e StoreEncoder) validatePage(*types.Page) (err error) {
 	return
 }
+
+func (e StoreEncoder) setRecordDefaults(*types.Record) (err error) {
+	return
+}
+
+func (e StoreEncoder) validateRecord(*types.Record) (err error) {
+	return
+}
+
+func (e StoreEncoder) prepare(ctx context.Context, p envoyx.EncodeParams, s store.Storer, rt string, nn envoyx.NodeSet) (err error) {
+	switch rt {
+	case ComposeRecordDatasourceAuxType:
+		return e.prepareRecordDatasource(ctx, p, s, nn)
+	}
+
+	return
+}
+
+func (e StoreEncoder) encodeModuleExtend(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, nested envoyx.NodeSet, tree envoyx.Traverser) (err error) {
+
+	// Push fields under mod
+	mod := n.Resource.(*types.Module)
+	for _, n := range nested {
+		if n.ResourceType != types.ModuleFieldResourceType {
+			continue
+		}
+
+		mod.Fields = append(mod.Fields, n.Resource.(*types.ModuleField))
+	}
+
+	// Register to DAL
+	dl, err := e.grabDal(p)
+	if err != nil {
+		return
+	}
+
+	nsNode := tree.ParentForRef(n, n.References["NamespaceID"])
+	ns := nsNode.Resource.(*types.Namespace)
+
+	// @todo get connection and things from there
+	model, err := service.ModuleToModel(ns, mod, "compose_record")
+	if err != nil {
+		return err
+	}
+
+	return dl.ReplaceModel(ctx, model)
+}
+
+func (e StoreEncoder) encodeModuleExtendSubResources(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
+	cc := tree.ChildrenForResourceType(n, ComposeRecordDatasourceAuxType)
+
+	dl, err := e.grabDal(p)
+	if err != nil {
+		return
+	}
+
+	return e.encodeRecordDatasources(ctx, p, s, dl, cc, tree)
+}
+
+func (e *StoreEncoder) grabDal(p envoyx.EncodeParams) (dl dal.FullService, err error) {
+	auxs, ok := p.Params["dal"]
+	if !ok {
+		err = fmt.Errorf("dal not defined")
+		return
+	}
+
+	dl, ok = auxs.(dal.FullService)
+	if !ok {
+		err = fmt.Errorf("invalid dal provided")
+		return
+	}
+
+	return
+}

server/compose/envoy/store_encode_datasource.go

@@ -0,0 +1,208 @@
+package envoy
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/cortezaproject/corteza/server/compose/dalutils"
+	"github.com/cortezaproject/corteza/server/compose/types"
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/pkg/id"
+	"github.com/cortezaproject/corteza/server/store"
+)
+
+func (e StoreEncoder) prepareRecordDatasource(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
+	// @todo match existing records; for now use just the ID like V1
+
+	for _, n := range nn {
+		if n.Datasource == nil {
+			panic("unexpected state: cannot call prepareRecordDatasource with nodes without a defined Datasource")
+		}
+
+		ds, ok := n.Datasource.(*RecordDatasource)
+		if !ok {
+			panic("unexpected datasource type: node expecting type of RecordDatasource")
+		}
+
+		err = e.prepareRecords(ctx, p, s, ds)
+		if err != nil {
+			return
+		}
+		err = ds.Reset()
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (e StoreEncoder) prepareRecords(ctx context.Context, p envoyx.EncodeParams, s store.Storer, ds *RecordDatasource) (err error) {
+	var (
+		aux   = make(map[string]string)
+		more  bool
+		ident string
+		rec   types.Record
+	)
+
+	ds.refToID = make(map[string]uint64)
+
+	for {
+		ident, more, err = ds.Next(aux)
+		if err != nil || !more {
+			return
+		}
+
+		ds.refToID[ident] = id.Next()
+
+		rec, err = e.auxToRecord(aux)
+		if err != nil {
+			return err
+		}
+
+		// @note defaults and validation will have to happen again when encoding
+		//       since we won't persist it.
+		//       Consider supporting updating the datasource's data.
+		err = e.setRecordDefaults(&rec)
+		if err != nil {
+			return err
+		}
+
+		err = e.validateRecord(&rec)
+		if err != nil {
+			return err
+		}
+	}
+}
+
+func (e StoreEncoder) encodeRecordDatasources(ctx context.Context, p envoyx.EncodeParams, s store.Storer, dl dal.FullService, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
+	for _, n := range nn {
+		err = e.encodeRecordDatasource(ctx, p, s, dl, n, tree)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (e StoreEncoder) encodeRecordDatasource(ctx context.Context, p envoyx.EncodeParams, s store.Storer, dl dal.FullService, n *envoyx.Node, tree envoyx.Traverser) (err error) {
+	var (
+		auxRec = make(map[string]string)
+		more   bool
+		ident  string
+		rec    types.Record
+
+		nsNode  *envoyx.Node
+		ns      *types.Namespace
+		modNode *envoyx.Node
+		mod     *types.Module
+
+		// This was already validated so we can blindly cast
+		ds = n.Datasource.(*RecordDatasource)
+	)
+
+	// Get the parent namespace
+	nsNode = tree.ParentForRef(n, n.References["NamespaceID"])
+	if nsNode == nil {
+		err = fmt.Errorf("cannot encode record datasource: missing reference for NamespaceID")
+		return
+	}
+	ns = nsNode.Resource.(*types.Namespace)
+
+	// Get the parent module
+	modNode = tree.ParentForRef(n, n.References["ModuleID"])
+	if modNode == nil {
+		err = fmt.Errorf("cannot encode record datasource: missing reference for ModuleID")
+		return
+	}
+	mod = modNode.Resource.(*types.Module)
+
+	// Prepare getters for reference fields
+	// @todo user refs
+	var (
+		modIndex = make(map[string]envoyx.Ref)
+		dsIndex  = make(map[string]envoyx.Ref)
+	)
+	// - first pass collects all the references
+	for k, v := range n.References {
+		if k == "NamespaceID" || k == "ModuleID" {
+			continue
+		}
+
+		// Encoded as <fieldName>.<refKind>
+		pp := strings.Split(k, ".")
+		f := pp[0]
+		kind := pp[1]
+
+		switch kind {
+		case "module":
+			modIndex[f] = v
+		case "datasource":
+			dsIndex[f] = v
+		}
+	}
+
+	// - second pass makes the getters
+	getters := make(map[string]*recordGetter)
+	for k := range modIndex {
+		aux := makeRecordGetter(dl, tree, n, modIndex[k], dsIndex[k])
+		getters[k] = aux
+	}
+
+	// Iterate and encode
+	//
+	// @todo utilize batching
+	for {
+		ident, more, err = ds.Next(auxRec)
+		if err != nil || !more {
+			return
+		}
+
+		rec, err = e.auxToRecord(auxRec)
+		if err != nil {
+			return err
+		}
+
+		rec.NamespaceID = ns.ID
+		rec.ModuleID = mod.ID
+
+		// @todo temp
+		rec.CreatedAt = time.Now()
+
+		// Values and refs
+		rec.ID = ds.refToID[ident]
+		for i, v := range rec.Values {
+			if getters[v.Name] == nil {
+				continue
+			}
+
+			v.Ref, err = getters[v.Name].resolve(ctx, v.Value)
+			if err != nil {
+				return err
+			}
+
+			rec.Values[i] = v
+		}
+
+		// Save it
+		err = dalutils.ComposeRecordCreate(ctx, dl, mod, &rec)
+		if err != nil {
+			return
+		}
+	}
+}
+
+func (e StoreEncoder) auxToRecord(aux map[string]string) (out types.Record, err error) {
+	for k, v := range aux {
+		err = out.SetValue(k, 0, v)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}

server/compose/envoy/yaml_decode.gen.go

@@ -361,6 +361,7 @@ func (d *auxYamlDoc) unmarshalChartNode(dctx documentContext, n *yaml.Node, meta
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
@@ -388,17 +389,24 @@ func (d *auxYamlDoc) unmarshalChartNode(dctx documentContext, n *yaml.Node, meta
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 
@@ -473,6 +481,22 @@ func (d *auxYamlDoc) unmarshalModuleMap(dctx documentContext, n *yaml.Node) (out
 	return
 }
 
+// unmarshalSourceExtendedSeq unmarshals Source when provided as a sequence node
+func (d *auxYamlDoc) unmarshalExtendedSourceSeq(dctx documentContext, n *yaml.Node) (out envoyx.NodeSet, err error) {
+	var aux envoyx.NodeSet
+	err = y7s.EachSeq(n, func(n *yaml.Node) error {
+		aux, err = d.unmarshalSourceExtendedNode(dctx, n)
+		if err != nil {
+			return err
+		}
+		out = append(out, aux...)
+
+		return nil
+	})
+
+	return
+}
+
 // unmarshalModuleNode is a cookie-cutter function to unmarshal
 // the yaml node into the corresponding Corteza type & Node
 func (d *auxYamlDoc) unmarshalModuleNode(dctx documentContext, n *yaml.Node, meta ...*yaml.Node) (out envoyx.NodeSet, err error) {
@@ -615,11 +639,34 @@ func (d *auxYamlDoc) unmarshalModuleNode(dctx documentContext, n *yaml.Node, met
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
 		switch strings.ToLower(k.Value) {
 
+		case "modulefield", "module_fields", "modulefields", "fields":
+			if y7s.IsSeq(n) {
+				nestedNodes, err = d.unmarshalModuleFieldSeq(dctx, n)
+				if err != nil {
+					return err
+				}
+			} else {
+				nestedNodes, err = d.unmarshalModuleFieldMap(dctx, n)
+				if err != nil {
+					return err
+				}
+			}
+			break
+
+		case "source", "datasource":
+			if y7s.IsSeq(n) {
+				nestedNodes, err = d.unmarshalExtendedSourceSeq(dctx, n)
+				if err != nil {
+					return err
+				}
+			}
+			break
 		}
 
 		// Iterate nested nodes and update their reference to the current resource
@@ -642,17 +689,29 @@ func (d *auxYamlDoc) unmarshalModuleNode(dctx documentContext, n *yaml.Node, met
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	auxNestedNodes, err = d.postProcessNestedModuleNodes(auxNestedNodes)
+	if err != nil {
+		return
+	}
+
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 
@@ -802,6 +861,25 @@ func (d *auxYamlDoc) unmarshalModuleFieldNode(dctx documentContext, n *yaml.Node
 
 			break
 
+		case "options":
+
+			// Handle custom node decoder
+			//
+			// The decoder may update the passed resource with arbitrary values
+			// as well as provide additional references and identifiers for the node.
+			var (
+				auxRefs   map[string]envoyx.Ref
+				auxIdents envoyx.Identifiers
+			)
+			auxRefs, auxIdents, err = unmarshalModuleFieldOptionsNode(r, n)
+			if err != nil {
+				return err
+			}
+			refs = envoyx.MergeRefs(refs, auxRefs)
+			ii = ii.Merge(auxIdents)
+
+			break
+
 		// Handle RBAC rules
 		case "allow":
 			auxOut, err = unmarshalAllowNode(n)
@@ -862,6 +940,7 @@ func (d *auxYamlDoc) unmarshalModuleFieldNode(dctx documentContext, n *yaml.Node
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
@@ -889,17 +968,24 @@ func (d *auxYamlDoc) unmarshalModuleFieldNode(dctx documentContext, n *yaml.Node
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 
@@ -1096,11 +1182,68 @@ func (d *auxYamlDoc) unmarshalNamespaceNode(dctx documentContext, n *yaml.Node,
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
 		switch strings.ToLower(k.Value) {
 
+		case "chart", "charts", "chrt":
+			if y7s.IsSeq(n) {
+				nestedNodes, err = d.unmarshalChartSeq(dctx, n)
+				if err != nil {
+					return err
+				}
+			} else {
+				nestedNodes, err = d.unmarshalChartMap(dctx, n)
+				if err != nil {
+					return err
+				}
+			}
+			break
+
+		case "module", "modules", "mod":
+			if y7s.IsSeq(n) {
+				nestedNodes, err = d.unmarshalModuleSeq(dctx, n)
+				if err != nil {
+					return err
+				}
+			} else {
+				nestedNodes, err = d.unmarshalModuleMap(dctx, n)
+				if err != nil {
+					return err
+				}
+			}
+			break
+
+		case "modulefield", "module_fields", "modulefields", "fields":
+			if y7s.IsSeq(n) {
+				nestedNodes, err = d.unmarshalModuleFieldSeq(dctx, n)
+				if err != nil {
+					return err
+				}
+			} else {
+				nestedNodes, err = d.unmarshalModuleFieldMap(dctx, n)
+				if err != nil {
+					return err
+				}
+			}
+			break
+
+		case "page", "pages", "pg":
+			if y7s.IsSeq(n) {
+				nestedNodes, err = d.unmarshalPageSeq(dctx, n)
+				if err != nil {
+					return err
+				}
+			} else {
+				nestedNodes, err = d.unmarshalPageMap(dctx, n)
+				if err != nil {
+					return err
+				}
+			}
+			break
+
 		}
 
 		// Iterate nested nodes and update their reference to the current resource
@@ -1123,17 +1266,24 @@ func (d *auxYamlDoc) unmarshalNamespaceNode(dctx documentContext, n *yaml.Node,
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 
@@ -1369,6 +1519,7 @@ func (d *auxYamlDoc) unmarshalPageNode(dctx documentContext, n *yaml.Node, meta
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
@@ -1396,17 +1547,24 @@ func (d *auxYamlDoc) unmarshalPageNode(dctx documentContext, n *yaml.Node, meta
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 

server/compose/envoy/yaml_decode.go

@@ -10,6 +10,30 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+type (
+	mapEntry struct {
+		Column string
+		Field  string
+		Skip   bool
+	}
+
+	fieldMapping struct {
+		m map[string]mapEntry
+	}
+
+	datasourceMapping struct {
+		SourceIdent string `yaml:"source"`
+		KeyField    string `yaml:"key"`
+		References  map[string]string
+		Scope       map[string]string
+		Mapping     fieldMapping
+	}
+)
+
+const (
+	ComposeRecordDatasourceAuxType = "corteza::compose:record-datasource"
+)
+
 func unmarshalChartConfigNode(r *types.Chart, n *yaml.Node) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
 	err = y7s.EachMap(n, func(k, v *yaml.Node) error {
 		if k.Value != "reports" {
@@ -61,6 +85,177 @@ func unmarshalChartConfigReportNode(r *types.Chart, n *yaml.Node, index int) (re
 	return
 }
 
+func unmarshalModuleFieldOptionsNode(r *types.ModuleField, n *yaml.Node) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
+	refs = make(map[string]envoyx.Ref)
+
+	err = y7s.EachMap(n, func(k, v *yaml.Node) error {
+		switch strings.ToLower(k.Value) {
+		case "module", "module_id", "moduleid":
+			var aux any
+			err = y7s.DecodeScalar(v, "moduleID", &aux)
+			if err != nil {
+				return err
+			}
+			refs["Options.ModuleID"] = envoyx.Ref{
+				ResourceType: types.ModuleResourceType,
+				Identifiers:  envoyx.MakeIdentifiers(aux),
+			}
+		default:
+			return nil
+		}
+		return nil
+	})
+
+	return
+}
+
 func (d *auxYamlDoc) unmarshalYAML(k string, n *yaml.Node) (out envoyx.NodeSet, err error) {
 	return
 }
+
+func (d *auxYamlDoc) postProcessNestedModuleNodes(nn envoyx.NodeSet) (out envoyx.NodeSet, err error) {
+	// Get all references from all module fields
+	refs := make(map[string]envoyx.Ref)
+	for _, n := range nn {
+		if n.ResourceType != types.ModuleFieldResourceType {
+			continue
+		}
+
+		if len(n.References) == 0 {
+			continue
+		}
+
+		r, ok := n.References["Options.ModuleID"]
+		if !ok {
+			continue
+		}
+
+		refs[fmt.Sprintf("%s.module", n.Identifiers.FriendlyIdentifier())] = r
+		refs[fmt.Sprintf("%s.datasource", n.Identifiers.FriendlyIdentifier())] = envoyx.Ref{
+			ResourceType: ComposeRecordDatasourceAuxType,
+			Identifiers:  r.Identifiers,
+			Scope:        r.Scope,
+			Optional:     true,
+		}
+	}
+
+	// Update datasources with references
+
+	for _, n := range nn {
+		if n.Datasource == nil {
+			continue
+		}
+
+		n.References = envoyx.MergeRefs(n.References, refs)
+	}
+	out = nn
+	return
+}
+
+func (d *auxYamlDoc) unmarshalSourceExtendedNode(dctx documentContext, n *yaml.Node, meta ...*yaml.Node) (out envoyx.NodeSet, err error) {
+	var r datasourceMapping
+
+	// @todo we're omitting errors because there will be a bunch due to invalid
+	//       resource field types. This might be a bit unstable as other errors may
+	//       also get ignored.
+	//
+	//       A potential fix would be to firstly unmarshal into an any, check errors
+	//       and then unmarshal into the resource while omitting errors.
+	n.Decode(&r)
+
+	// @todo for now we only support record datasources; extend when needed
+	auxN := &envoyx.Node{
+		Datasource: &RecordDatasource{
+			mapping: r,
+		},
+
+		ResourceType: ComposeRecordDatasourceAuxType,
+		Identifiers:  envoyx.MakeIdentifiers(r.SourceIdent),
+	}
+	auxN.References, auxN.Scope = d.procMappingRefs(r.References)
+	out = append(out, auxN)
+
+	return
+}
+
+// UnmarshalYAML is used to get the yaml parsed into a series of nodes so
+// we can easily pass it down
+func (d *fieldMapping) UnmarshalYAML(n *yaml.Node) (err error) {
+	d.m = make(map[string]mapEntry)
+	if y7s.IsSeq(n) {
+		err = y7s.EachSeq(n, func(n *yaml.Node) error {
+			a, err := d.unmarshalMappingNode(n)
+			d.m[a.Column] = a
+			return err
+		})
+	} else {
+		err = y7s.EachMap(n, func(k, n *yaml.Node) error {
+			a, err := d.unmarshalMappingNode(n)
+			if a.Column == "" {
+				err = y7s.DecodeScalar(k, "fieldMapping column", &a.Column)
+				if err != nil {
+					return err
+				}
+			}
+
+			d.m[a.Column] = a
+			return err
+		})
+	}
+
+	return
+}
+
+func (d *fieldMapping) unmarshalMappingNode(n *yaml.Node) (out mapEntry, err error) {
+	if y7s.IsKind(n, yaml.ScalarNode) {
+		err = y7s.DecodeScalar(n, "Column", &out.Column)
+		if err != nil {
+			return
+		}
+		err = y7s.DecodeScalar(n, "Field", &out.Field)
+		return
+	}
+
+	// @todo we're omitting errors because there will be a bunch due to invalid
+	//       resource field types. This might be a bit unstable as other errors may
+	//       also get ignored.
+	//
+	//       A potential fix would be to firstly unmarshal into an any, check errors
+	//       and then unmarshal into the resource while omitting errors.
+	n.Decode(&out)
+
+	err = y7s.EachMap(n, func(k, v *yaml.Node) error {
+		switch strings.ToLower(k.Value) {
+		case "skip":
+			if v.Value == "/" {
+				out.Skip = true
+			}
+		}
+		return nil
+	})
+
+	return
+}
+
+func (d *auxYamlDoc) procMappingRefs(in map[string]string) (out map[string]envoyx.Ref, scope envoyx.Scope) {
+	out = make(map[string]envoyx.Ref)
+
+	scope = envoyx.Scope{
+		ResourceType: types.NamespaceResourceType,
+		Identifiers:  envoyx.MakeIdentifiers(in["namespace"]),
+	}
+
+	out["NamespaceID"] = envoyx.Ref{
+		ResourceType: types.NamespaceResourceType,
+		Identifiers:  envoyx.MakeIdentifiers(in["namespace"]),
+		Scope:        scope,
+	}
+
+	out["ModuleID"] = envoyx.Ref{
+		ResourceType: types.ModuleResourceType,
+		Identifiers:  envoyx.MakeIdentifiers(in["module"]),
+		Scope:        scope,
+	}
+
+	return
+}

server/compose/envoy/yaml_encode.gen.go

@@ -238,6 +238,17 @@ func (e YamlEncoder) encodeModule(ctx context.Context, p envoyx.EncodeParams, no
 	var aux *yaml.Node
 	_ = aux
 
+	aux, err = e.encodeModuleFields(ctx, p, tt.ChildrenForResourceType(node, types.ModuleFieldResourceType), tt)
+	if err != nil {
+		return
+	}
+	out, err = y7s.AddMap(out,
+		"moduleField", aux,
+	)
+	if err != nil {
+		return
+	}
+
 	return
 }
 
@@ -374,6 +385,39 @@ func (e YamlEncoder) encodeNamespace(ctx context.Context, p envoyx.EncodeParams,
 	var aux *yaml.Node
 	_ = aux
 
+	aux, err = e.encodeCharts(ctx, p, tt.ChildrenForResourceType(node, types.ChartResourceType), tt)
+	if err != nil {
+		return
+	}
+	out, err = y7s.AddMap(out,
+		"chart", aux,
+	)
+	if err != nil {
+		return
+	}
+
+	aux, err = e.encodeModules(ctx, p, tt.ChildrenForResourceType(node, types.ModuleResourceType), tt)
+	if err != nil {
+		return
+	}
+	out, err = y7s.AddMap(out,
+		"module", aux,
+	)
+	if err != nil {
+		return
+	}
+
+	aux, err = e.encodePages(ctx, p, tt.ChildrenForResourceType(node, types.PageResourceType), tt)
+	if err != nil {
+		return
+	}
+	out, err = y7s.AddMap(out,
+		"page", aux,
+	)
+	if err != nil {
+		return
+	}
+
 	return
 }
 

server/compose/module.cue

@@ -89,8 +89,18 @@ module: {
 			supportMappedInput: true
 			mappedField: "Handle"
 			identKeyAlias: ["modules", "mod"]
+
+			extendedResourcePostProcess: true
+			extendedResourceDecoders: [{
+				ident: "source"
+				expIdent: "Source"
+				identKeys: ["source", "datasource"]
+				supportMappedInput: false
+			}]
 		}
 		store: {
+			extendedEncoder: true
+			extendedSubResources: true
 			extendedFilterBuilder: true
 			extendedDecoder: true
 		}

server/compose/module_field.cue

@@ -39,6 +39,11 @@ moduleField: {
 				dal: { type: "JSON", defaultEmptyObject: true }
 				omitSetter: true
 				omitGetter: true
+				envoy: {
+					yaml: {
+						customDecoder: true
+					}
+				}
 			}
 			name: {
 				sortable: true

server/compose/page.cue

@@ -92,7 +92,7 @@ page: {
 
 	filter: {
 		struct: {
-			page_id: { goType: "uint64", ident: "pageID", storeIdent: "id" }
+			page_id: { goType: "[]uint64", ident: "pageID", storeIdent: "id" }
 			namespace_id: { goType: "uint64", ident: "namespaceID", storeIdent: "rel_namespace" }
 			parent_id: { goType: "uint64", ident: "parentID" }
 			module_id: { goType: "uint64", ident: "moduleID", storeIdent: "rel_module" }

server/compose/record.cue

@@ -13,6 +13,9 @@ record: {
 	model: {
 		ident: "compose_record"
 
+		defaultSetter: true
+		defaultGetter: true
+
 		attributes: {
 			id: schema.IdField
 			revision: {
@@ -71,9 +74,6 @@ record: {
 		omit: true
 	}
 
-	defaultGetter: true
-	defaultSetter: true
-
 	filter: {
 		struct: {
 			module_id: { goType: "uint64" }

server/compose/types/getters_setters.gen.go

@@ -351,6 +351,9 @@ func (r *Record) GetValue(name string, pos uint) (any, error) {
 	case "updatedBy", "UpdatedBy":
 		return r.UpdatedBy, nil
 
+	default:
+		return r.getValue(name, pos)
+
 	}
 	return nil, nil
 }
@@ -382,6 +385,9 @@ func (r *Record) SetValue(name string, pos uint, value any) (err error) {
 	case "updatedBy", "UpdatedBy":
 		return cast2.Uint64(value, &r.UpdatedBy)
 
+	default:
+		return r.setValue(name, pos, value)
+
 	}
 	return nil
 }

server/pkg/envoyx/csv/decoder.go

@@ -57,7 +57,7 @@ func Decoder(r io.Reader, ident string) (out *decoder, err error) {
 	}
 
 	r, err = out.flushTemp(r)
-	defer out.src.Seek(0, 0)
+	defer out.Reset()
 	if err != nil {
 		return
 	}
@@ -111,12 +111,12 @@ func (d *decoder) Reset() error {
 
 // Next returns the field: value mapping for the next row
 func (d *decoder) Next(out map[string]string) (more bool, err error) {
-	if !d.skipHead {
+	if d.skipHead {
 		_, err = d.reader.Read()
 		if err != nil {
 			return
 		}
-		d.skipHead = true
+		d.skipHead = false
 	}
 
 	aux, err := d.reader.Read()

server/pkg/envoyx/datasource.go

@@ -0,0 +1,29 @@
+package envoyx
+
+type (
+	Provider interface {
+		Next(out map[string]string) (more bool, err error)
+		Reset() error
+		Ident() string
+	}
+
+	Datasource interface {
+		Next(out map[string]string) (ident string, more bool, err error)
+		Reset() error
+		SetProvider(Provider) bool
+	}
+)
+
+func SetDecoderSources(nn NodeSet, dd ...Provider) {
+	for _, n := range nn {
+		if n.Datasource == nil {
+			continue
+		}
+
+		for _, d := range dd {
+			if n.Datasource.SetProvider(d) {
+				break
+			}
+		}
+	}
+}

server/pkg/envoyx/envoy.go

@@ -137,6 +137,8 @@ func (svc *service) Decode(ctx context.Context, p DecodeParams) (nn NodeSet, err
 		return svc.decodeUri(ctx, p)
 	case DecodeTypeStore:
 		return svc.decodeStore(ctx, p)
+	default:
+		err = fmt.Errorf("unsupported decoder type %s", p.Type)
 	}
 
 	return
@@ -156,7 +158,8 @@ func (svc *service) Encode(ctx context.Context, p EncodeParams, dg *depGraph) (e
 		return svc.encodeStore(ctx, dg, p)
 	case EncodeTypeIo:
 		return svc.encodeIo(ctx, dg, p)
-
+	default:
+		err = fmt.Errorf("unsupported encoder type %s", p.Type)
 	}
 	return
 }

server/pkg/envoyx/node.go

@@ -9,7 +9,8 @@ import (
 type (
 	// Node is a wrapper around a Corteza resource for use within Envoy
 	Node struct {
-		Resource resource
+		Resource   resource
+		Datasource Datasource
 
 		ResourceType string
 		Identifiers  Identifiers
@@ -57,6 +58,9 @@ type (
 		ResourceType string
 		Identifiers  Identifiers
 		Scope        Scope
+		// @todo consider replacing with something that indicates
+		//       it can't be fetched from the DB
+		Optional bool
 	}
 )
 

server/pkg/envoyx/util_rbac.gen.go

@@ -275,6 +275,11 @@ func SplitResourceIdentifier(ref string) (out map[string]Ref) {
 			return
 		}
 
+		out["0"] = Ref{
+			ResourceType: "corteza::compose:namespace",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+		}
+
 		if gRef(pp, 1) == "" {
 			return
 		}
@@ -289,6 +294,11 @@ func SplitResourceIdentifier(ref string) (out map[string]Ref) {
 			return
 		}
 
+		out["0"] = Ref{
+			ResourceType: "corteza::compose:namespace",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+		}
+
 		if gRef(pp, 1) == "" {
 			return
 		}
@@ -303,10 +313,20 @@ func SplitResourceIdentifier(ref string) (out map[string]Ref) {
 			return
 		}
 
+		out["0"] = Ref{
+			ResourceType: "corteza::compose:namespace",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+		}
+
 		if gRef(pp, 1) == "" {
 			return
 		}
 
+		out["1"] = Ref{
+			ResourceType: "corteza::compose:module",
+			Identifiers:  MakeIdentifiers(gRef(pp, 1)),
+		}
+
 		if gRef(pp, 2) == "" {
 			return
 		}
@@ -331,6 +351,11 @@ func SplitResourceIdentifier(ref string) (out map[string]Ref) {
 			return
 		}
 
+		out["0"] = Ref{
+			ResourceType: "corteza::compose:namespace",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+		}
+
 		if gRef(pp, 1) == "" {
 			return
 		}
@@ -345,10 +370,20 @@ func SplitResourceIdentifier(ref string) (out map[string]Ref) {
 			return
 		}
 
+		out["0"] = Ref{
+			ResourceType: "corteza::compose:namespace",
+			Identifiers:  MakeIdentifiers(gRef(pp, 0)),
+		}
+
 		if gRef(pp, 1) == "" {
 			return
 		}
 
+		out["1"] = Ref{
+			ResourceType: "corteza::compose:module",
+			Identifiers:  MakeIdentifiers(gRef(pp, 1)),
+		}
+
 		if gRef(pp, 2) == "" {
 			return
 		}

server/pkg/envoyx/utils.gen.go

@@ -25,12 +25,14 @@ var (
 		"corteza::federation:exposed-module": true,
 		"corteza::federation:module-mapping": true,
 
-		"corteza::federation:shared-module": true,
+		"corteza::federation:shared-module":  true,
+		"corteza::compose:record-datasource": true,
 	}
 
 	// superNeedyResources is the second level of filtering in case the first
 	// pass removes everything
 	superNeedyResources = map[string]bool{
-		"corteza::compose:module-field": true,
+		"corteza::compose:module-field":      true,
+		"corteza::compose:record-datasource": true,
 	}
 )

server/store/adapters/rdbms/filters.gen.go

@@ -203,8 +203,8 @@ func ApigwFilterFilter(d drivers.Dialect, f systemType.ApigwFilterFilter) (ee []
 		ee = append(ee, expr)
 	}
 
-	if f.ApigwFilterID > 0 {
-		ee = append(ee, goqu.C("id").Eq(f.ApigwFilterID))
+	if len(f.ApigwFilterID) > 0 {
+		ee = append(ee, goqu.C("id").In(f.ApigwFilterID))
 	}
 
 	if f.RouteID > 0 {
@@ -231,8 +231,8 @@ func ApigwRouteFilter(d drivers.Dialect, f systemType.ApigwRouteFilter) (ee []go
 		ee = append(ee, expr)
 	}
 
-	if len(f.ApigwrouteID) > 0 {
-		ee = append(ee, goqu.C("id").In(f.ApigwrouteID))
+	if len(f.ApigwRouteID) > 0 {
+		ee = append(ee, goqu.C("id").In(f.ApigwRouteID))
 	}
 
 	if val := strings.TrimSpace(f.Route); len(val) > 0 {
@@ -651,8 +651,8 @@ func ComposePageFilter(d drivers.Dialect, f composeType.PageFilter) (ee []goqu.E
 		ee = append(ee, expr)
 	}
 
-	if f.PageID > 0 {
-		ee = append(ee, goqu.C("id").Eq(f.PageID))
+	if len(f.PageID) > 0 {
+		ee = append(ee, goqu.C("id").In(f.PageID))
 	}
 
 	if val := strings.TrimSpace(f.Handle); len(val) > 0 {
@@ -723,8 +723,8 @@ func DalConnectionFilter(d drivers.Dialect, f systemType.DalConnectionFilter) (e
 		ee = append(ee, expr)
 	}
 
-	if len(f.ConnectionID) > 0 {
-		ee = append(ee, goqu.C("id").In(f.ConnectionID))
+	if len(f.DalConnectionID) > 0 {
+		ee = append(ee, goqu.C("id").In(f.DalConnectionID))
 	}
 
 	if val := strings.TrimSpace(f.Handle); len(val) > 0 {
@@ -751,8 +751,8 @@ func DalSensitivityLevelFilter(d drivers.Dialect, f systemType.DalSensitivityLev
 		ee = append(ee, expr)
 	}
 
-	if len(f.SensitivityLevelID) > 0 {
-		ee = append(ee, goqu.C("id").In(f.SensitivityLevelID))
+	if len(f.DalSensitivityLevelID) > 0 {
+		ee = append(ee, goqu.C("id").In(f.DalSensitivityLevelID))
 	}
 
 	if val := strings.TrimSpace(f.Handle); len(val) > 0 {
@@ -1002,8 +1002,8 @@ func QueueFilter(d drivers.Dialect, f systemType.QueueFilter) (ee []goqu.Express
 		ee = append(ee, expr)
 	}
 
-	if f.QueueID > 0 {
-		ee = append(ee, goqu.C("id").Eq(f.QueueID))
+	if len(f.QueueID) > 0 {
+		ee = append(ee, goqu.C("id").In(f.QueueID))
 	}
 
 	if f.Query != "" {

server/system/apigw_filter.cue

@@ -69,7 +69,7 @@ apigw_filter: {
 
 	filter: {
 		struct: {
-			apigw_filter_id: {goType: "uint64", ident: "apigwFilterID", storeIdent: "id"}
+			apigw_filter_id: {goType: "[]uint64", ident: "apigwFilterID", storeIdent: "id"}
 			route_id: {goType: "uint64", ident: "routeID", storeIdent: "rel_route"}
 			deleted:  {goType: "filter.State", storeIdent: "deleted_at"}
 			disabled: {goType: "filter.State", storeIdent: "enabled"}

server/system/apigw_route.cue

@@ -73,7 +73,7 @@ apigw_route: {
 
 	filter: {
 		struct: {
-			apigw_route_id: { goType: "[]uint64", ident: "apigwrouteID", storeIdent: "id" }
+			apigw_route_id: { goType: "[]uint64", ident: "apigwRouteID", storeIdent: "id" }
 			route: {goType: "string", storeIdent: "id"}
 			endpoint: {goType: "string"}
 			method: {goType: "string"}

server/system/dal_connection.cue

@@ -42,7 +42,7 @@ dal_connection: {
 
 	filter: {
 		struct: {
-			dal_connection_id: {goType: "[]uint64", ident: "connectionID", storeIdent: "id"}
+			dal_connection_id: {goType: "[]uint64", ident: "dalConnectionID", storeIdent: "id"}
 			handle: {goType: "string"}
 			type: {goType: "string"}
 

server/system/dal_sensitivity_level.cue

@@ -36,7 +36,7 @@ dal_sensitivity_level: {
 
 	filter: {
 		struct: {
-			dal_sensitivity_level_id: {goType: "[]uint64", ident: "sensitivityLevelID", storeIdent: "id"}
+			dal_sensitivity_level_id: {goType: "[]uint64", ident: "dalSensitivityLevelID", storeIdent: "id"}
 			handle: { goType: "string" }
 
 			deleted: {goType: "filter.State", storeIdent: "deleted_at"}

server/system/envoy/store_encode.gen.go

@@ -69,6 +69,8 @@ func (e StoreEncoder) Prepare(ctx context.Context, p envoyx.EncodeParams, rt str
 		return e.prepareDalConnection(ctx, p, s, nn)
 	case types.DalSensitivityLevelResourceType:
 		return e.prepareDalSensitivityLevel(ctx, p, s, nn)
+	default:
+		return e.prepare(ctx, p, s, rt, nn)
 	}
 
 	return

server/system/envoy/yaml_decode.gen.go

@@ -370,6 +370,7 @@ func (d *auxYamlDoc) unmarshalApplicationNode(dctx documentContext, n *yaml.Node
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
@@ -397,17 +398,24 @@ func (d *auxYamlDoc) unmarshalApplicationNode(dctx documentContext, n *yaml.Node
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 
@@ -622,6 +630,7 @@ func (d *auxYamlDoc) unmarshalApigwRouteNode(dctx documentContext, n *yaml.Node,
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
@@ -649,17 +658,24 @@ func (d *auxYamlDoc) unmarshalApigwRouteNode(dctx documentContext, n *yaml.Node,
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 
@@ -838,6 +854,7 @@ func (d *auxYamlDoc) unmarshalApigwFilterNode(dctx documentContext, n *yaml.Node
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
@@ -865,17 +882,24 @@ func (d *auxYamlDoc) unmarshalApigwFilterNode(dctx documentContext, n *yaml.Node
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 
@@ -1091,6 +1115,7 @@ func (d *auxYamlDoc) unmarshalAuthClientNode(dctx documentContext, n *yaml.Node,
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
@@ -1118,17 +1143,24 @@ func (d *auxYamlDoc) unmarshalAuthClientNode(dctx documentContext, n *yaml.Node,
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 
@@ -1351,6 +1383,7 @@ func (d *auxYamlDoc) unmarshalQueueNode(dctx documentContext, n *yaml.Node, meta
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
@@ -1378,17 +1411,24 @@ func (d *auxYamlDoc) unmarshalQueueNode(dctx documentContext, n *yaml.Node, meta
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 
@@ -1615,6 +1655,7 @@ func (d *auxYamlDoc) unmarshalReportNode(dctx documentContext, n *yaml.Node, met
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
@@ -1642,17 +1683,24 @@ func (d *auxYamlDoc) unmarshalReportNode(dctx documentContext, n *yaml.Node, met
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 
@@ -1825,6 +1873,7 @@ func (d *auxYamlDoc) unmarshalRoleNode(dctx documentContext, n *yaml.Node, meta
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
@@ -1852,17 +1901,24 @@ func (d *auxYamlDoc) unmarshalRoleNode(dctx documentContext, n *yaml.Node, meta
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 
@@ -2048,6 +2104,7 @@ func (d *auxYamlDoc) unmarshalTemplateNode(dctx documentContext, n *yaml.Node, m
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
@@ -2075,17 +2132,24 @@ func (d *auxYamlDoc) unmarshalTemplateNode(dctx documentContext, n *yaml.Node, m
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 
@@ -2258,6 +2322,7 @@ func (d *auxYamlDoc) unmarshalUserNode(dctx documentContext, n *yaml.Node, meta
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
@@ -2285,17 +2350,24 @@ func (d *auxYamlDoc) unmarshalUserNode(dctx documentContext, n *yaml.Node, meta
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 
@@ -2507,6 +2579,7 @@ func (d *auxYamlDoc) unmarshalDalConnectionNode(dctx documentContext, n *yaml.No
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
@@ -2534,17 +2607,24 @@ func (d *auxYamlDoc) unmarshalDalConnectionNode(dctx documentContext, n *yaml.No
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 
@@ -2739,6 +2819,7 @@ func (d *auxYamlDoc) unmarshalDalSensitivityLevelNode(dctx documentContext, n *y
 	// This operation is done in the second pass of the document so we have
 	// the complete context of the current resource; such as the identifier,
 	// references, and scope.
+	var auxNestedNodes envoyx.NodeSet
 	err = y7s.EachMap(n, func(k, n *yaml.Node) error {
 		nestedNodes = nil
 
@@ -2766,17 +2847,24 @@ func (d *auxYamlDoc) unmarshalDalSensitivityLevelNode(dctx documentContext, n *y
 				Scope:        scope,
 			}
 
+			for f, ref := range a.References {
+				ref.Scope = scope
+				a.References[f] = ref
+			}
+
 			for f, ref := range refs {
 				a.References[f] = ref
 			}
 		}
-		auxOut = append(auxOut, nestedNodes...)
+		auxNestedNodes = append(auxNestedNodes, nestedNodes...)
 		return nil
 	})
 	if err != nil {
 		return
 	}
 
+	out = append(out, auxNestedNodes...)
+
 	a := &envoyx.Node{
 		Resource: r,
 

server/system/queue.cue

@@ -48,7 +48,7 @@ queue: {
 
 	filter: {
 		struct: {
-			queue_id: {goType: "uint64", ident: "queueID", storeIdent: "id"}
+			queue_id: {goType: "[]uint64", ident: "queueID", storeIdent: "id"}
 			query: {goType: "string"}
 			deleted: {goType: "filter.State", storeIdent: "deleted_at"}
 		}

server/system/types/user.go

@@ -3,6 +3,7 @@ package types
 import (
 	"database/sql/driver"
 	"encoding/json"
+	"fmt"
 	"time"
 
 	"github.com/cortezaproject/corteza/server/pkg/sql"
@@ -117,9 +118,9 @@ const (
 	SystemUser UserKind = "sys"
 )
 
-// func (u User) String() string {
-// 	return fmt.Sprintf("%d", u.ID)
-// }
+func (u User) String() string {
+	return fmt.Sprintf("%d", u.ID)
+}
 
 func (u *User) Valid() bool {
 	return u.ID > 0 && u.SuspendedAt == nil && u.DeletedAt == nil