Cleanup record encoding logic, add tests

server/automation/envoy/yaml_decode.gen.go

@@ -28,7 +28,8 @@ type (
 	// which are then managed by envoy and imported via an encoder.
 	YamlDecoder     struct{}
 	documentContext struct {
-		references map[string]string
+		references  map[string]string
+		parentIdent envoyx.Identifiers
 	}
 	auxYamlDoc struct {
 		nodes envoyx.NodeSet
@@ -351,6 +352,9 @@ func (d *auxYamlDoc) unmarshalWorkflowNode(dctx documentContext, n *yaml.Node, m
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Apply the scope to all of the references of the same type
 	for k, ref := range refs {
 		if ref.ResourceType != scope.ResourceType {
@@ -636,6 +640,9 @@ func (d *auxYamlDoc) unmarshalTriggerNode(dctx documentContext, n *yaml.Node, me
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Apply the scope to all of the references of the same type
 	for k, ref := range refs {
 		if ref.ResourceType != scope.ResourceType {

server/codegen/assets/templates/gocode/envoy/yaml_decode.go.tpl

@@ -30,6 +30,7 @@ type (
 	YamlDecoder struct{}
 	documentContext struct {
 		references map[string]string
+		parentIdent envoyx.Identifiers
 	}
 	auxYamlDoc  struct {
 		nodes envoyx.NodeSet
@@ -393,6 +394,9 @@ func (d *auxYamlDoc) unmarshal{{ .expIdent }}Node(dctx documentContext, n *yaml.
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	{{ if eq $cmpIdent "compose" }}
 	// Handle global namespace reference which can be provided as the doc. context
 	//

server/compose/envoy/store_encode_datasource.go

@@ -7,11 +7,25 @@ import (
 	"time"
 
 	"github.com/cortezaproject/corteza/server/compose/dalutils"
+	"github.com/cortezaproject/corteza/server/compose/service"
+	"github.com/cortezaproject/corteza/server/compose/service/values"
 	"github.com/cortezaproject/corteza/server/compose/types"
 	"github.com/cortezaproject/corteza/server/pkg/dal"
 	"github.com/cortezaproject/corteza/server/pkg/envoyx"
 	"github.com/cortezaproject/corteza/server/pkg/id"
 	"github.com/cortezaproject/corteza/server/store"
+	systemEnvoy "github.com/cortezaproject/corteza/server/system/envoy"
+	"github.com/spf13/cast"
+)
+
+const (
+	recordBatchMaxChunk = 100
+)
+
+var (
+	rvSanitizer = values.Sanitizer()
+	rvValidator = values.Validator()
+	rvFormatter = values.Formatter()
 )
 
 func (e StoreEncoder) prepareRecordDatasource(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
@@ -45,7 +59,6 @@ func (e StoreEncoder) prepareRecords(ctx context.Context, p envoyx.EncodeParams,
 		aux   = make(map[string]string)
 		more  bool
 		ident []string
-		rec   types.Record
 	)
 
 	ds.refToID = make(map[string]uint64)
@@ -58,9 +71,11 @@ func (e StoreEncoder) prepareRecords(ctx context.Context, p envoyx.EncodeParams,
 
 		ds.AddRef(id.Next(), ident...)
 
-		rec, err = e.auxToRecord(aux)
-		if err != nil {
-			return err
+		// Construct a simple record for basic validation/preprocessing
+		rec := types.Record{}
+		for k, v := range aux {
+			// Ignore errors at this point since some values can't yet be properly casted
+			rec.SetValue(k, 0, v)
 		}
 
 		// @note defaults and validation will have to happen again when encoding
@@ -94,7 +109,6 @@ func (e StoreEncoder) encodeRecordDatasource(ctx context.Context, p envoyx.Encod
 		auxRec = make(map[string]string)
 		more   bool
 		ident  []string
-		rec    types.Record
 
 		nsNode  *envoyx.Node
 		ns      *types.Namespace
@@ -121,13 +135,77 @@ func (e StoreEncoder) encodeRecordDatasource(ctx context.Context, p envoyx.Encod
 	}
 	mod = modNode.Resource.(*types.Module)
 
-	// Prepare getters for reference fields
-	// @todo user refs
+	// Prepare getters for related resources
+	recordGetters := e.makeRecordGetters(dl, tree, n)
+	userGetters := e.makeUserGetters(s, dl, tree, mod, n)
+
+	maykr := e.recordMaker(ns, mod, recordGetters, userGetters)
+
+	var (
+		rec     types.Record
+		records types.RecordSet
+		rve     *types.RecordValueErrorSet
+	)
+	for {
+		ident, more, err = ds.Next(ctx, auxRec)
+		if err != nil || !more {
+			break
+		}
+		rec, err = maykr(ctx, auxRec)
+		if err != nil {
+			return
+		}
+
+		// Do these at the end so they can't be overwritten
+		rec.NamespaceID = ns.ID
+		rec.ModuleID = mod.ID
+		rec.CreatedAt = time.Now()
+		rec.OwnedBy = service.CalcRecordOwner(0, rec.OwnedBy, p.Encoder.DefaultUserID)
+		rec.ID, err = ds.ResolveRefS(ident...)
+		if err != nil {
+			return err
+		}
+
+		// Standard record processing
+		rec.Values.SetUpdatedFlag(true)
+		//
+		rve = service.RecordValueUpdateOpCheck(ctx, nil, mod, rec.Values)
+		if !rve.IsValid() {
+			return rve
+		}
+		//
+		rve = service.RecordPreparer(ctx, s, rvSanitizer, rvValidator, rvFormatter, mod, &rec)
+		if !rve.IsValid() {
+			return rve
+		}
+
+		ax := rec
+		records = append(records, &ax)
+
+		if len(records) > recordBatchMaxChunk {
+			err = dalutils.ComposeRecordCreate(ctx, dl, mod, records...)
+			if err != nil {
+				return
+			}
+
+			records = make(types.RecordSet, 0, recordBatchMaxChunk/2)
+		}
+	}
+
+	if len(records) > 0 {
+		err = dalutils.ComposeRecordCreate(ctx, dl, mod, records...)
+	}
+	return
+}
+
+// maceRecordGetters returns a map of getters where the key is the field name
+func (e StoreEncoder) makeRecordGetters(dl dal.FullService, tree envoyx.Traverser, n *envoyx.Node) (getters map[string]*recordGetter) {
 	var (
 		modIndex = make(map[string]envoyx.Ref)
 		dsIndex  = make(map[string]envoyx.Ref)
 	)
-	// - first pass collects all the references
+
+	// First pass collects all the references
 	for k, v := range n.References {
 		if k == "NamespaceID" || k == "ModuleID" {
 			continue
@@ -147,65 +225,103 @@ func (e StoreEncoder) encodeRecordDatasource(ctx context.Context, p envoyx.Encod
 	}
 
 	// - second pass makes the getters
-	getters := make(map[string]*recordGetter)
+	getters = make(map[string]*recordGetter)
 	for k := range modIndex {
 		getters[k] = makeRecordGetter(dl, tree, n, modIndex[k], dsIndex[k])
 	}
 
-	// Iterate and encode
-	//
-	// @todo utilize batching
-	for {
-		ident, more, err = ds.Next(ctx, auxRec)
-		if err != nil || !more {
-			return
-		}
-
-		rec, err = e.auxToRecord(auxRec)
-		if err != nil {
-			return err
-		}
-
-		rec.NamespaceID = ns.ID
-		rec.ModuleID = mod.ID
-
-		// @todo temp
-		rec.CreatedAt = time.Now()
-
-		// Values and refs
-		rec.ID, err = ds.ResolveRefS(ident...)
-		if err != nil {
-			return err
-		}
-
-		for i, v := range rec.Values {
-			if getters[v.Name] == nil {
-				continue
-			}
-
-			v.Ref, err = getters[v.Name].resolve(ctx, v.Value)
-			if err != nil {
-				return err
-			}
-
-			rec.Values[i] = v
-		}
-
-		// Save it
-		err = dalutils.ComposeRecordCreate(ctx, dl, mod, &rec)
-		if err != nil {
-			return
-		}
-	}
+	return
 }
 
-func (e StoreEncoder) auxToRecord(aux map[string]string) (out types.Record, err error) {
-	for k, v := range aux {
-		err = out.SetValue(k, 0, v)
-		if err != nil {
-			return
+func (e StoreEncoder) makeUserGetters(s store.Storer, dl dal.FullService, tree envoyx.Traverser, mod *types.Module, n *envoyx.Node) (getters map[string]*systemEnvoy.UserGetter) {
+	userGetter := systemEnvoy.MakeUserGetter(s, tree)
+	getters = map[string]*systemEnvoy.UserGetter{
+		// These are all of the supported sys user ref fields
+		"ownedby":    userGetter,
+		"owned_by":   userGetter,
+		"createdby":  userGetter,
+		"created_by": userGetter,
+		"updatedby":  userGetter,
+		"updated_by": userGetter,
+		"deletedby":  userGetter,
+		"deleted_by": userGetter,
+	}
+	for _, f := range mod.Fields {
+		if f.Kind == "User" {
+			getters[f.Name] = userGetter
 		}
 	}
 
 	return
 }
+
+func (e StoreEncoder) recordMaker(ns *types.Namespace, mod *types.Module, recordGetters map[string]*recordGetter, userGetters map[string]*systemEnvoy.UserGetter) func(ctx context.Context, auxRec map[string]string) (r types.Record, err error) {
+	return func(ctx context.Context, auxRec map[string]string) (rec types.Record, err error) {
+		// Iterate mapified record values and populate the provided values
+		var auxv *types.RecordValue
+		for k, v := range auxRec {
+			if recordGetters[k] != nil {
+				auxv, err = e.resolveRecordRef(ctx, recordGetters, k, v)
+				if err != nil {
+					return
+				}
+				err = rec.SetValue(k, auxv.Place, auxv.Ref)
+				if err != nil {
+					return
+				}
+				continue
+			}
+
+			if userGetters[k] != nil {
+				auxv, err = e.resolveUserRef(ctx, userGetters, k, v)
+				if err != nil {
+					return
+				}
+				err = rec.SetValue(k, auxv.Place, auxv.Ref)
+				if err != nil {
+					return
+				}
+				continue
+			}
+
+			// Default is regular values
+			err = rec.SetValue(k, 0, v)
+			if err != nil {
+				return
+			}
+		}
+
+		return
+	}
+}
+
+func (e StoreEncoder) resolveRecordRef(ctx context.Context, getters map[string]*recordGetter, k, v string) (out *types.RecordValue, err error) {
+	if getters[k] == nil {
+		return nil, nil
+	}
+
+	out = &types.RecordValue{Name: k}
+	out.Ref, err = getters[out.Name].resolve(ctx, v)
+	if err != nil {
+		return
+	}
+	out.Value = cast.ToString(out.Ref)
+
+	return out, nil
+}
+
+func (e StoreEncoder) resolveUserRef(ctx context.Context, getters map[string]*systemEnvoy.UserGetter, k, v string) (out *types.RecordValue, err error) {
+	k = strings.ToLower(k)
+	if getters[k] == nil {
+		return nil, nil
+	}
+
+	out = &types.RecordValue{Name: k}
+	out.Ref, err = getters[out.Name].Resolve(ctx, v)
+	if err != nil {
+		return
+	}
+	out.Value = cast.ToString(out.Ref)
+
+	return out, nil
+}

server/compose/envoy/yaml_decode.gen.go

@@ -28,7 +28,8 @@ type (
 	// which are then managed by envoy and imported via an encoder.
 	YamlDecoder     struct{}
 	documentContext struct {
-		references map[string]string
+		references  map[string]string
+		parentIdent envoyx.Identifiers
 	}
 	auxYamlDoc struct {
 		nodes envoyx.NodeSet
@@ -319,6 +320,9 @@ func (d *auxYamlDoc) unmarshalChartNode(dctx documentContext, n *yaml.Node, meta
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Handle global namespace reference which can be provided as the doc. context
 	//
 	// @todo this is a temporary solution and should be extended when the document
@@ -606,6 +610,9 @@ func (d *auxYamlDoc) unmarshalModuleNode(dctx documentContext, n *yaml.Node, met
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Handle global namespace reference which can be provided as the doc. context
 	//
 	// @todo this is a temporary solution and should be extended when the document
@@ -961,6 +968,9 @@ func (d *auxYamlDoc) unmarshalModuleFieldNode(dctx documentContext, n *yaml.Node
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Handle global namespace reference which can be provided as the doc. context
 	//
 	// @todo this is a temporary solution and should be extended when the document
@@ -1213,6 +1223,9 @@ func (d *auxYamlDoc) unmarshalNamespaceNode(dctx documentContext, n *yaml.Node,
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Handle global namespace reference which can be provided as the doc. context
 	//
 	// @todo this is a temporary solution and should be extended when the document
@@ -1643,6 +1656,9 @@ func (d *auxYamlDoc) unmarshalPageNode(dctx documentContext, n *yaml.Node, meta
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Handle global namespace reference which can be provided as the doc. context
 	//
 	// @todo this is a temporary solution and should be extended when the document

server/compose/envoy/yaml_decode.go

@@ -433,7 +433,7 @@ func (d *auxYamlDoc) unmarshalSourceExtendedNode(dctx documentContext, n *yaml.N
 		},
 
 		ResourceType: ComposeRecordDatasourceAuxType,
-		Identifiers:  envoyx.MakeIdentifiers(r.SourceIdent),
+		Identifiers:  dctx.parentIdent,
 	}
 	auxN.References, auxN.Scope = d.procMappingRefs(r.References)
 	out = append(out, auxN)

server/compose/record.cue

@@ -54,10 +54,18 @@ record: {
 			created_at: schema.SortableTimestampNowField
 			updated_at: schema.SortableTimestampNilField
 			deleted_at: schema.SortableTimestampNilField
-			owned_by:   schema.AttributeUserRef
-			created_by: schema.AttributeUserRef
-			updated_by: schema.AttributeUserRef
-			deleted_by: schema.AttributeUserRef
+			owned_by:   schema.AttributeUserRef & {
+				identAlias: ["ownedBy", "OwnedBy", "owned_by"]
+			}
+			created_by: schema.AttributeUserRef & {
+				identAlias: ["createdBy", "CreatedBy", "created_by"]
+			}
+			updated_by: schema.AttributeUserRef & {
+				identAlias: ["updatedBy", "UpdatedBy", "updated_by"]
+			}
+			deleted_by: schema.AttributeUserRef & {
+				identAlias: ["deletedBy", "DeletedBy", "deleted_by"]
+			}
 		}
 
 		indexes: {

server/compose/types/getters_setters.gen.go

@@ -337,11 +337,11 @@ func (r *Record) GetValue(name string, pos uint) (any, error) {
 	switch name {
 	case "createdAt", "CreatedAt":
 		return r.CreatedAt, nil
-	case "createdBy", "CreatedBy":
+	case "createdBy", "CreatedBy", "created_by":
 		return r.CreatedBy, nil
 	case "deletedAt", "DeletedAt":
 		return r.DeletedAt, nil
-	case "deletedBy", "DeletedBy":
+	case "deletedBy", "DeletedBy", "deleted_by":
 		return r.DeletedBy, nil
 	case "id", "ID":
 		return r.ID, nil
@@ -351,13 +351,13 @@ func (r *Record) GetValue(name string, pos uint) (any, error) {
 		return r.ModuleID, nil
 	case "namespaceID", "NamespaceID":
 		return r.NamespaceID, nil
-	case "ownedBy", "OwnedBy":
+	case "ownedBy", "OwnedBy", "owned_by":
 		return r.OwnedBy, nil
 	case "revision", "Revision":
 		return r.Revision, nil
 	case "updatedAt", "UpdatedAt":
 		return r.UpdatedAt, nil
-	case "updatedBy", "UpdatedBy":
+	case "updatedBy", "UpdatedBy", "updated_by":
 		return r.UpdatedBy, nil
 
 	default:
@@ -371,11 +371,11 @@ func (r *Record) SetValue(name string, pos uint, value any) (err error) {
 	switch name {
 	case "createdAt", "CreatedAt":
 		return cast2.Time(value, &r.CreatedAt)
-	case "createdBy", "CreatedBy":
+	case "createdBy", "CreatedBy", "created_by":
 		return cast2.Uint64(value, &r.CreatedBy)
 	case "deletedAt", "DeletedAt":
 		return cast2.TimePtr(value, &r.DeletedAt)
-	case "deletedBy", "DeletedBy":
+	case "deletedBy", "DeletedBy", "deleted_by":
 		return cast2.Uint64(value, &r.DeletedBy)
 	case "id", "ID":
 		return cast2.Uint64(value, &r.ID)
@@ -385,13 +385,13 @@ func (r *Record) SetValue(name string, pos uint, value any) (err error) {
 		return cast2.Uint64(value, &r.ModuleID)
 	case "namespaceID", "NamespaceID":
 		return cast2.Uint64(value, &r.NamespaceID)
-	case "ownedBy", "OwnedBy":
+	case "ownedBy", "OwnedBy", "owned_by":
 		return cast2.Uint64(value, &r.OwnedBy)
 	case "revision", "Revision":
 		return cast2.Uint(value, &r.Revision)
 	case "updatedAt", "UpdatedAt":
 		return cast2.TimePtr(value, &r.UpdatedAt)
-	case "updatedBy", "UpdatedBy":
+	case "updatedBy", "UpdatedBy", "updated_by":
 		return cast2.Uint64(value, &r.UpdatedBy)
 
 	default:

server/compose/types/record.go

@@ -219,21 +219,26 @@ func (r *Record) setValue(name string, pos uint, value any) (err error) {
 	}
 
 	rv := &RecordValue{Name: name, Place: pos}
-	var auxv string
 
-	switch aux := value.(type) {
-	case *time.Time:
-		auxv = aux.Format(time.RFC3339)
+	if cv, ok := value.(*RecordValue); ok {
+		rv = cv
+	} else {
+		var auxv string
+		switch aux := value.(type) {
+		case *time.Time:
+			auxv = aux.Format(time.RFC3339)
 
-	case time.Time:
-		auxv = aux.Format(time.RFC3339)
+		case time.Time:
+			auxv = aux.Format(time.RFC3339)
 
-	default:
-		auxv, err = cast.ToStringE(aux)
-	}
+		default:
+			auxv, err = cast.ToStringE(aux)
+		}
+		if err != nil {
+			return
+		}
 
-	if err != nil {
-		return
+		rv.Value = auxv
 	}
 
 	// Try to utilize the module when possible
@@ -248,7 +253,6 @@ func (r *Record) setValue(name string, pos uint, value any) (err error) {
 		}
 	}
 
-	rv.Value = auxv
 	r.Values = r.Values.Set(rv)
 
 	return

server/pkg/envoyx/dep.go

@@ -312,6 +312,17 @@ func (g DepGraph) ChildrenForResourceType(n *Node, rt string) (out NodeSet) {
 	return
 }
 
+func (g DepGraph) NodeForRef(ref Ref) (out *Node) {
+	aux := make([]*Node, 0, 10)
+	for _, sg := range g.graphs {
+		for _, n := range sg.nodes {
+			aux = append(aux, n.Node)
+		}
+	}
+
+	return NodeForRef(ref, aux...)
+}
+
 // Children returns all child nodes of n
 func (g DepGraph) Children(n *Node) (out NodeSet) {
 	for _, sg := range g.graphs {

server/pkg/envoyx/envoy.go

@@ -30,6 +30,9 @@ type (
 
 		// Children returns all of the children of the provided node
 		Children(*Node) NodeSet
+
+		// NodeForRef returns the node which matches the provided ref
+		NodeForRef(Ref) *Node
 	}
 
 	Preparer interface {
@@ -77,6 +80,7 @@ type (
 		Encoder EncoderConfig
 	}
 	EncoderConfig struct {
+		DefaultUserID       uint64
 		PreferredTimeLayout string
 		PreferredTimezone   string
 	}

server/system/envoy/user_getter.go

@@ -0,0 +1,88 @@
+package envoy
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoyx"
+	"github.com/cortezaproject/corteza/server/store"
+	"github.com/cortezaproject/corteza/server/system/types"
+	"github.com/spf13/cast"
+)
+
+type (
+	// UserGetter is a utility struct to resolve user references from
+	// different parts of the system such as the dep graph and the database
+	UserGetter struct {
+		depGraph *envoyx.DepGraph
+
+		store      store.Storer
+		baseFilter types.UserFilter
+	}
+)
+
+func MakeUserGetter(s store.Storer, tt envoyx.Traverser) (g *UserGetter) {
+	g = &UserGetter{
+		store: s,
+	}
+
+	g.baseFilter = types.UserFilter{}
+
+	return
+}
+
+// Resolve returns the user ID for the provided reference
+//
+// If the user can not be resolved, 0 is returned.
+func (g *UserGetter) Resolve(ctx context.Context, ref any) (out uint64, err error) {
+	// Try to get from datasource
+	if g.depGraph != nil {
+		out, err = g.getDS(ref)
+		if err != nil {
+			return
+		}
+	}
+	if out > 0 {
+		return
+	}
+
+	// Try to get from the database
+	out, err = g.getDB(ctx, ref)
+	return
+}
+
+func (g *UserGetter) getDS(ref any) (out uint64, err error) {
+	n := g.depGraph.NodeForRef(envoyx.Ref{
+		ResourceType: types.UserResourceType,
+		Identifiers:  envoyx.MakeIdentifiers(ref),
+	})
+
+	if n == nil {
+		return
+	}
+
+	return n.Resource.GetID(), nil
+}
+
+// @todo this can be improved by prefetching and indexing refs
+func (g *UserGetter) getDB(ctx context.Context, ref any) (out uint64, err error) {
+	f := g.baseFilter
+	// @todo expand this
+	f.Query = cast.ToString(ref)
+
+	set, _, err := store.SearchUsers(ctx, g.store, f)
+	if err != nil {
+		return
+	}
+
+	if len(set) == 0 {
+		return
+	}
+
+	if len(set) > 1 {
+		err = fmt.Errorf("ambiguous reference %v: matches more then one user", ref)
+		return
+	}
+
+	return set[0].ID, nil
+}

server/system/envoy/yaml_decode.gen.go

@@ -28,7 +28,8 @@ type (
 	// which are then managed by envoy and imported via an encoder.
 	YamlDecoder     struct{}
 	documentContext struct {
-		references map[string]string
+		references  map[string]string
+		parentIdent envoyx.Identifiers
 	}
 	auxYamlDoc struct {
 		nodes envoyx.NodeSet
@@ -380,6 +381,9 @@ func (d *auxYamlDoc) unmarshalApplicationNode(dctx documentContext, n *yaml.Node
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Apply the scope to all of the references of the same type
 	for k, ref := range refs {
 		if ref.ResourceType != scope.ResourceType {
@@ -690,6 +694,9 @@ func (d *auxYamlDoc) unmarshalApigwRouteNode(dctx documentContext, n *yaml.Node,
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Apply the scope to all of the references of the same type
 	for k, ref := range refs {
 		if ref.ResourceType != scope.ResourceType {
@@ -956,6 +963,9 @@ func (d *auxYamlDoc) unmarshalApigwFilterNode(dctx documentContext, n *yaml.Node
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Apply the scope to all of the references of the same type
 	for k, ref := range refs {
 		if ref.ResourceType != scope.ResourceType {
@@ -1261,6 +1271,9 @@ func (d *auxYamlDoc) unmarshalAuthClientNode(dctx documentContext, n *yaml.Node,
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Apply the scope to all of the references of the same type
 	for k, ref := range refs {
 		if ref.ResourceType != scope.ResourceType {
@@ -1546,6 +1559,9 @@ func (d *auxYamlDoc) unmarshalQueueNode(dctx documentContext, n *yaml.Node, meta
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Apply the scope to all of the references of the same type
 	for k, ref := range refs {
 		if ref.ResourceType != scope.ResourceType {
@@ -1850,6 +1866,9 @@ func (d *auxYamlDoc) unmarshalReportNode(dctx documentContext, n *yaml.Node, met
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Apply the scope to all of the references of the same type
 	for k, ref := range refs {
 		if ref.ResourceType != scope.ResourceType {
@@ -2078,6 +2097,9 @@ func (d *auxYamlDoc) unmarshalRoleNode(dctx documentContext, n *yaml.Node, meta
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Apply the scope to all of the references of the same type
 	for k, ref := range refs {
 		if ref.ResourceType != scope.ResourceType {
@@ -2325,6 +2347,9 @@ func (d *auxYamlDoc) unmarshalTemplateNode(dctx documentContext, n *yaml.Node, m
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Apply the scope to all of the references of the same type
 	for k, ref := range refs {
 		if ref.ResourceType != scope.ResourceType {
@@ -2572,6 +2597,9 @@ func (d *auxYamlDoc) unmarshalUserNode(dctx documentContext, n *yaml.Node, meta
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Apply the scope to all of the references of the same type
 	for k, ref := range refs {
 		if ref.ResourceType != scope.ResourceType {
@@ -2857,6 +2885,9 @@ func (d *auxYamlDoc) unmarshalDalConnectionNode(dctx documentContext, n *yaml.No
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Apply the scope to all of the references of the same type
 	for k, ref := range refs {
 		if ref.ResourceType != scope.ResourceType {
@@ -3125,6 +3156,9 @@ func (d *auxYamlDoc) unmarshalDalSensitivityLevelNode(dctx documentContext, n *y
 		return
 	}
 
+	// Make parent identifiers available through the dctx
+	dctx.parentIdent = ii
+
 	// Apply the scope to all of the references of the same type
 	for k, ref := range refs {
 		if ref.ResourceType != scope.ResourceType {

server/tests/envoy/records_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strconv"
 	"testing"
 
 	"github.com/cortezaproject/corteza/server/compose/dalutils"
@@ -12,6 +13,7 @@ import (
 	"github.com/cortezaproject/corteza/server/pkg/dal"
 	"github.com/cortezaproject/corteza/server/pkg/envoyx"
 	"github.com/cortezaproject/corteza/server/store"
+	systemTypes "github.com/cortezaproject/corteza/server/system/types"
 	"github.com/davecgh/go-spew/spew"
 	"github.com/stretchr/testify/require"
 )
@@ -196,6 +198,146 @@ func TestRecordsImportExport(t *testing.T) {
 	})
 }
 
+func TestRecordsXrefs(t *testing.T) {
+	var (
+		ctx       = context.Background()
+		req       = require.New(t)
+		nodes     envoyx.NodeSet
+		providers []envoyx.Provider
+		gg        *envoyx.DepGraph
+		err       error
+	)
+	_ = gg
+
+	cleanup(t)
+
+	t.Run("parse configs", func(t *testing.T) {
+		nodes, providers, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
+			Type: envoyx.DecodeTypeURI,
+			Params: map[string]any{
+				"uri": "file://testdata/records/cross_ref",
+			},
+		})
+		req.NoError(err)
+	})
+
+	t.Run("bake", func(t *testing.T) {
+		gg, err = defaultEnvoy.Bake(ctx, envoyx.EncodeParams{
+			Type: envoyx.EncodeTypeStore,
+			Params: map[string]any{
+				"storer": defaultStore,
+				"dal":    defaultDal,
+			},
+		}, providers, nodes...)
+		req.NoError(err)
+	})
+
+	t.Run("import into DB", func(t *testing.T) {
+		err = defaultEnvoy.Encode(ctx, envoyx.EncodeParams{
+			Type: envoyx.EncodeTypeStore,
+			Params: map[string]any{
+				"storer": defaultStore,
+				"dal":    defaultDal,
+			},
+		}, gg)
+		req.NoError(err)
+	})
+
+	ns, err := store.LookupComposeNamespaceBySlug(ctx, defaultStore, "test_ns")
+	req.NoError(err)
+
+	modParent, err := store.LookupComposeModuleByNamespaceIDHandle(ctx, defaultStore, ns.ID, "mod_parent")
+	req.NoError(err)
+
+	modChild, err := store.LookupComposeModuleByNamespaceIDHandle(ctx, defaultStore, ns.ID, "mod_child")
+	req.NoError(err)
+
+	parentRecords, _, err := dalutils.ComposeRecordsList(ctx, defaultDal, modParent, types.RecordFilter{})
+	req.NoError(err)
+
+	compareValues(req, parentRecords[0].Values, types.RecordValueSet{{Name: "name", Value: "Jane Doe"}, {Name: "code", Value: "3124"}})
+	compareValues(req, parentRecords[1].Values, types.RecordValueSet{{Name: "name", Value: "Ana Banana"}, {Name: "code", Value: "4153"}})
+	compareValues(req, parentRecords[2].Values, types.RecordValueSet{{Name: "name", Value: "Qwerty"}, {Name: "code", Value: "83172"}})
+
+	childRecords, _, err := dalutils.ComposeRecordsList(ctx, defaultDal, modChild, types.RecordFilter{})
+	req.NoError(err)
+
+	compareValues(req, childRecords[0].Values, types.RecordValueSet{{Name: "user_name", Value: "Daniel Adams", Ref: 0}, {Name: "parent", Value: strconv.FormatUint(parentRecords[0].ID, 10), Ref: parentRecords[0].ID}, {Name: "alt", Value: strconv.FormatUint(childRecords[2].ID, 10), Ref: childRecords[2].ID}})
+	compareValues(req, childRecords[1].Values, types.RecordValueSet{{Name: "user_name", Value: "Jane Alex", Ref: 0}, {Name: "parent", Value: strconv.FormatUint(parentRecords[1].ID, 10), Ref: parentRecords[1].ID}, {Name: "alt", Value: strconv.FormatUint(childRecords[0].ID, 10), Ref: childRecords[0].ID}})
+	compareValues(req, childRecords[2].Values, types.RecordValueSet{{Name: "user_name", Value: "Fish", Ref: 0}, {Name: "parent", Value: strconv.FormatUint(parentRecords[2].ID, 10), Ref: parentRecords[2].ID}, {Name: "alt", Value: strconv.FormatUint(childRecords[1].ID, 10), Ref: childRecords[1].ID}})
+}
+
+func TestRecordsUserRefs(t *testing.T) {
+	var (
+		ctx       = context.Background()
+		req       = require.New(t)
+		nodes     envoyx.NodeSet
+		providers []envoyx.Provider
+		gg        *envoyx.DepGraph
+		err       error
+	)
+	_ = gg
+
+	cleanup(t)
+
+	t.Run("parse configs", func(t *testing.T) {
+		nodes, providers, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
+			Type: envoyx.DecodeTypeURI,
+			Params: map[string]any{
+				"uri": "file://testdata/records/user_ref",
+			},
+		})
+		req.NoError(err)
+	})
+
+	t.Run("bake", func(t *testing.T) {
+		gg, err = defaultEnvoy.Bake(ctx, envoyx.EncodeParams{
+			Type: envoyx.EncodeTypeStore,
+			Params: map[string]any{
+				"storer": defaultStore,
+				"dal":    defaultDal,
+			},
+		}, providers, nodes...)
+		req.NoError(err)
+	})
+
+	t.Run("import into DB", func(t *testing.T) {
+		err = defaultEnvoy.Encode(ctx, envoyx.EncodeParams{
+			Type: envoyx.EncodeTypeStore,
+			Params: map[string]any{
+				"storer": defaultStore,
+				"dal":    defaultDal,
+			},
+		}, gg)
+		req.NoError(err)
+	})
+
+	ns, err := store.LookupComposeNamespaceBySlug(ctx, defaultStore, "test_ns")
+	req.NoError(err)
+
+	mod, err := store.LookupComposeModuleByNamespaceIDHandle(ctx, defaultStore, ns.ID, "test_mod")
+	req.NoError(err)
+
+	rr, _, err := dalutils.ComposeRecordsList(ctx, defaultDal, mod, types.RecordFilter{})
+	req.NoError(err)
+
+	uu, _, err := store.SearchUsers(ctx, defaultStore, systemTypes.UserFilter{})
+	req.NoError(err)
+
+	usr0 := uu[0]
+	usr1 := uu[1]
+	usr2 := uu[2]
+	usrx := uu[3]
+
+	compareValues(req, rr[0].Values, types.RecordValueSet{{Name: "name", Value: "Jane Doe"}, {Name: "code", Value: "3124"}, {Name: "rel_user", Value: strconv.FormatUint(usr0.ID, 10), Ref: usr0.ID}})
+	compareValues(req, rr[1].Values, types.RecordValueSet{{Name: "name", Value: "Ana Banana"}, {Name: "code", Value: "4153"}, {Name: "rel_user", Value: strconv.FormatUint(usr1.ID, 10), Ref: usr1.ID}})
+	compareValues(req, rr[2].Values, types.RecordValueSet{{Name: "name", Value: "Qwerty"}, {Name: "code", Value: "83172"}, {Name: "rel_user", Value: strconv.FormatUint(usr2.ID, 10), Ref: usr2.ID}})
+
+	req.Equal(usrx.ID, rr[0].CreatedBy)
+	req.Equal(usrx.ID, rr[1].CreatedBy)
+	req.Equal(usrx.ID, rr[2].CreatedBy)
+}
+
 func assertRecordState(ctx context.Context, t *testing.T, s store.Storer, dl dal.FullService, req *require.Assertions) {
 	t.Run("check state", func(t *testing.T) {
 		ns, err := store.LookupComposeNamespaceBySlug(ctx, defaultStore, "test_ns_1")

server/tests/envoy/testdata/records/cross_ref/child_records.csv

@@ -0,0 +1,4 @@
+ID,user_name,parent,alt
+c11,Daniel Adams,p1,c13
+c12,Jane Alex,p2,c11
+c13,Fish,p3,c12

server/tests/envoy/testdata/records/cross_ref/def.yaml

@@ -0,0 +1,49 @@
+namespace:
+  test_ns:
+    name: Test Namespace
+    modules:
+      mod_parent:
+        name: Test Namespace Module Parent
+        fields:
+          name:
+            title: Name
+          code:
+            title: Code
+            kind: Number
+        source:
+          - from: parent_records.csv
+            key: ID
+            map:
+              - column: ID
+                skip: true
+              - column: name
+                field: name
+              - column: code
+                field: code
+      mod_child:
+        name: Test Namespace Module Child
+        fields:
+          user_name:
+            title: User Name
+          parent:
+            title: Parent
+            kind: Record
+            options:
+              module: mod_parent
+          alt:
+            title: alt
+            kind: Record
+            options:
+              module: mod_child
+        source:
+          - from: child_records.csv
+            key: ID
+            map:
+              - column: ID
+                skip: true
+              - column: user_name
+                field: user_name
+              - column: parent
+                field: parent
+              - column: alt
+                field: alt

server/tests/envoy/testdata/records/cross_ref/parent_records.csv

@@ -0,0 +1,4 @@
+ID,name,code
+p1,Jane Doe,3124
+p2,Ana Banana,4153
+p3,Qwerty,83172

server/tests/envoy/testdata/records/user_ref/def.yaml

@@ -0,0 +1,39 @@
+namespace:
+  test_ns:
+    name: Test Namespace
+    modules:
+      test_mod:
+        name: Test Namespace Module
+        fields:
+          name:
+            title: Name
+          code:
+            title: Code
+            kind: Number
+          rel_user:
+            title: Rel User
+            kind: User
+        source:
+          - from: records.csv
+            key: ID
+            map:
+              - column: ID
+                skip: true
+              - column: name
+                field: name
+              - column: code
+                field: code
+              - column: rel_user
+                field: rel_user
+              - column: created_by
+                field: created_by
+
+user:
+  usr_0:
+    email: usr_0@test.tld
+  usr_1:
+    email: usr_1@test.tld
+  usr_2:
+    email: usr_2@test.tld
+  usr_x:
+    email: usr_x@test.tld

server/tests/envoy/testdata/records/user_ref/records.csv

@@ -0,0 +1,4 @@
+ID,name,code,rel_user,created_by
+p1,Jane Doe,3124,usr_0,usr_x
+p2,Ana Banana,4153,usr_1,usr_x
+p3,Qwerty,83172,usr_2,usr_x