diff --git a/server/automation/envoy/yaml_decode.gen.go b/server/automation/envoy/yaml_decode.gen.go index 4476b9a20..3d6e6426f 100644 --- a/server/automation/envoy/yaml_decode.gen.go +++ b/server/automation/envoy/yaml_decode.gen.go @@ -28,7 +28,8 @@ type ( // which are then managed by envoy and imported via an encoder. YamlDecoder struct{} documentContext struct { - references map[string]string + references map[string]string + parentIdent envoyx.Identifiers } auxYamlDoc struct { nodes envoyx.NodeSet @@ -351,6 +352,9 @@ func (d *auxYamlDoc) unmarshalWorkflowNode(dctx documentContext, n *yaml.Node, m return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Apply the scope to all of the references of the same type for k, ref := range refs { if ref.ResourceType != scope.ResourceType { @@ -636,6 +640,9 @@ func (d *auxYamlDoc) unmarshalTriggerNode(dctx documentContext, n *yaml.Node, me return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Apply the scope to all of the references of the same type for k, ref := range refs { if ref.ResourceType != scope.ResourceType { diff --git a/server/codegen/assets/templates/gocode/envoy/yaml_decode.go.tpl b/server/codegen/assets/templates/gocode/envoy/yaml_decode.go.tpl index 779620294..e3daa03c6 100644 --- a/server/codegen/assets/templates/gocode/envoy/yaml_decode.go.tpl +++ b/server/codegen/assets/templates/gocode/envoy/yaml_decode.go.tpl @@ -30,6 +30,7 @@ type ( YamlDecoder struct{} documentContext struct { references map[string]string + parentIdent envoyx.Identifiers } auxYamlDoc struct { nodes envoyx.NodeSet @@ -393,6 +394,9 @@ func (d *auxYamlDoc) unmarshal{{ .expIdent }}Node(dctx documentContext, n *yaml. return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + {{ if eq $cmpIdent "compose" }} // Handle global namespace reference which can be provided as the doc. context // diff --git a/server/compose/envoy/store_encode_datasource.go b/server/compose/envoy/store_encode_datasource.go index bf5b2b848..b5812aaf3 100644 --- a/server/compose/envoy/store_encode_datasource.go +++ b/server/compose/envoy/store_encode_datasource.go @@ -7,11 +7,25 @@ import ( "time" "github.com/cortezaproject/corteza/server/compose/dalutils" + "github.com/cortezaproject/corteza/server/compose/service" + "github.com/cortezaproject/corteza/server/compose/service/values" "github.com/cortezaproject/corteza/server/compose/types" "github.com/cortezaproject/corteza/server/pkg/dal" "github.com/cortezaproject/corteza/server/pkg/envoyx" "github.com/cortezaproject/corteza/server/pkg/id" "github.com/cortezaproject/corteza/server/store" + systemEnvoy "github.com/cortezaproject/corteza/server/system/envoy" + "github.com/spf13/cast" +) + +const ( + recordBatchMaxChunk = 100 +) + +var ( + rvSanitizer = values.Sanitizer() + rvValidator = values.Validator() + rvFormatter = values.Formatter() ) func (e StoreEncoder) prepareRecordDatasource(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) { @@ -45,7 +59,6 @@ func (e StoreEncoder) prepareRecords(ctx context.Context, p envoyx.EncodeParams, aux = make(map[string]string) more bool ident []string - rec types.Record ) ds.refToID = make(map[string]uint64) @@ -58,9 +71,11 @@ func (e StoreEncoder) prepareRecords(ctx context.Context, p envoyx.EncodeParams, ds.AddRef(id.Next(), ident...) - rec, err = e.auxToRecord(aux) - if err != nil { - return err + // Construct a simple record for basic validation/preprocessing + rec := types.Record{} + for k, v := range aux { + // Ignore errors at this point since some values can't yet be properly casted + rec.SetValue(k, 0, v) } // @note defaults and validation will have to happen again when encoding @@ -94,7 +109,6 @@ func (e StoreEncoder) encodeRecordDatasource(ctx context.Context, p envoyx.Encod auxRec = make(map[string]string) more bool ident []string - rec types.Record nsNode *envoyx.Node ns *types.Namespace @@ -121,13 +135,77 @@ func (e StoreEncoder) encodeRecordDatasource(ctx context.Context, p envoyx.Encod } mod = modNode.Resource.(*types.Module) - // Prepare getters for reference fields - // @todo user refs + // Prepare getters for related resources + recordGetters := e.makeRecordGetters(dl, tree, n) + userGetters := e.makeUserGetters(s, dl, tree, mod, n) + + maykr := e.recordMaker(ns, mod, recordGetters, userGetters) + + var ( + rec types.Record + records types.RecordSet + rve *types.RecordValueErrorSet + ) + for { + ident, more, err = ds.Next(ctx, auxRec) + if err != nil || !more { + break + } + rec, err = maykr(ctx, auxRec) + if err != nil { + return + } + + // Do these at the end so they can't be overwritten + rec.NamespaceID = ns.ID + rec.ModuleID = mod.ID + rec.CreatedAt = time.Now() + rec.OwnedBy = service.CalcRecordOwner(0, rec.OwnedBy, p.Encoder.DefaultUserID) + rec.ID, err = ds.ResolveRefS(ident...) + if err != nil { + return err + } + + // Standard record processing + rec.Values.SetUpdatedFlag(true) + // + rve = service.RecordValueUpdateOpCheck(ctx, nil, mod, rec.Values) + if !rve.IsValid() { + return rve + } + // + rve = service.RecordPreparer(ctx, s, rvSanitizer, rvValidator, rvFormatter, mod, &rec) + if !rve.IsValid() { + return rve + } + + ax := rec + records = append(records, &ax) + + if len(records) > recordBatchMaxChunk { + err = dalutils.ComposeRecordCreate(ctx, dl, mod, records...) + if err != nil { + return + } + + records = make(types.RecordSet, 0, recordBatchMaxChunk/2) + } + } + + if len(records) > 0 { + err = dalutils.ComposeRecordCreate(ctx, dl, mod, records...) + } + return +} + +// maceRecordGetters returns a map of getters where the key is the field name +func (e StoreEncoder) makeRecordGetters(dl dal.FullService, tree envoyx.Traverser, n *envoyx.Node) (getters map[string]*recordGetter) { var ( modIndex = make(map[string]envoyx.Ref) dsIndex = make(map[string]envoyx.Ref) ) - // - first pass collects all the references + + // First pass collects all the references for k, v := range n.References { if k == "NamespaceID" || k == "ModuleID" { continue @@ -147,65 +225,103 @@ func (e StoreEncoder) encodeRecordDatasource(ctx context.Context, p envoyx.Encod } // - second pass makes the getters - getters := make(map[string]*recordGetter) + getters = make(map[string]*recordGetter) for k := range modIndex { getters[k] = makeRecordGetter(dl, tree, n, modIndex[k], dsIndex[k]) } - // Iterate and encode - // - // @todo utilize batching - for { - ident, more, err = ds.Next(ctx, auxRec) - if err != nil || !more { - return - } - - rec, err = e.auxToRecord(auxRec) - if err != nil { - return err - } - - rec.NamespaceID = ns.ID - rec.ModuleID = mod.ID - - // @todo temp - rec.CreatedAt = time.Now() - - // Values and refs - rec.ID, err = ds.ResolveRefS(ident...) - if err != nil { - return err - } - - for i, v := range rec.Values { - if getters[v.Name] == nil { - continue - } - - v.Ref, err = getters[v.Name].resolve(ctx, v.Value) - if err != nil { - return err - } - - rec.Values[i] = v - } - - // Save it - err = dalutils.ComposeRecordCreate(ctx, dl, mod, &rec) - if err != nil { - return - } - } + return } -func (e StoreEncoder) auxToRecord(aux map[string]string) (out types.Record, err error) { - for k, v := range aux { - err = out.SetValue(k, 0, v) - if err != nil { - return +func (e StoreEncoder) makeUserGetters(s store.Storer, dl dal.FullService, tree envoyx.Traverser, mod *types.Module, n *envoyx.Node) (getters map[string]*systemEnvoy.UserGetter) { + userGetter := systemEnvoy.MakeUserGetter(s, tree) + getters = map[string]*systemEnvoy.UserGetter{ + // These are all of the supported sys user ref fields + "ownedby": userGetter, + "owned_by": userGetter, + "createdby": userGetter, + "created_by": userGetter, + "updatedby": userGetter, + "updated_by": userGetter, + "deletedby": userGetter, + "deleted_by": userGetter, + } + for _, f := range mod.Fields { + if f.Kind == "User" { + getters[f.Name] = userGetter } } return } + +func (e StoreEncoder) recordMaker(ns *types.Namespace, mod *types.Module, recordGetters map[string]*recordGetter, userGetters map[string]*systemEnvoy.UserGetter) func(ctx context.Context, auxRec map[string]string) (r types.Record, err error) { + return func(ctx context.Context, auxRec map[string]string) (rec types.Record, err error) { + // Iterate mapified record values and populate the provided values + var auxv *types.RecordValue + for k, v := range auxRec { + if recordGetters[k] != nil { + auxv, err = e.resolveRecordRef(ctx, recordGetters, k, v) + if err != nil { + return + } + err = rec.SetValue(k, auxv.Place, auxv.Ref) + if err != nil { + return + } + continue + } + + if userGetters[k] != nil { + auxv, err = e.resolveUserRef(ctx, userGetters, k, v) + if err != nil { + return + } + err = rec.SetValue(k, auxv.Place, auxv.Ref) + if err != nil { + return + } + continue + } + + // Default is regular values + err = rec.SetValue(k, 0, v) + if err != nil { + return + } + } + + return + } +} + +func (e StoreEncoder) resolveRecordRef(ctx context.Context, getters map[string]*recordGetter, k, v string) (out *types.RecordValue, err error) { + if getters[k] == nil { + return nil, nil + } + + out = &types.RecordValue{Name: k} + out.Ref, err = getters[out.Name].resolve(ctx, v) + if err != nil { + return + } + out.Value = cast.ToString(out.Ref) + + return out, nil +} + +func (e StoreEncoder) resolveUserRef(ctx context.Context, getters map[string]*systemEnvoy.UserGetter, k, v string) (out *types.RecordValue, err error) { + k = strings.ToLower(k) + if getters[k] == nil { + return nil, nil + } + + out = &types.RecordValue{Name: k} + out.Ref, err = getters[out.Name].Resolve(ctx, v) + if err != nil { + return + } + out.Value = cast.ToString(out.Ref) + + return out, nil +} diff --git a/server/compose/envoy/yaml_decode.gen.go b/server/compose/envoy/yaml_decode.gen.go index 78f2aafc7..ba9e9a8d5 100644 --- a/server/compose/envoy/yaml_decode.gen.go +++ b/server/compose/envoy/yaml_decode.gen.go @@ -28,7 +28,8 @@ type ( // which are then managed by envoy and imported via an encoder. YamlDecoder struct{} documentContext struct { - references map[string]string + references map[string]string + parentIdent envoyx.Identifiers } auxYamlDoc struct { nodes envoyx.NodeSet @@ -319,6 +320,9 @@ func (d *auxYamlDoc) unmarshalChartNode(dctx documentContext, n *yaml.Node, meta return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Handle global namespace reference which can be provided as the doc. context // // @todo this is a temporary solution and should be extended when the document @@ -606,6 +610,9 @@ func (d *auxYamlDoc) unmarshalModuleNode(dctx documentContext, n *yaml.Node, met return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Handle global namespace reference which can be provided as the doc. context // // @todo this is a temporary solution and should be extended when the document @@ -961,6 +968,9 @@ func (d *auxYamlDoc) unmarshalModuleFieldNode(dctx documentContext, n *yaml.Node return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Handle global namespace reference which can be provided as the doc. context // // @todo this is a temporary solution and should be extended when the document @@ -1213,6 +1223,9 @@ func (d *auxYamlDoc) unmarshalNamespaceNode(dctx documentContext, n *yaml.Node, return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Handle global namespace reference which can be provided as the doc. context // // @todo this is a temporary solution and should be extended when the document @@ -1643,6 +1656,9 @@ func (d *auxYamlDoc) unmarshalPageNode(dctx documentContext, n *yaml.Node, meta return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Handle global namespace reference which can be provided as the doc. context // // @todo this is a temporary solution and should be extended when the document diff --git a/server/compose/envoy/yaml_decode.go b/server/compose/envoy/yaml_decode.go index e776fa8fb..92ba77742 100644 --- a/server/compose/envoy/yaml_decode.go +++ b/server/compose/envoy/yaml_decode.go @@ -433,7 +433,7 @@ func (d *auxYamlDoc) unmarshalSourceExtendedNode(dctx documentContext, n *yaml.N }, ResourceType: ComposeRecordDatasourceAuxType, - Identifiers: envoyx.MakeIdentifiers(r.SourceIdent), + Identifiers: dctx.parentIdent, } auxN.References, auxN.Scope = d.procMappingRefs(r.References) out = append(out, auxN) diff --git a/server/compose/record.cue b/server/compose/record.cue index f83d7be2b..66747d87a 100644 --- a/server/compose/record.cue +++ b/server/compose/record.cue @@ -54,10 +54,18 @@ record: { created_at: schema.SortableTimestampNowField updated_at: schema.SortableTimestampNilField deleted_at: schema.SortableTimestampNilField - owned_by: schema.AttributeUserRef - created_by: schema.AttributeUserRef - updated_by: schema.AttributeUserRef - deleted_by: schema.AttributeUserRef + owned_by: schema.AttributeUserRef & { + identAlias: ["ownedBy", "OwnedBy", "owned_by"] + } + created_by: schema.AttributeUserRef & { + identAlias: ["createdBy", "CreatedBy", "created_by"] + } + updated_by: schema.AttributeUserRef & { + identAlias: ["updatedBy", "UpdatedBy", "updated_by"] + } + deleted_by: schema.AttributeUserRef & { + identAlias: ["deletedBy", "DeletedBy", "deleted_by"] + } } indexes: { diff --git a/server/compose/types/getters_setters.gen.go b/server/compose/types/getters_setters.gen.go index d31c474be..48c797fa0 100644 --- a/server/compose/types/getters_setters.gen.go +++ b/server/compose/types/getters_setters.gen.go @@ -337,11 +337,11 @@ func (r *Record) GetValue(name string, pos uint) (any, error) { switch name { case "createdAt", "CreatedAt": return r.CreatedAt, nil - case "createdBy", "CreatedBy": + case "createdBy", "CreatedBy", "created_by": return r.CreatedBy, nil case "deletedAt", "DeletedAt": return r.DeletedAt, nil - case "deletedBy", "DeletedBy": + case "deletedBy", "DeletedBy", "deleted_by": return r.DeletedBy, nil case "id", "ID": return r.ID, nil @@ -351,13 +351,13 @@ func (r *Record) GetValue(name string, pos uint) (any, error) { return r.ModuleID, nil case "namespaceID", "NamespaceID": return r.NamespaceID, nil - case "ownedBy", "OwnedBy": + case "ownedBy", "OwnedBy", "owned_by": return r.OwnedBy, nil case "revision", "Revision": return r.Revision, nil case "updatedAt", "UpdatedAt": return r.UpdatedAt, nil - case "updatedBy", "UpdatedBy": + case "updatedBy", "UpdatedBy", "updated_by": return r.UpdatedBy, nil default: @@ -371,11 +371,11 @@ func (r *Record) SetValue(name string, pos uint, value any) (err error) { switch name { case "createdAt", "CreatedAt": return cast2.Time(value, &r.CreatedAt) - case "createdBy", "CreatedBy": + case "createdBy", "CreatedBy", "created_by": return cast2.Uint64(value, &r.CreatedBy) case "deletedAt", "DeletedAt": return cast2.TimePtr(value, &r.DeletedAt) - case "deletedBy", "DeletedBy": + case "deletedBy", "DeletedBy", "deleted_by": return cast2.Uint64(value, &r.DeletedBy) case "id", "ID": return cast2.Uint64(value, &r.ID) @@ -385,13 +385,13 @@ func (r *Record) SetValue(name string, pos uint, value any) (err error) { return cast2.Uint64(value, &r.ModuleID) case "namespaceID", "NamespaceID": return cast2.Uint64(value, &r.NamespaceID) - case "ownedBy", "OwnedBy": + case "ownedBy", "OwnedBy", "owned_by": return cast2.Uint64(value, &r.OwnedBy) case "revision", "Revision": return cast2.Uint(value, &r.Revision) case "updatedAt", "UpdatedAt": return cast2.TimePtr(value, &r.UpdatedAt) - case "updatedBy", "UpdatedBy": + case "updatedBy", "UpdatedBy", "updated_by": return cast2.Uint64(value, &r.UpdatedBy) default: diff --git a/server/compose/types/record.go b/server/compose/types/record.go index e60a6ff39..6d9569bc7 100644 --- a/server/compose/types/record.go +++ b/server/compose/types/record.go @@ -219,21 +219,26 @@ func (r *Record) setValue(name string, pos uint, value any) (err error) { } rv := &RecordValue{Name: name, Place: pos} - var auxv string - switch aux := value.(type) { - case *time.Time: - auxv = aux.Format(time.RFC3339) + if cv, ok := value.(*RecordValue); ok { + rv = cv + } else { + var auxv string + switch aux := value.(type) { + case *time.Time: + auxv = aux.Format(time.RFC3339) - case time.Time: - auxv = aux.Format(time.RFC3339) + case time.Time: + auxv = aux.Format(time.RFC3339) - default: - auxv, err = cast.ToStringE(aux) - } + default: + auxv, err = cast.ToStringE(aux) + } + if err != nil { + return + } - if err != nil { - return + rv.Value = auxv } // Try to utilize the module when possible @@ -248,7 +253,6 @@ func (r *Record) setValue(name string, pos uint, value any) (err error) { } } - rv.Value = auxv r.Values = r.Values.Set(rv) return diff --git a/server/pkg/envoyx/dep.go b/server/pkg/envoyx/dep.go index 1c756a00a..8434cdcfc 100644 --- a/server/pkg/envoyx/dep.go +++ b/server/pkg/envoyx/dep.go @@ -312,6 +312,17 @@ func (g DepGraph) ChildrenForResourceType(n *Node, rt string) (out NodeSet) { return } +func (g DepGraph) NodeForRef(ref Ref) (out *Node) { + aux := make([]*Node, 0, 10) + for _, sg := range g.graphs { + for _, n := range sg.nodes { + aux = append(aux, n.Node) + } + } + + return NodeForRef(ref, aux...) +} + // Children returns all child nodes of n func (g DepGraph) Children(n *Node) (out NodeSet) { for _, sg := range g.graphs { diff --git a/server/pkg/envoyx/envoy.go b/server/pkg/envoyx/envoy.go index 7e60f56e1..f17b290f1 100644 --- a/server/pkg/envoyx/envoy.go +++ b/server/pkg/envoyx/envoy.go @@ -30,6 +30,9 @@ type ( // Children returns all of the children of the provided node Children(*Node) NodeSet + + // NodeForRef returns the node which matches the provided ref + NodeForRef(Ref) *Node } Preparer interface { @@ -77,6 +80,7 @@ type ( Encoder EncoderConfig } EncoderConfig struct { + DefaultUserID uint64 PreferredTimeLayout string PreferredTimezone string } diff --git a/server/system/envoy/user_getter.go b/server/system/envoy/user_getter.go new file mode 100644 index 000000000..fe2d8deca --- /dev/null +++ b/server/system/envoy/user_getter.go @@ -0,0 +1,88 @@ +package envoy + +import ( + "context" + "fmt" + + "github.com/cortezaproject/corteza/server/pkg/envoyx" + "github.com/cortezaproject/corteza/server/store" + "github.com/cortezaproject/corteza/server/system/types" + "github.com/spf13/cast" +) + +type ( + // UserGetter is a utility struct to resolve user references from + // different parts of the system such as the dep graph and the database + UserGetter struct { + depGraph *envoyx.DepGraph + + store store.Storer + baseFilter types.UserFilter + } +) + +func MakeUserGetter(s store.Storer, tt envoyx.Traverser) (g *UserGetter) { + g = &UserGetter{ + store: s, + } + + g.baseFilter = types.UserFilter{} + + return +} + +// Resolve returns the user ID for the provided reference +// +// If the user can not be resolved, 0 is returned. +func (g *UserGetter) Resolve(ctx context.Context, ref any) (out uint64, err error) { + // Try to get from datasource + if g.depGraph != nil { + out, err = g.getDS(ref) + if err != nil { + return + } + } + if out > 0 { + return + } + + // Try to get from the database + out, err = g.getDB(ctx, ref) + return +} + +func (g *UserGetter) getDS(ref any) (out uint64, err error) { + n := g.depGraph.NodeForRef(envoyx.Ref{ + ResourceType: types.UserResourceType, + Identifiers: envoyx.MakeIdentifiers(ref), + }) + + if n == nil { + return + } + + return n.Resource.GetID(), nil +} + +// @todo this can be improved by prefetching and indexing refs +func (g *UserGetter) getDB(ctx context.Context, ref any) (out uint64, err error) { + f := g.baseFilter + // @todo expand this + f.Query = cast.ToString(ref) + + set, _, err := store.SearchUsers(ctx, g.store, f) + if err != nil { + return + } + + if len(set) == 0 { + return + } + + if len(set) > 1 { + err = fmt.Errorf("ambiguous reference %v: matches more then one user", ref) + return + } + + return set[0].ID, nil +} diff --git a/server/system/envoy/yaml_decode.gen.go b/server/system/envoy/yaml_decode.gen.go index 1608bdb9e..78b3c9fa8 100644 --- a/server/system/envoy/yaml_decode.gen.go +++ b/server/system/envoy/yaml_decode.gen.go @@ -28,7 +28,8 @@ type ( // which are then managed by envoy and imported via an encoder. YamlDecoder struct{} documentContext struct { - references map[string]string + references map[string]string + parentIdent envoyx.Identifiers } auxYamlDoc struct { nodes envoyx.NodeSet @@ -380,6 +381,9 @@ func (d *auxYamlDoc) unmarshalApplicationNode(dctx documentContext, n *yaml.Node return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Apply the scope to all of the references of the same type for k, ref := range refs { if ref.ResourceType != scope.ResourceType { @@ -690,6 +694,9 @@ func (d *auxYamlDoc) unmarshalApigwRouteNode(dctx documentContext, n *yaml.Node, return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Apply the scope to all of the references of the same type for k, ref := range refs { if ref.ResourceType != scope.ResourceType { @@ -956,6 +963,9 @@ func (d *auxYamlDoc) unmarshalApigwFilterNode(dctx documentContext, n *yaml.Node return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Apply the scope to all of the references of the same type for k, ref := range refs { if ref.ResourceType != scope.ResourceType { @@ -1261,6 +1271,9 @@ func (d *auxYamlDoc) unmarshalAuthClientNode(dctx documentContext, n *yaml.Node, return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Apply the scope to all of the references of the same type for k, ref := range refs { if ref.ResourceType != scope.ResourceType { @@ -1546,6 +1559,9 @@ func (d *auxYamlDoc) unmarshalQueueNode(dctx documentContext, n *yaml.Node, meta return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Apply the scope to all of the references of the same type for k, ref := range refs { if ref.ResourceType != scope.ResourceType { @@ -1850,6 +1866,9 @@ func (d *auxYamlDoc) unmarshalReportNode(dctx documentContext, n *yaml.Node, met return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Apply the scope to all of the references of the same type for k, ref := range refs { if ref.ResourceType != scope.ResourceType { @@ -2078,6 +2097,9 @@ func (d *auxYamlDoc) unmarshalRoleNode(dctx documentContext, n *yaml.Node, meta return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Apply the scope to all of the references of the same type for k, ref := range refs { if ref.ResourceType != scope.ResourceType { @@ -2325,6 +2347,9 @@ func (d *auxYamlDoc) unmarshalTemplateNode(dctx documentContext, n *yaml.Node, m return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Apply the scope to all of the references of the same type for k, ref := range refs { if ref.ResourceType != scope.ResourceType { @@ -2572,6 +2597,9 @@ func (d *auxYamlDoc) unmarshalUserNode(dctx documentContext, n *yaml.Node, meta return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Apply the scope to all of the references of the same type for k, ref := range refs { if ref.ResourceType != scope.ResourceType { @@ -2857,6 +2885,9 @@ func (d *auxYamlDoc) unmarshalDalConnectionNode(dctx documentContext, n *yaml.No return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Apply the scope to all of the references of the same type for k, ref := range refs { if ref.ResourceType != scope.ResourceType { @@ -3125,6 +3156,9 @@ func (d *auxYamlDoc) unmarshalDalSensitivityLevelNode(dctx documentContext, n *y return } + // Make parent identifiers available through the dctx + dctx.parentIdent = ii + // Apply the scope to all of the references of the same type for k, ref := range refs { if ref.ResourceType != scope.ResourceType { diff --git a/server/tests/envoy/records_test.go b/server/tests/envoy/records_test.go index 4e73fe363..faff16f9c 100644 --- a/server/tests/envoy/records_test.go +++ b/server/tests/envoy/records_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "strconv" "testing" "github.com/cortezaproject/corteza/server/compose/dalutils" @@ -12,6 +13,7 @@ import ( "github.com/cortezaproject/corteza/server/pkg/dal" "github.com/cortezaproject/corteza/server/pkg/envoyx" "github.com/cortezaproject/corteza/server/store" + systemTypes "github.com/cortezaproject/corteza/server/system/types" "github.com/davecgh/go-spew/spew" "github.com/stretchr/testify/require" ) @@ -196,6 +198,146 @@ func TestRecordsImportExport(t *testing.T) { }) } +func TestRecordsXrefs(t *testing.T) { + var ( + ctx = context.Background() + req = require.New(t) + nodes envoyx.NodeSet + providers []envoyx.Provider + gg *envoyx.DepGraph + err error + ) + _ = gg + + cleanup(t) + + t.Run("parse configs", func(t *testing.T) { + nodes, providers, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ + Type: envoyx.DecodeTypeURI, + Params: map[string]any{ + "uri": "file://testdata/records/cross_ref", + }, + }) + req.NoError(err) + }) + + t.Run("bake", func(t *testing.T) { + gg, err = defaultEnvoy.Bake(ctx, envoyx.EncodeParams{ + Type: envoyx.EncodeTypeStore, + Params: map[string]any{ + "storer": defaultStore, + "dal": defaultDal, + }, + }, providers, nodes...) + req.NoError(err) + }) + + t.Run("import into DB", func(t *testing.T) { + err = defaultEnvoy.Encode(ctx, envoyx.EncodeParams{ + Type: envoyx.EncodeTypeStore, + Params: map[string]any{ + "storer": defaultStore, + "dal": defaultDal, + }, + }, gg) + req.NoError(err) + }) + + ns, err := store.LookupComposeNamespaceBySlug(ctx, defaultStore, "test_ns") + req.NoError(err) + + modParent, err := store.LookupComposeModuleByNamespaceIDHandle(ctx, defaultStore, ns.ID, "mod_parent") + req.NoError(err) + + modChild, err := store.LookupComposeModuleByNamespaceIDHandle(ctx, defaultStore, ns.ID, "mod_child") + req.NoError(err) + + parentRecords, _, err := dalutils.ComposeRecordsList(ctx, defaultDal, modParent, types.RecordFilter{}) + req.NoError(err) + + compareValues(req, parentRecords[0].Values, types.RecordValueSet{{Name: "name", Value: "Jane Doe"}, {Name: "code", Value: "3124"}}) + compareValues(req, parentRecords[1].Values, types.RecordValueSet{{Name: "name", Value: "Ana Banana"}, {Name: "code", Value: "4153"}}) + compareValues(req, parentRecords[2].Values, types.RecordValueSet{{Name: "name", Value: "Qwerty"}, {Name: "code", Value: "83172"}}) + + childRecords, _, err := dalutils.ComposeRecordsList(ctx, defaultDal, modChild, types.RecordFilter{}) + req.NoError(err) + + compareValues(req, childRecords[0].Values, types.RecordValueSet{{Name: "user_name", Value: "Daniel Adams", Ref: 0}, {Name: "parent", Value: strconv.FormatUint(parentRecords[0].ID, 10), Ref: parentRecords[0].ID}, {Name: "alt", Value: strconv.FormatUint(childRecords[2].ID, 10), Ref: childRecords[2].ID}}) + compareValues(req, childRecords[1].Values, types.RecordValueSet{{Name: "user_name", Value: "Jane Alex", Ref: 0}, {Name: "parent", Value: strconv.FormatUint(parentRecords[1].ID, 10), Ref: parentRecords[1].ID}, {Name: "alt", Value: strconv.FormatUint(childRecords[0].ID, 10), Ref: childRecords[0].ID}}) + compareValues(req, childRecords[2].Values, types.RecordValueSet{{Name: "user_name", Value: "Fish", Ref: 0}, {Name: "parent", Value: strconv.FormatUint(parentRecords[2].ID, 10), Ref: parentRecords[2].ID}, {Name: "alt", Value: strconv.FormatUint(childRecords[1].ID, 10), Ref: childRecords[1].ID}}) +} + +func TestRecordsUserRefs(t *testing.T) { + var ( + ctx = context.Background() + req = require.New(t) + nodes envoyx.NodeSet + providers []envoyx.Provider + gg *envoyx.DepGraph + err error + ) + _ = gg + + cleanup(t) + + t.Run("parse configs", func(t *testing.T) { + nodes, providers, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ + Type: envoyx.DecodeTypeURI, + Params: map[string]any{ + "uri": "file://testdata/records/user_ref", + }, + }) + req.NoError(err) + }) + + t.Run("bake", func(t *testing.T) { + gg, err = defaultEnvoy.Bake(ctx, envoyx.EncodeParams{ + Type: envoyx.EncodeTypeStore, + Params: map[string]any{ + "storer": defaultStore, + "dal": defaultDal, + }, + }, providers, nodes...) + req.NoError(err) + }) + + t.Run("import into DB", func(t *testing.T) { + err = defaultEnvoy.Encode(ctx, envoyx.EncodeParams{ + Type: envoyx.EncodeTypeStore, + Params: map[string]any{ + "storer": defaultStore, + "dal": defaultDal, + }, + }, gg) + req.NoError(err) + }) + + ns, err := store.LookupComposeNamespaceBySlug(ctx, defaultStore, "test_ns") + req.NoError(err) + + mod, err := store.LookupComposeModuleByNamespaceIDHandle(ctx, defaultStore, ns.ID, "test_mod") + req.NoError(err) + + rr, _, err := dalutils.ComposeRecordsList(ctx, defaultDal, mod, types.RecordFilter{}) + req.NoError(err) + + uu, _, err := store.SearchUsers(ctx, defaultStore, systemTypes.UserFilter{}) + req.NoError(err) + + usr0 := uu[0] + usr1 := uu[1] + usr2 := uu[2] + usrx := uu[3] + + compareValues(req, rr[0].Values, types.RecordValueSet{{Name: "name", Value: "Jane Doe"}, {Name: "code", Value: "3124"}, {Name: "rel_user", Value: strconv.FormatUint(usr0.ID, 10), Ref: usr0.ID}}) + compareValues(req, rr[1].Values, types.RecordValueSet{{Name: "name", Value: "Ana Banana"}, {Name: "code", Value: "4153"}, {Name: "rel_user", Value: strconv.FormatUint(usr1.ID, 10), Ref: usr1.ID}}) + compareValues(req, rr[2].Values, types.RecordValueSet{{Name: "name", Value: "Qwerty"}, {Name: "code", Value: "83172"}, {Name: "rel_user", Value: strconv.FormatUint(usr2.ID, 10), Ref: usr2.ID}}) + + req.Equal(usrx.ID, rr[0].CreatedBy) + req.Equal(usrx.ID, rr[1].CreatedBy) + req.Equal(usrx.ID, rr[2].CreatedBy) +} + func assertRecordState(ctx context.Context, t *testing.T, s store.Storer, dl dal.FullService, req *require.Assertions) { t.Run("check state", func(t *testing.T) { ns, err := store.LookupComposeNamespaceBySlug(ctx, defaultStore, "test_ns_1") diff --git a/server/tests/envoy/testdata/records/cross_ref/child_records.csv b/server/tests/envoy/testdata/records/cross_ref/child_records.csv new file mode 100644 index 000000000..9de6ce2ac --- /dev/null +++ b/server/tests/envoy/testdata/records/cross_ref/child_records.csv @@ -0,0 +1,4 @@ +ID,user_name,parent,alt +c11,Daniel Adams,p1,c13 +c12,Jane Alex,p2,c11 +c13,Fish,p3,c12 diff --git a/server/tests/envoy/testdata/records/cross_ref/def.yaml b/server/tests/envoy/testdata/records/cross_ref/def.yaml new file mode 100644 index 000000000..d52cbf536 --- /dev/null +++ b/server/tests/envoy/testdata/records/cross_ref/def.yaml @@ -0,0 +1,49 @@ +namespace: + test_ns: + name: Test Namespace + modules: + mod_parent: + name: Test Namespace Module Parent + fields: + name: + title: Name + code: + title: Code + kind: Number + source: + - from: parent_records.csv + key: ID + map: + - column: ID + skip: true + - column: name + field: name + - column: code + field: code + mod_child: + name: Test Namespace Module Child + fields: + user_name: + title: User Name + parent: + title: Parent + kind: Record + options: + module: mod_parent + alt: + title: alt + kind: Record + options: + module: mod_child + source: + - from: child_records.csv + key: ID + map: + - column: ID + skip: true + - column: user_name + field: user_name + - column: parent + field: parent + - column: alt + field: alt diff --git a/server/tests/envoy/testdata/records/cross_ref/parent_records.csv b/server/tests/envoy/testdata/records/cross_ref/parent_records.csv new file mode 100644 index 000000000..07b16f188 --- /dev/null +++ b/server/tests/envoy/testdata/records/cross_ref/parent_records.csv @@ -0,0 +1,4 @@ +ID,name,code +p1,Jane Doe,3124 +p2,Ana Banana,4153 +p3,Qwerty,83172 diff --git a/server/tests/envoy/testdata/records/user_ref/def.yaml b/server/tests/envoy/testdata/records/user_ref/def.yaml new file mode 100644 index 000000000..7cd80148e --- /dev/null +++ b/server/tests/envoy/testdata/records/user_ref/def.yaml @@ -0,0 +1,39 @@ +namespace: + test_ns: + name: Test Namespace + modules: + test_mod: + name: Test Namespace Module + fields: + name: + title: Name + code: + title: Code + kind: Number + rel_user: + title: Rel User + kind: User + source: + - from: records.csv + key: ID + map: + - column: ID + skip: true + - column: name + field: name + - column: code + field: code + - column: rel_user + field: rel_user + - column: created_by + field: created_by + +user: + usr_0: + email: usr_0@test.tld + usr_1: + email: usr_1@test.tld + usr_2: + email: usr_2@test.tld + usr_x: + email: usr_x@test.tld diff --git a/server/tests/envoy/testdata/records/user_ref/records.csv b/server/tests/envoy/testdata/records/user_ref/records.csv new file mode 100644 index 000000000..02f09a0fe --- /dev/null +++ b/server/tests/envoy/testdata/records/user_ref/records.csv @@ -0,0 +1,4 @@ +ID,name,code,rel_user,created_by +p1,Jane Doe,3124,usr_0,usr_x +p2,Ana Banana,4153,usr_1,usr_x +p3,Qwerty,83172,usr_2,usr_x