From 1699e3f94b2c344eafb1fae045ced994accfa53c Mon Sep 17 00:00:00 2001 From: Vivek Patel Date: Tue, 15 Nov 2022 19:13:19 +0530 Subject: [PATCH] Remove handle requirement from resource API We removed handle requirement for all resource like namespace(slug), module, user, page, chart. Due to that namespace import/export was broken as we were using handle as reference, hence fixed it by replacing handle usage with ID of resource. It also fixes index of template table for handle column. --- server/compose/rest.yaml | 2 - server/compose/service/namespace.go | 45 ++++++++------ .../pkg/envoy/yaml/compose_chart_marshal.go | 1 + .../pkg/envoy/yaml/compose_chart_unmarshal.go | 3 + .../pkg/envoy/yaml/compose_module_marshal.go | 1 + .../envoy/yaml/compose_module_unmarshal.go | 3 + .../envoy/yaml/compose_namespace_marshal.go | 1 + .../envoy/yaml/compose_namespace_unmarshal.go | 3 + .../pkg/envoy/yaml/compose_page_unmarshal.go | 2 +- server/pkg/envoy/yaml/decoder.go | 14 ++++- .../yaml/resource_translation_marshal.go | 59 +++++++++++++++---- server/system/model/models.gen.go | 8 +-- server/system/rest.yaml | 4 +- server/system/template.cue | 2 +- 14 files changed, 106 insertions(+), 42 deletions(-) diff --git a/server/compose/rest.yaml b/server/compose/rest.yaml index 80be98ae3..453a820fc 100644 --- a/server/compose/rest.yaml +++ b/server/compose/rest.yaml @@ -154,7 +154,6 @@ endpoints: title: Duplicate name - type: string name: slug - required: true title: Duplicate slug - name: export path: "/{namespaceID}/export/{filename}.zip" @@ -201,7 +200,6 @@ endpoints: title: Imported namespace name - type: string name: slug - required: true title: Imported namespace slug - name: triggerScript method: POST diff --git a/server/compose/service/namespace.go b/server/compose/service/namespace.go index f6cd34863..102c40418 100644 --- a/server/compose/service/namespace.go +++ b/server/compose/service/namespace.go @@ -39,10 +39,11 @@ type ( } namespaceImportSession struct { - Name string `json:"name"` - Slug string `json:"handle"` - SessionID uint64 `json:"sessionID,string"` - UserID uint64 `json:"userID,string"` + Name string `json:"name"` + Slug string `json:"handle"` + NamespaceID uint64 `json:"namespaceID,string"` + SessionID uint64 `json:"sessionID,string"` + UserID uint64 `json:"userID,string"` CreatedAt time.Time `json:"createdAt"` UpdatedAt time.Time `json:"updatedAt"` @@ -285,12 +286,14 @@ func (svc namespace) Clone(ctx context.Context, namespaceID uint64, dup *types.N aProps.setNamespace(targetNs) // - destination namespace - dstNs, err := store.LookupComposeNamespaceBySlug(ctx, svc.store, dup.Slug) - if err != nil && err != store.ErrNotFound { - return err - } - if dstNs != nil { - return NamespaceErrHandleNotUnique() + if dup.Slug != "" { + dstNs, err := store.LookupComposeNamespaceBySlug(ctx, svc.store, dup.Slug) + if err != nil && err != store.ErrNotFound { + return err + } + if dstNs != nil { + return NamespaceErrHandleNotUnique() + } } // Access control @@ -403,6 +406,8 @@ func (svc namespace) ImportInit(ctx context.Context, f multipart.File, size int6 return NamespaceErrImportMissingNamespace() } + // session needs to have namespaceID if ns Handle is not provided + session.NamespaceID = ns.ID session.Name = ns.Name session.Slug = ns.Slug namespaceSessionStore[session.SessionID] = session @@ -425,17 +430,19 @@ func (svc namespace) ImportRun(ctx context.Context, sessionID uint64, dup *types return err } - if dup.Slug == "" || !handle.IsValid(dup.Slug) { + if !handle.IsValid(dup.Slug) { return NamespaceErrInvalidHandle() } - // check for duplicate - dstNs, err := store.LookupComposeNamespaceBySlug(ctx, svc.store, dup.Slug) - if err != nil && err != store.ErrNotFound { - return err - } - if dstNs != nil { - return NamespaceErrHandleNotUnique() + if dup.Slug != "" { + // check for duplicate + dstNs, err := store.LookupComposeNamespaceBySlug(ctx, svc.store, dup.Slug) + if err != nil && err != store.ErrNotFound { + return err + } + if dstNs != nil { + return NamespaceErrHandleNotUnique() + } } // session @@ -453,7 +460,7 @@ func (svc namespace) ImportRun(ctx context.Context, sessionID uint64, dup *types aProps.setNamespace(dup) - newNS, err = svc.envoyRun(ctx, session.Resources, &types.Namespace{Slug: session.Slug, Name: session.Name}, dup, encoder) + newNS, err = svc.envoyRun(ctx, session.Resources, &types.Namespace{ID: session.NamespaceID, Slug: session.Slug, Name: session.Name}, dup, encoder) if err != nil { return err } diff --git a/server/pkg/envoy/yaml/compose_chart_marshal.go b/server/pkg/envoy/yaml/compose_chart_marshal.go index 0e90a7c97..6d444f4e4 100644 --- a/server/pkg/envoy/yaml/compose_chart_marshal.go +++ b/server/pkg/envoy/yaml/compose_chart_marshal.go @@ -85,6 +85,7 @@ func (n *composeChart) Encode(ctx context.Context, doc *Document, state *envoy.R func (c *composeChart) MarshalYAML() (interface{}, error) { nn, err := makeMap( + "chartID", c.res.ID, "handle", c.res.Handle, "name", c.res.Name, "config", c.chartConfig, diff --git a/server/pkg/envoy/yaml/compose_chart_unmarshal.go b/server/pkg/envoy/yaml/compose_chart_unmarshal.go index 88de13f4f..0d96b2c93 100644 --- a/server/pkg/envoy/yaml/compose_chart_unmarshal.go +++ b/server/pkg/envoy/yaml/compose_chart_unmarshal.go @@ -58,6 +58,9 @@ func (wrap *composeChart) UnmarshalYAML(n *yaml.Node) (err error) { return y7s.EachMap(n, func(k, v *yaml.Node) (err error) { switch k.Value { + case "chartID": + return y7s.DecodeScalar(v, "chart ID", &wrap.res.ID) + case "handle": return y7s.DecodeScalar(v, "chart handle", &wrap.res.Handle) diff --git a/server/pkg/envoy/yaml/compose_module_marshal.go b/server/pkg/envoy/yaml/compose_module_marshal.go index f7b21ac5e..7d600c39d 100644 --- a/server/pkg/envoy/yaml/compose_module_marshal.go +++ b/server/pkg/envoy/yaml/compose_module_marshal.go @@ -123,6 +123,7 @@ func (c *composeModule) MarshalYAML() (interface{}, error) { } nn, err := makeMap( + "moduleID", c.res.ID, "handle", c.res.Handle, "name", c.res.Name, "meta", auxMeta, diff --git a/server/pkg/envoy/yaml/compose_module_unmarshal.go b/server/pkg/envoy/yaml/compose_module_unmarshal.go index 8b9ec29e7..22bf66139 100644 --- a/server/pkg/envoy/yaml/compose_module_unmarshal.go +++ b/server/pkg/envoy/yaml/compose_module_unmarshal.go @@ -63,6 +63,9 @@ func (wrap *composeModule) UnmarshalYAML(n *yaml.Node) (err error) { return y7s.EachMap(n, func(k, v *yaml.Node) (err error) { switch k.Value { + case "moduleID": + return y7s.DecodeScalar(v, "module ID", &wrap.res.ID) + case "name": return y7s.DecodeScalar(v, "module name", &wrap.res.Name) diff --git a/server/pkg/envoy/yaml/compose_namespace_marshal.go b/server/pkg/envoy/yaml/compose_namespace_marshal.go index 9bfd8812f..e9b68dbb2 100644 --- a/server/pkg/envoy/yaml/compose_namespace_marshal.go +++ b/server/pkg/envoy/yaml/compose_namespace_marshal.go @@ -40,6 +40,7 @@ func (c *composeNamespace) MarshalYAML() (interface{}, error) { var err error nn, err := makeMap( + "namespaceID", c.res.ID, "name", c.res.Name, "slug", c.res.Slug, "enabled", c.res.Enabled, diff --git a/server/pkg/envoy/yaml/compose_namespace_unmarshal.go b/server/pkg/envoy/yaml/compose_namespace_unmarshal.go index d9f663489..5ac03b9ce 100644 --- a/server/pkg/envoy/yaml/compose_namespace_unmarshal.go +++ b/server/pkg/envoy/yaml/compose_namespace_unmarshal.go @@ -75,6 +75,9 @@ func (wrap *composeNamespace) UnmarshalYAML(n *yaml.Node) (err error) { return y7s.Each(n, func(k, v *yaml.Node) (err error) { switch k.Value { + case "namespaceID": + return y7s.DecodeScalar(v, "namespace ID", &wrap.res.ID) + case "modules": return v.Decode(&wrap.modules) diff --git a/server/pkg/envoy/yaml/compose_page_unmarshal.go b/server/pkg/envoy/yaml/compose_page_unmarshal.go index 550057c02..4a852de38 100644 --- a/server/pkg/envoy/yaml/compose_page_unmarshal.go +++ b/server/pkg/envoy/yaml/compose_page_unmarshal.go @@ -73,7 +73,7 @@ func (wrap *composePage) UnmarshalYAML(n *yaml.Node) (err error) { return y7s.EachMap(n, func(k, v *yaml.Node) (err error) { switch strings.ToLower(k.Value) { - case "id", "pageid": + case "id", "pageid", "pageID": return y7s.DecodeScalar(v, "page ID", &wrap.res.ID) case "title": diff --git a/server/pkg/envoy/yaml/decoder.go b/server/pkg/envoy/yaml/decoder.go index 0a60e8203..a193c373d 100644 --- a/server/pkg/envoy/yaml/decoder.go +++ b/server/pkg/envoy/yaml/decoder.go @@ -3,9 +3,9 @@ package yaml import ( "context" "io" + "regexp" "strings" - "github.com/cortezaproject/corteza/server/pkg/handle" "github.com/cortezaproject/corteza/server/pkg/y7s" . "github.com/cortezaproject/corteza/server/pkg/y7s" @@ -23,6 +23,10 @@ type ( } ) +var ( + validReference = regexp.MustCompile(`^[A-Za-z1-9][0-9A-Za-z_\-.]*[A-Za-z0-9]$`) +) + func Decoder() *decoder { return &decoder{} } @@ -68,10 +72,14 @@ func decodeRef(n *yaml.Node, refType string, ref *string) error { return y7s.NodeErr(n, "%s reference must be scalar", refType) } - if !handle.IsValid(n.Value) { - return y7s.NodeErr(n, "%s reference must be a valid handle", refType) + if !IsValidRef(n.Value) { + return y7s.NodeErr(n, "%s reference must be a valid ID or Handle", refType) } *ref = n.Value return nil } + +func IsValidRef(s string) bool { + return len(s) >= 2 && validReference.MatchString(s) +} diff --git a/server/pkg/envoy/yaml/resource_translation_marshal.go b/server/pkg/envoy/yaml/resource_translation_marshal.go index f85439058..7474789fa 100644 --- a/server/pkg/envoy/yaml/resource_translation_marshal.go +++ b/server/pkg/envoy/yaml/resource_translation_marshal.go @@ -4,12 +4,12 @@ import ( "context" "fmt" "sort" - "strconv" "strings" composeTypes "github.com/cortezaproject/corteza/server/compose/types" "github.com/cortezaproject/corteza/server/pkg/envoy" "github.com/cortezaproject/corteza/server/pkg/envoy/resource" + "github.com/spf13/cast" "golang.org/x/text/language" ) @@ -92,7 +92,11 @@ func (r *resourceTranslation) makeResourceTranslationResource(state *envoy.Resou if p1 == nil { return "", resource.ComposeNamespaceErrUnresolved(res.RefRes.Identifiers) } - p1ID = p1.Slug + if p1.Slug == "" { + p1ID = cast.ToString(p1.ID) + } else { + p1ID = p1.Slug + } } return fmt.Sprintf(composeTypes.NamespaceResourceTranslationTpl(), composeTypes.NamespaceResourceTranslationType, p1ID), nil @@ -103,7 +107,12 @@ func (r *resourceTranslation) makeResourceTranslationResource(state *envoy.Resou if p0 == nil { return "", resource.ComposeNamespaceErrUnresolved(res.RefPath[0].Identifiers) } - p0ID = p0.Slug + + if p0.Slug == "" { + p0ID = cast.ToString(p0.ID) + } else { + p0ID = p0.Slug + } } if res.RefRes != nil { @@ -111,7 +120,12 @@ func (r *resourceTranslation) makeResourceTranslationResource(state *envoy.Resou if p1 == nil { return "", resource.ComposeModuleErrUnresolved(res.RefRes.Identifiers) } - p1ID = p1.Handle + + if p1.Handle == "" { + p1ID = cast.ToString(p1.ID) + } else { + p1ID = p1.Handle + } } return fmt.Sprintf(composeTypes.ModuleResourceTranslationTpl(), composeTypes.ModuleResourceTranslationType, p0ID, p1ID), nil @@ -122,7 +136,12 @@ func (r *resourceTranslation) makeResourceTranslationResource(state *envoy.Resou if p0 == nil { return "", resource.ComposeNamespaceErrUnresolved(res.RefPath[0].Identifiers) } - p0ID = p0.Slug + + if p0.Slug == "" { + p0ID = cast.ToString(p0.ID) + } else { + p0ID = p0.Slug + } } if res.RefRes != nil { @@ -132,7 +151,7 @@ func (r *resourceTranslation) makeResourceTranslationResource(state *envoy.Resou } if p1.Handle == "" { - p1ID = strconv.FormatUint(p1.ID, 10) + p1ID = cast.ToString(p1.ID) } else { p1ID = p1.Handle } @@ -145,13 +164,23 @@ func (r *resourceTranslation) makeResourceTranslationResource(state *envoy.Resou if p0 == nil { return "", resource.ComposeNamespaceErrUnresolved(res.RefPath[0].Identifiers) } - p0ID = p0.Slug + + if p0.Slug == "" { + p0ID = cast.ToString(p0.ID) + } else { + p0ID = p0.Slug + } p1 := resource.FindComposeModule(state.ParentResources, res.RefPath[1].Identifiers) if p1 == nil { return "", resource.ComposeModuleErrUnresolved(res.RefPath[1].Identifiers) } - p1ID = p1.Handle + + if p1.Handle == "" { + p1ID = cast.ToString(p1.ID) + } else { + p1ID = p1.Handle + } // field f := resource.FindComposeModuleField(state.ParentResources, res.RefPath[1].Identifiers, res.RefRes.Identifiers) @@ -168,7 +197,12 @@ func (r *resourceTranslation) makeResourceTranslationResource(state *envoy.Resou if p0 == nil { return "", resource.ComposeNamespaceErrUnresolved(res.RefPath[0].Identifiers) } - p0ID = p0.Slug + + if p0.Slug == "" { + p0ID = cast.ToString(p0.ID) + } else { + p0ID = p0.Slug + } } if res.RefRes != nil { @@ -176,7 +210,12 @@ func (r *resourceTranslation) makeResourceTranslationResource(state *envoy.Resou if p1 == nil { return "", resource.ComposeChartErrUnresolved(res.RefRes.Identifiers) } - p1ID = p1.Handle + + if p1.Handle == "" { + p1ID = cast.ToString(p1.ID) + } else { + p1ID = p1.Handle + } } return fmt.Sprintf(composeTypes.ChartResourceTranslationTpl(), composeTypes.ChartResourceTranslationType, p0ID, p1ID), nil diff --git a/server/system/model/models.gen.go b/server/system/model/models.gen.go index 243cd39e0..be7f2036a 100644 --- a/server/system/model/models.gen.go +++ b/server/system/model/models.gen.go @@ -2328,10 +2328,10 @@ var Template = &dal.Model{ }, &dal.Index{ - Ident: "templates_uniqueLanguageHandle", - Type: "BTREE", - Unique: true, - + Ident: "templates_uniqueLanguageHandle", + Type: "BTREE", + Unique: true, + Predicate: "handle != '' AND deleted_at IS NULL", Fields: []*dal.IndexField{ { AttributeIdent: "Language", diff --git a/server/system/rest.yaml b/server/system/rest.yaml index 51d0cf6b7..59ea43d73 100644 --- a/server/system/rest.yaml +++ b/server/system/rest.yaml @@ -871,7 +871,7 @@ endpoints: path: "/" parameters: post: - - { name: handle, type: "string", required: true } + - { name: handle, type: "string", } - { name: type, type: "string", required: true } - { name: meta, type: "types.ConnectionMeta", required: true, parser: "types.ParseConnectionMeta" } - { name: config, type: "types.ConnectionConfig", required: true, parser: "types.ParseConnectionConfig" } @@ -883,7 +883,7 @@ endpoints: parameters: path: [ { type: uint64, name: connectionID, required: true, title: "Connection ID" } ] post: - - { name: handle, type: "string", required: true } + - { name: handle, type: "string", } - { name: type, type: "string", required: true } - { name: meta, type: "types.ConnectionMeta", required: true, parser: "types.ParseConnectionMeta" } - { name: config, type: "types.ConnectionConfig", required: true, parser: "types.ParseConnectionConfig" } diff --git a/server/system/template.cue b/server/system/template.cue index 2ae309eff..5c926aec2 100644 --- a/server/system/template.cue +++ b/server/system/template.cue @@ -49,11 +49,11 @@ template: { indexes: { "primary": { attribute: "id" } "unique_language_handle": { - unique: true fields: [ { attribute: "language" }, { attribute: "handle", modifier: [ "LOWERCASE" ] } ] + predicate: "handle != '' AND deleted_at IS NULL" } } }