From 7f795ef418515721f10476935978f2a53837da2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Sun, 10 Oct 2021 12:43:21 +0200 Subject: [PATCH] Split namespace import to init and run --- compose/rest.yaml | 25 ++++- compose/rest/handlers/namespace.go | 31 ++++-- compose/rest/namespace.go | 39 +++++--- compose/rest/request/namespace.go | 113 +++++++++++++++++++-- compose/service/namespace.go | 120 +++++++++++++++++++++-- compose/service/namespace_actions.gen.go | 62 +++++++++++- compose/service/namespace_actions.yaml | 9 +- 7 files changed, 355 insertions(+), 44 deletions(-) diff --git a/compose/rest.yaml b/compose/rest.yaml index 2fa51002b..ea8ac226b 100644 --- a/compose/rest.yaml +++ b/compose/rest.yaml @@ -171,16 +171,35 @@ endpoints: name: ext required: true title: Output file ext - - name: import + - name: importInit path: "/import" method: POST - title: Import namespace + title: Initiate namespace import session parameters: post: - name: upload type: "*multipart.FileHeader" required: true - title: Namespace import + title: File import + - name: importRun + path: "/import/{sessionID}" + method: POST + title: Run namespace import + parameters: + path: + - name: sessionID + type: uint64 + required: true + title: Import session + post: + - type: string + name: name + required: true + title: Imported namespace name + - type: string + name: slug + required: true + title: Imported namespace slug - name: triggerScript method: POST title: Fire compose:namespace trigger diff --git a/compose/rest/handlers/namespace.go b/compose/rest/handlers/namespace.go index 4ed76d0b3..1605307fc 100644 --- a/compose/rest/handlers/namespace.go +++ b/compose/rest/handlers/namespace.go @@ -27,7 +27,8 @@ type ( Upload(context.Context, *request.NamespaceUpload) (interface{}, error) Clone(context.Context, *request.NamespaceClone) (interface{}, error) Export(context.Context, *request.NamespaceExport) (interface{}, error) - Import(context.Context, *request.NamespaceImport) (interface{}, error) + ImportInit(context.Context, *request.NamespaceImportInit) (interface{}, error) + ImportRun(context.Context, *request.NamespaceImportRun) (interface{}, error) TriggerScript(context.Context, *request.NamespaceTriggerScript) (interface{}, error) ListTranslations(context.Context, *request.NamespaceListTranslations) (interface{}, error) UpdateTranslations(context.Context, *request.NamespaceUpdateTranslations) (interface{}, error) @@ -43,7 +44,8 @@ type ( Upload func(http.ResponseWriter, *http.Request) Clone func(http.ResponseWriter, *http.Request) Export func(http.ResponseWriter, *http.Request) - Import func(http.ResponseWriter, *http.Request) + ImportInit func(http.ResponseWriter, *http.Request) + ImportRun func(http.ResponseWriter, *http.Request) TriggerScript func(http.ResponseWriter, *http.Request) ListTranslations func(http.ResponseWriter, *http.Request) UpdateTranslations func(http.ResponseWriter, *http.Request) @@ -180,15 +182,31 @@ func NewNamespace(h NamespaceAPI) *Namespace { api.Send(w, r, value) }, - Import: func(w http.ResponseWriter, r *http.Request) { + ImportInit: func(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() - params := request.NewNamespaceImport() + params := request.NewNamespaceImportInit() if err := params.Fill(r); err != nil { api.Send(w, r, err) return } - value, err := h.Import(r.Context(), params) + value, err := h.ImportInit(r.Context(), params) + if err != nil { + api.Send(w, r, err) + return + } + + api.Send(w, r, value) + }, + ImportRun: func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + params := request.NewNamespaceImportRun() + if err := params.Fill(r); err != nil { + api.Send(w, r, err) + return + } + + value, err := h.ImportRun(r.Context(), params) if err != nil { api.Send(w, r, err) return @@ -258,7 +276,8 @@ func (h Namespace) MountRoutes(r chi.Router, middlewares ...func(http.Handler) h r.Post("/namespace/upload", h.Upload) r.Post("/namespace/{namespaceID}/clone", h.Clone) r.Get("/namespace/{namespaceID}/export/{filename}.zip", h.Export) - r.Post("/namespace/import", h.Import) + r.Post("/namespace/import", h.ImportInit) + r.Post("/namespace/import/{sessionID}", h.ImportRun) r.Post("/namespace/{namespaceID}/trigger", h.TriggerScript) r.Get("/namespace/{namespaceID}/translation", h.ListTranslations) r.Patch("/namespace/{namespaceID}/translation", h.UpdateTranslations) diff --git a/compose/rest/namespace.go b/compose/rest/namespace.go index 84f94f612..b2a854ce5 100644 --- a/compose/rest/namespace.go +++ b/compose/rest/namespace.go @@ -258,31 +258,42 @@ func (ctrl Namespace) Export(ctx context.Context, r *request.NamespaceExport) (i return ctrl.serveExport(ctx, file, rs, err) } -func (ctrl Namespace) Import(ctx context.Context, r *request.NamespaceImport) (interface{}, error) { +func (ctrl Namespace) ImportInit(ctx context.Context, r *request.NamespaceImportInit) (interface{}, error) { f, err := r.Upload.Open() if err != nil { return nil, err } defer f.Close() - encoder := func(nn resource.InterfaceSet) error { - se := envoyStore.NewStoreEncoder(service.DefaultStore, &envoyStore.EncoderConfig{}) + return ctrl.namespace.ImportInit(ctx, f, r.Upload.Size) +} - bld := envoy.NewBuilder(se) - g, err := bld.Build(ctx, nn...) - if err != nil { - return err +func (ctrl Namespace) ImportRun(ctx context.Context, r *request.NamespaceImportRun) (interface{}, error) { + var ( + dup = &types.Namespace{ + Name: r.Name, + Slug: r.Slug, } - err = envoy.Encode(ctx, g, se) - if err != nil { - return err + encoder = func(nn resource.InterfaceSet) error { + se := envoyStore.NewStoreEncoder(service.DefaultStore, &envoyStore.EncoderConfig{}) + + bld := envoy.NewBuilder(se) + g, err := bld.Build(ctx, nn...) + if err != nil { + return err + } + + err = envoy.Encode(ctx, g, se) + if err != nil { + return err + } + + return nil } + ) - return nil - } - - ns, err := ctrl.namespace.Import(ctx, f, r.Upload.Size, encoder) + ns, err := ctrl.namespace.ImportRun(ctx, r.SessionID, dup, encoder) return ctrl.makePayload(ctx, ns, err) } diff --git a/compose/rest/request/namespace.go b/compose/rest/request/namespace.go index aa1526b61..50839c27a 100644 --- a/compose/rest/request/namespace.go +++ b/compose/rest/request/namespace.go @@ -188,13 +188,30 @@ type ( Ext string } - NamespaceImport struct { + NamespaceImportInit struct { // Upload POST parameter // - // Namespace import + // File import Upload *multipart.FileHeader } + NamespaceImportRun struct { + // SessionID PATH parameter + // + // Import session + SessionID uint64 `json:",string"` + + // Name POST parameter + // + // Imported namespace name + Name string + + // Slug POST parameter + // + // Imported namespace slug + Slug string + } + NamespaceTriggerScript struct { // NamespaceID PATH parameter // @@ -826,25 +843,25 @@ func (r *NamespaceExport) Fill(req *http.Request) (err error) { return err } -// NewNamespaceImport request -func NewNamespaceImport() *NamespaceImport { - return &NamespaceImport{} +// NewNamespaceImportInit request +func NewNamespaceImportInit() *NamespaceImportInit { + return &NamespaceImportInit{} } // Auditable returns all auditable/loggable parameters -func (r NamespaceImport) Auditable() map[string]interface{} { +func (r NamespaceImportInit) Auditable() map[string]interface{} { return map[string]interface{}{ "upload": r.Upload, } } // Auditable returns all auditable/loggable parameters -func (r NamespaceImport) GetUpload() *multipart.FileHeader { +func (r NamespaceImportInit) GetUpload() *multipart.FileHeader { return r.Upload } // Fill processes request and fills internal variables -func (r *NamespaceImport) Fill(req *http.Request) (err error) { +func (r *NamespaceImportInit) Fill(req *http.Request) (err error) { if strings.ToLower(req.Header.Get("content-type")) == "application/json" { err = json.NewDecoder(req.Body).Decode(r) @@ -873,6 +890,86 @@ func (r *NamespaceImport) Fill(req *http.Request) (err error) { return err } +// NewNamespaceImportRun request +func NewNamespaceImportRun() *NamespaceImportRun { + return &NamespaceImportRun{} +} + +// Auditable returns all auditable/loggable parameters +func (r NamespaceImportRun) Auditable() map[string]interface{} { + return map[string]interface{}{ + "sessionID": r.SessionID, + "name": r.Name, + "slug": r.Slug, + } +} + +// Auditable returns all auditable/loggable parameters +func (r NamespaceImportRun) GetSessionID() uint64 { + return r.SessionID +} + +// Auditable returns all auditable/loggable parameters +func (r NamespaceImportRun) GetName() string { + return r.Name +} + +// Auditable returns all auditable/loggable parameters +func (r NamespaceImportRun) GetSlug() string { + return r.Slug +} + +// Fill processes request and fills internal variables +func (r *NamespaceImportRun) Fill(req *http.Request) (err error) { + + if strings.ToLower(req.Header.Get("content-type")) == "application/json" { + err = json.NewDecoder(req.Body).Decode(r) + + switch { + case err == io.EOF: + err = nil + case err != nil: + return fmt.Errorf("error parsing http request body: %w", err) + } + } + + { + if err = req.ParseForm(); err != nil { + return err + } + + // POST params + + if val, ok := req.Form["name"]; ok && len(val) > 0 { + r.Name, err = val[0], nil + if err != nil { + return err + } + } + + if val, ok := req.Form["slug"]; ok && len(val) > 0 { + r.Slug, err = val[0], nil + if err != nil { + return err + } + } + } + + { + var val string + // path params + + val = chi.URLParam(req, "sessionID") + r.SessionID, err = payload.ParseUint64(val), nil + if err != nil { + return err + } + + } + + return err +} + // NewNamespaceTriggerScript request func NewNamespaceTriggerScript() *NamespaceTriggerScript { return &NamespaceTriggerScript{} diff --git a/compose/service/namespace.go b/compose/service/namespace.go index d445ea44a..6b2419997 100644 --- a/compose/service/namespace.go +++ b/compose/service/namespace.go @@ -10,11 +10,13 @@ import ( "mime/multipart" "reflect" "strconv" + "time" automationTypes "github.com/cortezaproject/corteza-server/automation/types" "github.com/cortezaproject/corteza-server/compose/service/event" "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/actionlog" + "github.com/cortezaproject/corteza-server/pkg/auth" "github.com/cortezaproject/corteza-server/pkg/envoy" "github.com/cortezaproject/corteza-server/pkg/envoy/resource" "github.com/cortezaproject/corteza-server/pkg/envoy/yaml" @@ -41,6 +43,18 @@ type ( locale ResourceTranslationsManagerService } + namespaceImportSession struct { + Name string `json:"name"` + Slug string `json:"handle"` + SessionID uint64 `json:"sessionID,string"` + UserID uint64 `json:"userID,string"` + + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + + Resources resource.InterfaceSet `json:"-"` + } + namespaceAccessController interface { CanManageResourceTranslations(ctx context.Context) bool CanSearchNamespaces(context.Context) bool @@ -62,7 +76,8 @@ type ( Update(ctx context.Context, namespace *types.Namespace) (*types.Namespace, error) Clone(ctx context.Context, namespaceID uint64, dup *types.Namespace, decoder func() (resource.InterfaceSet, error), encoder func(resource.InterfaceSet) error) (ns *types.Namespace, err error) Export(ctx context.Context, namespaceID uint64, archive string, decoder func() (resource.InterfaceSet, error), encoder func(resource.InterfaceSet) (envoy.Streamer, error)) (r io.ReadSeeker, err error) - Import(ctx context.Context, f multipart.File, size int64, encoder func(resource.InterfaceSet) error) (ns *types.Namespace, err error) + ImportInit(ctx context.Context, f multipart.File, size int64) (namespaceImportSession, error) + ImportRun(ctx context.Context, sessionID uint64, dup *types.Namespace, encoder func(resource.InterfaceSet) error) (ns *types.Namespace, err error) DeleteByID(ctx context.Context, namespaceID uint64) error } @@ -76,6 +91,12 @@ const ( namespaceLabelsChanged namespaceChanges = 2 ) +var ( + // @todo this is a temporary implementation; we will rework resource import/export + // in the following versions + namespaceSessionStore = make(map[uint64]namespaceImportSession) +) + func Namespace() *namespace { return &namespace{ ac: DefaultAccessControl, @@ -418,9 +439,12 @@ func (svc namespace) Export(ctx context.Context, namespaceID uint64, archive str return bytes.NewReader(buf.Bytes()), svc.recordAction(ctx, aProps, NamespaceActionExport, err) } -func (svc namespace) Import(ctx context.Context, f multipart.File, size int64, encoder func(resource.InterfaceSet) error) (ns *types.Namespace, err error) { +func (svc namespace) ImportInit(ctx context.Context, f multipart.File, size int64) (namespaceImportSession, error) { var ( - aProps = &namespaceActionProps{} + aProps = &namespaceActionProps{} + err error + ns *types.Namespace + session namespaceImportSession ) err = func() error { @@ -468,10 +492,13 @@ func (svc namespace) Import(ctx context.Context, f multipart.File, size int64, e nn = append(nn, mm...) } - // encode - err = encoder(nn) - if err != nil { - return err + // store a session for later + session = namespaceImportSession{ + SessionID: nextID(), + UserID: auth.GetIdentityFromContext(ctx).Identity(), + + CreatedAt: *now(), + Resources: nn, } // find the ns node @@ -482,12 +509,87 @@ func (svc namespace) Import(ctx context.Context, f multipart.File, size int64, e } } - aProps.setNamespace(ns) + session.Name = ns.Name + session.Slug = ns.Slug + namespaceSessionStore[session.SessionID] = session + aProps.setNamespace(ns) return nil }() - return ns, svc.recordAction(ctx, aProps, NamespaceActionImport, err) + return session, svc.recordAction(ctx, aProps, NamespaceActionImportInit, err) +} + +func (svc namespace) ImportRun(ctx context.Context, sessionID uint64, dup *types.Namespace, encoder func(resource.InterfaceSet) error) (ns *types.Namespace, err error) { + var ( + aProps = &namespaceActionProps{namespace: dup} + ) + + err = func() error { + // access control + if err := svc.canImport(ctx); err != nil { + return err + } + + if dup.Slug == "" || !handle.IsValid(dup.Slug) { + return NamespaceErrInvalidHandle() + } + + // check for duplicate + dstNs, err := store.LookupComposeNamespaceBySlug(ctx, svc.store, dup.Slug) + if err != nil && err != store.ErrNotFound { + return err + } + if dstNs != nil { + return NamespaceErrHandleNotUnique() + } + + // session + var ( + session namespaceImportSession + ok bool + ) + if session, ok = namespaceSessionStore[sessionID]; !ok { + return NamespaceErrImportSessionNotFound() + } + defer func() { + delete(namespaceSessionStore, sessionID) + }() + + // Handle renames and references + oldNsRef := resource.MakeRef(types.NamespaceResourceType, resource.MakeIdentifiers(session.Slug, session.Name)) + newNsRef := resource.MakeRef(types.NamespaceResourceType, resource.MakeIdentifiers(dup.Slug, dup.Name)) + + auxNs := resource.FindComposeNamespace(session.Resources, oldNsRef.Identifiers) + auxNs.ID = 0 + auxNs.Name = dup.Name + auxNs.Slug = dup.Slug + dup = auxNs + aProps.setNamespace(dup) + + // Correct internal references + // - namespace identifiers + session.Resources.SearchForIdentifiers(oldNsRef.Identifiers).Walk(func(r resource.Interface) error { + r.ReID(newNsRef.Identifiers) + return nil + }) + // - relations + session.Resources.SearchForReferences(oldNsRef).Walk(func(r resource.Interface) error { + r.ReRef(resource.RefSet{oldNsRef}, resource.RefSet{newNsRef}) + return nil + }) + + // run the import + err = encoder(session.Resources) + if err != nil { + return err + } + + aProps.setNamespace(dup) + return nil + }() + + return dup, svc.recordAction(ctx, aProps, NamespaceActionImportRun, err) } func (svc namespace) DeleteByID(ctx context.Context, namespaceID uint64) error { diff --git a/compose/service/namespace_actions.gen.go b/compose/service/namespace_actions.gen.go index 077ad7ad7..49a20b14b 100644 --- a/compose/service/namespace_actions.gen.go +++ b/compose/service/namespace_actions.gen.go @@ -365,15 +365,35 @@ func NamespaceActionExport(props ...*namespaceActionProps) *namespaceAction { return a } -// NamespaceActionImport returns "compose:namespace.import" action +// NamespaceActionImportInit returns "compose:namespace.importInit" action // // This function is auto-generated. // -func NamespaceActionImport(props ...*namespaceActionProps) *namespaceAction { +func NamespaceActionImportInit(props ...*namespaceActionProps) *namespaceAction { a := &namespaceAction{ timestamp: time.Now(), resource: "compose:namespace", - action: "import", + action: "importInit", + log: "import initialized for {namespace}", + severity: actionlog.Notice, + } + + if len(props) > 0 { + a.props = props[0] + } + + return a +} + +// NamespaceActionImportRun returns "compose:namespace.importRun" action +// +// This function is auto-generated. +// +func NamespaceActionImportRun(props ...*namespaceActionProps) *namespaceAction { + a := &namespaceAction{ + timestamp: time.Now(), + resource: "compose:namespace", + action: "importRun", log: "imported {namespace}", severity: actionlog.Notice, } @@ -729,6 +749,42 @@ func NamespaceErrUnsupportedImportFormat(mm ...*namespaceActionProps) *errors.Er return e } +// NamespaceErrImportSessionNotFound returns "compose:namespace.importSessionNotFound" as *errors.Error +// +// +// This function is auto-generated. +// +func NamespaceErrImportSessionNotFound(mm ...*namespaceActionProps) *errors.Error { + var p = &namespaceActionProps{} + if len(mm) > 0 { + p = mm[0] + } + + var e = errors.New( + errors.KindInternal, + + p.Format("the import session does not exist", nil), + + errors.Meta("type", "importSessionNotFound"), + errors.Meta("resource", "compose:namespace"), + + // action log entry; no formatting, it will be applied inside recordAction fn. + errors.Meta(namespaceLogMetaKey{}, "could not import namespace {{namespace}}; the import session does not exist"), + errors.Meta(namespacePropsMetaKey{}, p), + + // translation namespace & key + errors.Meta(locale.ErrorMetaNamespace{}, "compose"), + errors.Meta(locale.ErrorMetaKey{}, "namespace.errors.importSessionNotFound"), + + errors.StackSkip(1), + ) + + if len(mm) > 0 { + } + + return e +} + // NamespaceErrCloneMultiple returns "compose:namespace.cloneMultiple" as *errors.Error // // diff --git a/compose/service/namespace_actions.yaml b/compose/service/namespace_actions.yaml index 1c1a2b262..345468444 100644 --- a/compose/service/namespace_actions.yaml +++ b/compose/service/namespace_actions.yaml @@ -46,7 +46,10 @@ actions: - action: export log: "exported {namespace}" - - action: import + - action: importInit + log: "import initialized for {namespace}" + + - action: importRun log: "imported {namespace}" - action: delete @@ -88,6 +91,10 @@ errors: message: "unsupported import format" log: "could not import namespace {{namespace}}; unsupported format {{archiveFormat}}" + - error: importSessionNotFound + message: "the import session does not exist" + log: "could not import namespace {{namespace}}; the import session does not exist" + - error: cloneMultiple message: "not allowed to clone multiple namespaces at once" log: "could not clone namespaces; multiple duplications requested at once"