DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Split namespace import to init and run

Browse Source
This commit is contained in:
Tomaž Jerman
2021-10-10 12:43:21 +02:00
parent 96a27b13ad
commit 7f795ef418
7 changed files with 355 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose/rest.yaml
+22 -3
View File
@@ -171,16 +171,35 @@ endpoints:
name: ext
required: true
title: Output file ext
- name: import
- name: importInit
path: "/import"
method: POST
title: Import namespace
title: Initiate namespace import session
parameters:
post:
- name: upload
type: "*multipart.FileHeader"
required: true
title: Namespace import
title: File import
- name: importRun
path: "/import/{sessionID}"
method: POST
title: Run namespace import
parameters:
path:
- name: sessionID
type: uint64
required: true
title: Import session
post:
- type: string
name: name
required: true
title: Imported namespace name
- type: string
name: slug
required: true
title: Imported namespace slug
- name: triggerScript
method: POST
title: Fire compose:namespace trigger
compose/rest/handlers/namespace.go
+25 -6
View File
@@ -27,7 +27,8 @@ type (
Upload(context.Context, *request.NamespaceUpload) (interface{}, error)
Clone(context.Context, *request.NamespaceClone) (interface{}, error)
Export(context.Context, *request.NamespaceExport) (interface{}, error)
Import(context.Context, *request.NamespaceImport) (interface{}, error)
ImportInit(context.Context, *request.NamespaceImportInit) (interface{}, error)
ImportRun(context.Context, *request.NamespaceImportRun) (interface{}, error)
TriggerScript(context.Context, *request.NamespaceTriggerScript) (interface{}, error)
ListTranslations(context.Context, *request.NamespaceListTranslations) (interface{}, error)
UpdateTranslations(context.Context, *request.NamespaceUpdateTranslations) (interface{}, error)
@@ -43,7 +44,8 @@ type (
Upload func(http.ResponseWriter, *http.Request)
Clone func(http.ResponseWriter, *http.Request)
Export func(http.ResponseWriter, *http.Request)
Import func(http.ResponseWriter, *http.Request)
ImportInit func(http.ResponseWriter, *http.Request)
ImportRun func(http.ResponseWriter, *http.Request)
TriggerScript func(http.ResponseWriter, *http.Request)
ListTranslations func(http.ResponseWriter, *http.Request)
UpdateTranslations func(http.ResponseWriter, *http.Request)
@@ -180,15 +182,31 @@ func NewNamespace(h NamespaceAPI) *Namespace {
api.Send(w, r, value)
},
Import: func(w http.ResponseWriter, r *http.Request) {
ImportInit: func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
params := request.NewNamespaceImport()
params := request.NewNamespaceImportInit()
if err := params.Fill(r); err != nil {
api.Send(w, r, err)
return
}
value, err := h.Import(r.Context(), params)
value, err := h.ImportInit(r.Context(), params)
if err != nil {
api.Send(w, r, err)
return
}
api.Send(w, r, value)
},
ImportRun: func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
params := request.NewNamespaceImportRun()
if err := params.Fill(r); err != nil {
api.Send(w, r, err)
return
}
value, err := h.ImportRun(r.Context(), params)
if err != nil {
api.Send(w, r, err)
return
@@ -258,7 +276,8 @@ func (h Namespace) MountRoutes(r chi.Router, middlewares ...func(http.Handler) h
r.Post("/namespace/upload", h.Upload)
r.Post("/namespace/{namespaceID}/clone", h.Clone)
r.Get("/namespace/{namespaceID}/export/{filename}.zip", h.Export)
r.Post("/namespace/import", h.Import)
r.Post("/namespace/import", h.ImportInit)
r.Post("/namespace/import/{sessionID}", h.ImportRun)
r.Post("/namespace/{namespaceID}/trigger", h.TriggerScript)
r.Get("/namespace/{namespaceID}/translation", h.ListTranslations)
r.Patch("/namespace/{namespaceID}/translation", h.UpdateTranslations)
compose/rest/namespace.go
+25 -14
View File
@@ -258,31 +258,42 @@ func (ctrl Namespace) Export(ctx context.Context, r *request.NamespaceExport) (i
return ctrl.serveExport(ctx, file, rs, err)
}
func (ctrl Namespace) Import(ctx context.Context, r *request.NamespaceImport) (interface{}, error) {
func (ctrl Namespace) ImportInit(ctx context.Context, r *request.NamespaceImportInit) (interface{}, error) {
f, err := r.Upload.Open()
if err != nil {
return nil, err
}
defer f.Close()
encoder := func(nn resource.InterfaceSet) error {
se := envoyStore.NewStoreEncoder(service.DefaultStore, &envoyStore.EncoderConfig{})
return ctrl.namespace.ImportInit(ctx, f, r.Upload.Size)
}
bld := envoy.NewBuilder(se)
g, err := bld.Build(ctx, nn...)
if err != nil {
return err
func (ctrl Namespace) ImportRun(ctx context.Context, r *request.NamespaceImportRun) (interface{}, error) {
var (
dup = &types.Namespace{
Name: r.Name,
Slug: r.Slug,
}
err = envoy.Encode(ctx, g, se)
if err != nil {
return err
encoder = func(nn resource.InterfaceSet) error {
se := envoyStore.NewStoreEncoder(service.DefaultStore, &envoyStore.EncoderConfig{})
bld := envoy.NewBuilder(se)
g, err := bld.Build(ctx, nn...)
if err != nil {
return err
}
err = envoy.Encode(ctx, g, se)
if err != nil {
return err
}
return nil
}
)
return nil
}
ns, err := ctrl.namespace.Import(ctx, f, r.Upload.Size, encoder)
ns, err := ctrl.namespace.ImportRun(ctx, r.SessionID, dup, encoder)
return ctrl.makePayload(ctx, ns, err)
}
compose/rest/request/namespace.go
+105 -8
View File
@@ -188,13 +188,30 @@ type (
Ext string
}
NamespaceImport struct {
NamespaceImportInit struct {
// Upload POST parameter
//
// Namespace import
// File import
Upload *multipart.FileHeader
}
NamespaceImportRun struct {
// SessionID PATH parameter
//
// Import session
SessionID uint64 `json:",string"`
// Name POST parameter
//
// Imported namespace name
Name string
// Slug POST parameter
//
// Imported namespace slug
Slug string
}
NamespaceTriggerScript struct {
// NamespaceID PATH parameter
//
@@ -826,25 +843,25 @@ func (r *NamespaceExport) Fill(req *http.Request) (err error) {
return err
}
// NewNamespaceImport request
func NewNamespaceImport() *NamespaceImport {
return &NamespaceImport{}
// NewNamespaceImportInit request
func NewNamespaceImportInit() *NamespaceImportInit {
return &NamespaceImportInit{}
}
// Auditable returns all auditable/loggable parameters
func (r NamespaceImport) Auditable() map[string]interface{} {
func (r NamespaceImportInit) Auditable() map[string]interface{} {
return map[string]interface{}{
"upload": r.Upload,
}
}
// Auditable returns all auditable/loggable parameters
func (r NamespaceImport) GetUpload() *multipart.FileHeader {
func (r NamespaceImportInit) GetUpload() *multipart.FileHeader {
return r.Upload
}
// Fill processes request and fills internal variables
func (r *NamespaceImport) Fill(req *http.Request) (err error) {
func (r *NamespaceImportInit) Fill(req *http.Request) (err error) {
if strings.ToLower(req.Header.Get("content-type")) == "application/json" {
err = json.NewDecoder(req.Body).Decode(r)
@@ -873,6 +890,86 @@ func (r *NamespaceImport) Fill(req *http.Request) (err error) {
return err
}
// NewNamespaceImportRun request
func NewNamespaceImportRun() *NamespaceImportRun {
return &NamespaceImportRun{}
}
// Auditable returns all auditable/loggable parameters
func (r NamespaceImportRun) Auditable() map[string]interface{} {
return map[string]interface{}{
"sessionID": r.SessionID,
"name": r.Name,
"slug": r.Slug,
}
}
// Auditable returns all auditable/loggable parameters
func (r NamespaceImportRun) GetSessionID() uint64 {
return r.SessionID
}
// Auditable returns all auditable/loggable parameters
func (r NamespaceImportRun) GetName() string {
return r.Name
}
// Auditable returns all auditable/loggable parameters
func (r NamespaceImportRun) GetSlug() string {
return r.Slug
}
// Fill processes request and fills internal variables
func (r *NamespaceImportRun) Fill(req *http.Request) (err error) {
if strings.ToLower(req.Header.Get("content-type")) == "application/json" {
err = json.NewDecoder(req.Body).Decode(r)
switch {
case err == io.EOF:
err = nil
case err != nil:
return fmt.Errorf("error parsing http request body: %w", err)
}
}
{
if err = req.ParseForm(); err != nil {
return err
}
// POST params
if val, ok := req.Form["name"]; ok && len(val) > 0 {
r.Name, err = val[0], nil
if err != nil {
return err
}
}
if val, ok := req.Form["slug"]; ok && len(val) > 0 {
r.Slug, err = val[0], nil
if err != nil {
return err
}
}
}
{
var val string
// path params
val = chi.URLParam(req, "sessionID")
r.SessionID, err = payload.ParseUint64(val), nil
if err != nil {
return err
}
}
return err
}
// NewNamespaceTriggerScript request
func NewNamespaceTriggerScript() *NamespaceTriggerScript {
return &NamespaceTriggerScript{}
compose/service/namespace.go
+111 -9
View File
@@ -10,11 +10,13 @@ import (
"mime/multipart"
"reflect"
"strconv"
"time"
automationTypes "github.com/cortezaproject/corteza-server/automation/types"
"github.com/cortezaproject/corteza-server/compose/service/event"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/actionlog"
"github.com/cortezaproject/corteza-server/pkg/auth"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
"github.com/cortezaproject/corteza-server/pkg/envoy/yaml"
@@ -41,6 +43,18 @@ type (
locale ResourceTranslationsManagerService
}
namespaceImportSession struct {
Name string `json:"name"`
Slug string `json:"handle"`
SessionID uint64 `json:"sessionID,string"`
UserID uint64 `json:"userID,string"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
Resources resource.InterfaceSet `json:"-"`
}
namespaceAccessController interface {
CanManageResourceTranslations(ctx context.Context) bool
CanSearchNamespaces(context.Context) bool
@@ -62,7 +76,8 @@ type (
Update(ctx context.Context, namespace *types.Namespace) (*types.Namespace, error)
Clone(ctx context.Context, namespaceID uint64, dup *types.Namespace, decoder func() (resource.InterfaceSet, error), encoder func(resource.InterfaceSet) error) (ns *types.Namespace, err error)
Export(ctx context.Context, namespaceID uint64, archive string, decoder func() (resource.InterfaceSet, error), encoder func(resource.InterfaceSet) (envoy.Streamer, error)) (r io.ReadSeeker, err error)
Import(ctx context.Context, f multipart.File, size int64, encoder func(resource.InterfaceSet) error) (ns *types.Namespace, err error)
ImportInit(ctx context.Context, f multipart.File, size int64) (namespaceImportSession, error)
ImportRun(ctx context.Context, sessionID uint64, dup *types.Namespace, encoder func(resource.InterfaceSet) error) (ns *types.Namespace, err error)
DeleteByID(ctx context.Context, namespaceID uint64) error
}
@@ -76,6 +91,12 @@ const (
namespaceLabelsChanged namespaceChanges = 2
)
var (
// @todo this is a temporary implementation; we will rework resource import/export
// in the following versions
namespaceSessionStore = make(map[uint64]namespaceImportSession)
)
func Namespace() *namespace {
return &namespace{
ac: DefaultAccessControl,
@@ -418,9 +439,12 @@ func (svc namespace) Export(ctx context.Context, namespaceID uint64, archive str
return bytes.NewReader(buf.Bytes()), svc.recordAction(ctx, aProps, NamespaceActionExport, err)
}
func (svc namespace) Import(ctx context.Context, f multipart.File, size int64, encoder func(resource.InterfaceSet) error) (ns *types.Namespace, err error) {
func (svc namespace) ImportInit(ctx context.Context, f multipart.File, size int64) (namespaceImportSession, error) {
var (
aProps = &namespaceActionProps{}
aProps = &namespaceActionProps{}
err error
ns *types.Namespace
session namespaceImportSession
)
err = func() error {
@@ -468,10 +492,13 @@ func (svc namespace) Import(ctx context.Context, f multipart.File, size int64, e
nn = append(nn, mm...)
}
// encode
err = encoder(nn)
if err != nil {
return err
// store a session for later
session = namespaceImportSession{
SessionID: nextID(),
UserID: auth.GetIdentityFromContext(ctx).Identity(),
CreatedAt: *now(),
Resources: nn,
}
// find the ns node
@@ -482,12 +509,87 @@ func (svc namespace) Import(ctx context.Context, f multipart.File, size int64, e
}
}
aProps.setNamespace(ns)
session.Name = ns.Name
session.Slug = ns.Slug
namespaceSessionStore[session.SessionID] = session
aProps.setNamespace(ns)
return nil
}()
return ns, svc.recordAction(ctx, aProps, NamespaceActionImport, err)
return session, svc.recordAction(ctx, aProps, NamespaceActionImportInit, err)
}
func (svc namespace) ImportRun(ctx context.Context, sessionID uint64, dup *types.Namespace, encoder func(resource.InterfaceSet) error) (ns *types.Namespace, err error) {
var (
aProps = &namespaceActionProps{namespace: dup}
)
err = func() error {
// access control
if err := svc.canImport(ctx); err != nil {
return err
}
if dup.Slug == "" || !handle.IsValid(dup.Slug) {
return NamespaceErrInvalidHandle()
}
// check for duplicate
dstNs, err := store.LookupComposeNamespaceBySlug(ctx, svc.store, dup.Slug)
if err != nil && err != store.ErrNotFound {
return err
}
if dstNs != nil {
return NamespaceErrHandleNotUnique()
}
// session
var (
session namespaceImportSession
ok bool
)
if session, ok = namespaceSessionStore[sessionID]; !ok {
return NamespaceErrImportSessionNotFound()
}
defer func() {
delete(namespaceSessionStore, sessionID)
}()
// Handle renames and references
oldNsRef := resource.MakeRef(types.NamespaceResourceType, resource.MakeIdentifiers(session.Slug, session.Name))
newNsRef := resource.MakeRef(types.NamespaceResourceType, resource.MakeIdentifiers(dup.Slug, dup.Name))
auxNs := resource.FindComposeNamespace(session.Resources, oldNsRef.Identifiers)
auxNs.ID = 0
auxNs.Name = dup.Name
auxNs.Slug = dup.Slug
dup = auxNs
aProps.setNamespace(dup)
// Correct internal references
// - namespace identifiers
session.Resources.SearchForIdentifiers(oldNsRef.Identifiers).Walk(func(r resource.Interface) error {
r.ReID(newNsRef.Identifiers)
return nil
})
// - relations
session.Resources.SearchForReferences(oldNsRef).Walk(func(r resource.Interface) error {
r.ReRef(resource.RefSet{oldNsRef}, resource.RefSet{newNsRef})
return nil
})
// run the import
err = encoder(session.Resources)
if err != nil {
return err
}
aProps.setNamespace(dup)
return nil
}()
return dup, svc.recordAction(ctx, aProps, NamespaceActionImportRun, err)
}
func (svc namespace) DeleteByID(ctx context.Context, namespaceID uint64) error {
compose/service/namespace_actions.gen.go
Generated
+59 -3
View File
@@ -365,15 +365,35 @@ func NamespaceActionExport(props ...*namespaceActionProps) *namespaceAction {
return a
}
// NamespaceActionImport returns "compose:namespace.import" action
// NamespaceActionImportInit returns "compose:namespace.importInit" action
//
// This function is auto-generated.
//
func NamespaceActionImport(props ...*namespaceActionProps) *namespaceAction {
func NamespaceActionImportInit(props ...*namespaceActionProps) *namespaceAction {
a := &namespaceAction{
timestamp: time.Now(),
resource: "compose:namespace",
action: "import",
action: "importInit",
log: "import initialized for {namespace}",
severity: actionlog.Notice,
}
if len(props) > 0 {
a.props = props[0]
}
return a
}
// NamespaceActionImportRun returns "compose:namespace.importRun" action
//
// This function is auto-generated.
//
func NamespaceActionImportRun(props ...*namespaceActionProps) *namespaceAction {
a := &namespaceAction{
timestamp: time.Now(),
resource: "compose:namespace",
action: "importRun",
log: "imported {namespace}",
severity: actionlog.Notice,
}
@@ -729,6 +749,42 @@ func NamespaceErrUnsupportedImportFormat(mm ...*namespaceActionProps) *errors.Er
return e
}
// NamespaceErrImportSessionNotFound returns "compose:namespace.importSessionNotFound" as *errors.Error
//
//
// This function is auto-generated.
//
func NamespaceErrImportSessionNotFound(mm ...*namespaceActionProps) *errors.Error {
var p = &namespaceActionProps{}
if len(mm) > 0 {
p = mm[0]
}
var e = errors.New(
errors.KindInternal,
p.Format("the import session does not exist", nil),
errors.Meta("type", "importSessionNotFound"),
errors.Meta("resource", "compose:namespace"),
// action log entry; no formatting, it will be applied inside recordAction fn.
errors.Meta(namespaceLogMetaKey{}, "could not import namespace {{namespace}}; the import session does not exist"),
errors.Meta(namespacePropsMetaKey{}, p),
// translation namespace & key
errors.Meta(locale.ErrorMetaNamespace{}, "compose"),
errors.Meta(locale.ErrorMetaKey{}, "namespace.errors.importSessionNotFound"),
errors.StackSkip(1),
)
if len(mm) > 0 {
}
return e
}
// NamespaceErrCloneMultiple returns "compose:namespace.cloneMultiple" as *errors.Error
//
//
compose/service/namespace_actions.yaml
+8 -1
View File
@@ -46,7 +46,10 @@ actions:
- action: export
log: "exported {namespace}"
- action: import
- action: importInit
log: "import initialized for {namespace}"
- action: importRun
log: "imported {namespace}"
- action: delete
@@ -88,6 +91,10 @@ errors:
message: "unsupported import format"
log: "could not import namespace {{namespace}}; unsupported format {{archiveFormat}}"
- error: importSessionNotFound
message: "the import session does not exist"
log: "could not import namespace {{namespace}}; the import session does not exist"
- error: cloneMultiple
message: "not allowed to clone multiple namespaces at once"
log: "could not clone namespaces; multiple duplications requested at once"