From 9142c7b5a3037cf5b5997d07efa8c26eeeb1738f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Wed, 16 Feb 2022 17:26:53 +0100 Subject: [PATCH] Add support for user export/import --- pkg/envoy/resource/user.go | 25 +++- pkg/envoy/store/user_marshal.go | 50 ++++++- pkg/envoy/yaml/user.go | 5 +- pkg/envoy/yaml/user_marshal.go | 19 +++ pkg/envoy/yaml/user_unmarshal.go | 25 +++- system/rest.yaml | 31 +++++ system/rest/handlers/user.go | 38 +++++ system/rest/request/user.go | 147 ++++++++++++++++++++ system/rest/role.go | 1 + system/rest/user.go | 230 ++++++++++++++++++++++++++++++- 10 files changed, 558 insertions(+), 13 deletions(-) diff --git a/pkg/envoy/resource/user.go b/pkg/envoy/resource/user.go index f38f822e2..c0c123f5d 100644 --- a/pkg/envoy/resource/user.go +++ b/pkg/envoy/resource/user.go @@ -12,22 +12,45 @@ type ( User struct { *base Res *types.User + + RoleMembership []Identifiers + RefRoles RefSet } ) -func NewUser(u *types.User) *User { +func NewUser(u *types.User, roles ...string) *User { r := &User{base: &base{}} r.SetResourceType(types.UserResourceType) r.Res = u r.AddIdentifier(identifiers(u.Handle, u.Email, u.Name, u.ID)...) + // Role membership + for _, role := range roles { + rid := MakeIdentifiers(role) + r.RoleMembership = append(r.RoleMembership, rid) + + r.RefRoles = append(r.RefRoles, r.AddRef(types.RoleResourceType, role)) + } + // Initial timestamps r.SetTimestamps(MakeTimestampsCUDAS(&u.CreatedAt, u.UpdatedAt, u.DeletedAt, nil, u.SuspendedAt)) return r } +func (u *User) AddRoles(roles ...*types.Role) *User { + for _, r := range roles { + idf := firstOkString(r.Handle, strconv.FormatUint(r.ID, 10)) + rid := MakeIdentifiers(idf) + u.RoleMembership = append(u.RoleMembership, rid) + + u.RefRoles = append(u.RefRoles, u.AddRef(types.RoleResourceType, idf)) + } + + return u +} + func (r *User) Resource() interface{} { return r.Res } diff --git a/pkg/envoy/store/user_marshal.go b/pkg/envoy/store/user_marshal.go index f1a72fb20..fcaedd514 100644 --- a/pkg/envoy/store/user_marshal.go +++ b/pkg/envoy/store/user_marshal.go @@ -5,6 +5,7 @@ import ( "github.com/cortezaproject/corteza-server/pkg/envoy/resource" "github.com/cortezaproject/corteza-server/store" + "github.com/cortezaproject/corteza-server/system/types" ) func NewUserFromResource(res *resource.User, cfg *EncoderConfig) resourceState { @@ -71,7 +72,10 @@ func (n *user) Encode(ctx context.Context, pl *payload) (err error) { // Create a fresh user if !exists { - return store.CreateUser(ctx, pl.s, res) + if err = store.CreateUser(ctx, pl.s, res); err != nil { + return + } + return n.membership(ctx, pl.s, res, pl.state.ParentResources, exists) } // Update existing user @@ -86,11 +90,47 @@ func (n *user) Encode(ctx context.Context, pl *payload) (err error) { res = mergeUsers(res, n.u) } - err = store.UpdateUser(ctx, pl.s, res) - if err != nil { + if err = store.UpdateUser(ctx, pl.s, res); err != nil { return err } - n.res.Res = res - return nil + + return n.membership(ctx, pl.s, res, pl.state.ParentResources, exists) +} + +func (n *user) membership(ctx context.Context, s store.Storer, res *types.User, pp resource.InterfaceSet, exists bool) (err error) { + // find all roles + roles := make([]uint64, 0, 10) + var r *types.Role + for _, m := range n.res.RoleMembership { + r, err = findRole(ctx, s, pp, m) + if err != nil { + return + } + + roles = append(roles, r.ID) + } + + // update + // @todo some smarter diff calculations; should be fine for now but could be improved. + var mm types.RoleMemberSet + if exists { + mm, _, err = s.SearchRoleMembers(ctx, types.RoleMemberFilter{UserID: n.u.ID}) + if err != nil { + return + } + } + + for _, m := range mm { + if err = store.DeleteRoleMember(ctx, s, m); err != nil { + return + } + } + for _, r := range roles { + if err = store.CreateRoleMember(ctx, s, &types.RoleMember{UserID: res.ID, RoleID: r}); err != nil { + return + } + } + + return } diff --git a/pkg/envoy/yaml/user.go b/pkg/envoy/yaml/user.go index caec550ce..aee74483d 100644 --- a/pkg/envoy/yaml/user.go +++ b/pkg/envoy/yaml/user.go @@ -7,8 +7,9 @@ import ( type ( user struct { - res *types.User - ts *resource.Timestamps + res *types.User + ts *resource.Timestamps + roles []string envoyConfig *resource.EnvoyConfig encoderConfig *EncoderConfig diff --git a/pkg/envoy/yaml/user_marshal.go b/pkg/envoy/yaml/user_marshal.go index 0781c0548..8014d75d8 100644 --- a/pkg/envoy/yaml/user_marshal.go +++ b/pkg/envoy/yaml/user_marshal.go @@ -9,8 +9,14 @@ import ( ) func userFromResource(r *resource.User, cfg *EncoderConfig) *user { + roles := make([]string, 0, len(r.RefRoles)) + for _, r := range r.RefRoles { + roles = append(roles, r.Identifiers.First()) + } + return &user{ res: r.Res, + roles: roles, encoderConfig: cfg, } } @@ -53,7 +59,20 @@ func (u *user) MarshalYAML() (interface{}, error) { "name", u.res.Name, "handle", u.res.Handle, "kind", u.res.Kind, + ) + if err != nil { + return nil, err + } + if len(u.roles) > 0 { + nsn, err = addMap(nsn, + "roles", u.roles, + ) + if err != nil { + return nil, err + } + } + nsn, err = addMap(nsn, "meta", u.res.Meta, "labels", u.res.Labels, diff --git a/pkg/envoy/yaml/user_unmarshal.go b/pkg/envoy/yaml/user_unmarshal.go index 0ce8a3782..75ff61b31 100644 --- a/pkg/envoy/yaml/user_unmarshal.go +++ b/pkg/envoy/yaml/user_unmarshal.go @@ -68,6 +68,10 @@ func (wrap *user) UnmarshalYAML(n *yaml.Node) (err error) { return } + if wrap.roles, err = decodeUserRoles(n); err != nil { + return + } + if wrap.ts, err = decodeTimestamps(n); err != nil { return } @@ -75,6 +79,25 @@ func (wrap *user) UnmarshalYAML(n *yaml.Node) (err error) { return nil } +func decodeUserRoles(n *yaml.Node) (roles []string, err error) { + var ecNode *yaml.Node + for i, k := range n.Content { + if k.Value == "roles" { + ecNode = n.Content[i+1] + break + } + } + + if ecNode == nil { + return + } + + return roles, y7s.EachSeq(ecNode, func(v *yaml.Node) (err error) { + roles = append(roles, v.Value) + return nil + }) +} + func (wset userSet) MarshalEnvoy() ([]resource.Interface, error) { nn := make([]resource.Interface, 0, len(wset)) @@ -91,7 +114,7 @@ func (wset userSet) MarshalEnvoy() ([]resource.Interface, error) { } func (wrap user) MarshalEnvoy() ([]resource.Interface, error) { - rs := resource.NewUser(wrap.res) + rs := resource.NewUser(wrap.res, wrap.roles...) rs.SetTimestamps(wrap.ts) rs.SetConfig(wrap.envoyConfig) diff --git a/system/rest.yaml b/system/rest.yaml index 32ed24a66..6edf0bd2c 100644 --- a/system/rest.yaml +++ b/system/rest.yaml @@ -670,6 +670,37 @@ endpoints: name: userID required: true title: ID + + - name: export + method: GET + title: Export users + path: "/export/{filename}.zip" + parameters: + path: + - name: filename + type: string + required: true + title: Output filename + get: + - name: inclRoleMembership + type: bool + required: false + title: Include role membership + - name: inclRoles + type: bool + required: false + title: Include roles + - name: import + method: POST + title: Import users + path: "/import" + parameters: + post: + - name: upload + type: "*multipart.FileHeader" + required: true + title: File import + - title: Applications path: "/application" entrypoint: application diff --git a/system/rest/handlers/user.go b/system/rest/handlers/user.go index ce9817c63..674638155 100644 --- a/system/rest/handlers/user.go +++ b/system/rest/handlers/user.go @@ -34,6 +34,8 @@ type ( MembershipRemove(context.Context, *request.UserMembershipRemove) (interface{}, error) TriggerScript(context.Context, *request.UserTriggerScript) (interface{}, error) SessionsRemove(context.Context, *request.UserSessionsRemove) (interface{}, error) + Export(context.Context, *request.UserExport) (interface{}, error) + Import(context.Context, *request.UserImport) (interface{}, error) } // HTTP API interface @@ -53,6 +55,8 @@ type ( MembershipRemove func(http.ResponseWriter, *http.Request) TriggerScript func(http.ResponseWriter, *http.Request) SessionsRemove func(http.ResponseWriter, *http.Request) + Export func(http.ResponseWriter, *http.Request) + Import func(http.ResponseWriter, *http.Request) } ) @@ -296,6 +300,38 @@ func NewUser(h UserAPI) *User { return } + api.Send(w, r, value) + }, + Export: func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + params := request.NewUserExport() + if err := params.Fill(r); err != nil { + api.Send(w, r, err) + return + } + + value, err := h.Export(r.Context(), params) + if err != nil { + api.Send(w, r, err) + return + } + + api.Send(w, r, value) + }, + Import: func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + params := request.NewUserImport() + if err := params.Fill(r); err != nil { + api.Send(w, r, err) + return + } + + value, err := h.Import(r.Context(), params) + if err != nil { + api.Send(w, r, err) + return + } + api.Send(w, r, value) }, } @@ -319,5 +355,7 @@ func (h User) MountRoutes(r chi.Router, middlewares ...func(http.Handler) http.H r.Delete("/users/{userID}/membership/{roleID}", h.MembershipRemove) r.Post("/users/{userID}/trigger", h.TriggerScript) r.Delete("/users/{userID}/sessions", h.SessionsRemove) + r.Get("/users/export/{filename}.zip", h.Export) + r.Post("/users/import", h.Import) }) } diff --git a/system/rest/request/user.go b/system/rest/request/user.go index 5b1c96266..dc379b46c 100644 --- a/system/rest/request/user.go +++ b/system/rest/request/user.go @@ -279,6 +279,30 @@ type ( // ID UserID uint64 `json:",string"` } + + UserExport struct { + // Filename PATH parameter + // + // Output filename + Filename string + + // InclRoleMembership GET parameter + // + // Include role membership + InclRoleMembership bool + + // InclRoles GET parameter + // + // Include roles + InclRoles bool + } + + UserImport struct { + // Upload POST parameter + // + // File import + Upload *multipart.FileHeader + } ) // NewUserList request @@ -1395,3 +1419,126 @@ func (r *UserSessionsRemove) Fill(req *http.Request) (err error) { return err } + +// NewUserExport request +func NewUserExport() *UserExport { + return &UserExport{} +} + +// Auditable returns all auditable/loggable parameters +func (r UserExport) Auditable() map[string]interface{} { + return map[string]interface{}{ + "filename": r.Filename, + "inclRoleMembership": r.InclRoleMembership, + "inclRoles": r.InclRoles, + } +} + +// Auditable returns all auditable/loggable parameters +func (r UserExport) GetFilename() string { + return r.Filename +} + +// Auditable returns all auditable/loggable parameters +func (r UserExport) GetInclRoleMembership() bool { + return r.InclRoleMembership +} + +// Auditable returns all auditable/loggable parameters +func (r UserExport) GetInclRoles() bool { + return r.InclRoles +} + +// Fill processes request and fills internal variables +func (r *UserExport) Fill(req *http.Request) (err error) { + + { + // GET params + tmp := req.URL.Query() + + if val, ok := tmp["inclRoleMembership"]; ok && len(val) > 0 { + r.InclRoleMembership, err = payload.ParseBool(val[0]), nil + if err != nil { + return err + } + } + if val, ok := tmp["inclRoles"]; ok && len(val) > 0 { + r.InclRoles, err = payload.ParseBool(val[0]), nil + if err != nil { + return err + } + } + } + + { + var val string + // path params + + val = chi.URLParam(req, "filename") + r.Filename, err = val, nil + if err != nil { + return err + } + + } + + return err +} + +// NewUserImport request +func NewUserImport() *UserImport { + return &UserImport{} +} + +// Auditable returns all auditable/loggable parameters +func (r UserImport) Auditable() map[string]interface{} { + return map[string]interface{}{ + "upload": r.Upload, + } +} + +// Auditable returns all auditable/loggable parameters +func (r UserImport) GetUpload() *multipart.FileHeader { + return r.Upload +} + +// Fill processes request and fills internal variables +func (r *UserImport) Fill(req *http.Request) (err error) { + + if strings.HasPrefix(strings.ToLower(req.Header.Get("content-type")), "application/json") { + err = json.NewDecoder(req.Body).Decode(r) + + switch { + case err == io.EOF: + err = nil + case err != nil: + return fmt.Errorf("error parsing http request body: %w", err) + } + } + + { + // Caching 32MB to memory, the rest to disk + if err = req.ParseMultipartForm(32 << 20); err != nil && err != http.ErrNotMultipart { + return err + } else if err == nil { + // Multipart params + + // Ignoring upload as its handled in the POST params section + } + } + + { + if err = req.ParseForm(); err != nil { + return err + } + + // POST params + + if _, r.Upload, err = req.FormFile("upload"); err != nil { + return fmt.Errorf("error processing uploaded file: %w", err) + } + + } + + return err +} diff --git a/system/rest/role.go b/system/rest/role.go index ff2fed6c6..af0523aa2 100644 --- a/system/rest/role.go +++ b/system/rest/role.go @@ -26,6 +26,7 @@ type ( roleAccessController interface { CanGrant(context.Context) bool + CanCreateRole(context.Context) bool CanUpdateRole(context.Context, *types.Role) bool CanDeleteRole(context.Context, *types.Role) bool CanManageMembersOnRole(context.Context, *types.Role) bool diff --git a/system/rest/user.go b/system/rest/user.go index 7cdf047d1..52d69dc93 100644 --- a/system/rest/user.go +++ b/system/rest/user.go @@ -1,19 +1,29 @@ package rest import ( + "archive/zip" + "bytes" "context" "encoding/json" "fmt" + "io" + "io/ioutil" "net/http" + "time" "github.com/cortezaproject/corteza-server/pkg/api" "github.com/cortezaproject/corteza-server/pkg/corredor" + "github.com/cortezaproject/corteza-server/pkg/envoy" + "github.com/cortezaproject/corteza-server/pkg/envoy/resource" + envoyStore "github.com/cortezaproject/corteza-server/pkg/envoy/store" + "github.com/cortezaproject/corteza-server/pkg/envoy/yaml" "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/pkg/payload" "github.com/cortezaproject/corteza-server/system/rest/request" "github.com/cortezaproject/corteza-server/system/service" "github.com/cortezaproject/corteza-server/system/service/event" "github.com/cortezaproject/corteza-server/system/types" + "github.com/gabriel-vasile/mimetype" "github.com/pkg/errors" "github.com/spf13/cast" ) @@ -24,19 +34,30 @@ type ( User struct { user service.UserService role service.RoleService + + userAc userAccessController + roleAc roleAccessController } userSetPayload struct { Filter types.UserFilter `json:"filter"` Set types.UserSet `json:"set"` } + + userAccessController interface { + CanCreateUser(context.Context) bool + CanUpdateUser(context.Context, *types.User) bool + } ) func (User) New() *User { - ctrl := &User{} - ctrl.user = service.DefaultUser - ctrl.role = service.DefaultRole - return ctrl + return &User{ + user: service.DefaultUser, + role: service.DefaultRole, + + userAc: service.DefaultAccessControl, + roleAc: service.DefaultAccessControl, + } } func (ctrl User) List(ctx context.Context, r *request.UserList) (interface{}, error) { @@ -256,6 +277,195 @@ func (ctrl *User) SessionsRemove(ctx context.Context, r *request.UserSessionsRem return } +// Export exports users with optional role membership and related roles +// +// @note this is a temporary implementation; it will be reworked when we rework Envoy and related bits. +func (ctrl *User) Export(ctx context.Context, r *request.UserExport) (rsp interface{}, err error) { + // Users + uu, _, err := ctrl.user.Find(ctx, types.UserFilter{}) + if err != nil { + return + } + + // Roles + roleIndex := make(map[uint64]*types.Role) + roleResIndex := make(map[uint64]resource.Interface) + rr, _, err := ctrl.role.Find(ctx, types.RoleFilter{Paging: filter.Paging{Limit: 0}}) + if err != nil { + return + } + for _, r := range rr { + roleIndex[r.ID] = r + } + + // Membership + resources := make(resource.InterfaceSet, 0, len(uu)) + var membership types.RoleMemberSet + for _, u := range uu { + usrRes := resource.NewUser(u) + + if r.InclRoleMembership { + membership, err = ctrl.role.Membership(ctx, u.ID) + if err != nil { + return + } + + aux := make(types.RoleSet, 0, 2) + + for _, m := range membership { + if _, ok := roleResIndex[m.RoleID]; !ok { + roleResIndex[m.RoleID] = resource.NewRole(roleIndex[m.RoleID]) + if r.InclRoles { + resources = append(resources, roleResIndex[m.RoleID]) + } + } + aux = append(aux, roleIndex[m.RoleID]) + } + + usrRes.AddRoles(aux...) + } + + resources = append(resources, usrRes) + } + + // Encode + ye := yaml.NewYamlEncoder(&yaml.EncoderConfig{}) + bld := envoy.NewBuilder(ye) + g, err := bld.Build(ctx, resources...) + if err != nil { + return nil, err + } + + err = envoy.Encode(ctx, g, ye) + if err != nil { + return + } + + // make archive + buf := bytes.NewBuffer(nil) + w := zip.NewWriter(buf) + + var ( + f io.Writer + bb []byte + ) + for _, s := range ye.Stream() { + // @todo generalize when needed + f, err = w.Create(fmt.Sprintf("%s.yaml", s.Resource)) + if err != nil { + return + } + + bb, err = ioutil.ReadAll(s.Source) + if err != nil { + return + } + + _, err = f.Write(bb) + if err != nil { + return + } + } + + err = w.Close() + if err != nil { + return + } + return ctrl.serve(ctx, fmt.Sprintf("%s.zip", r.Filename), bytes.NewReader(buf.Bytes()), nil) +} + +// Import imports users with optional role membership and related roles +// +// @note this is a temporary implementation; it will be reworked when we rework Envoy and related bits. +func (ctrl *User) Import(ctx context.Context, r *request.UserImport) (rsp interface{}, err error) { + // AC + // @todo refactor when we refactor this part of the sys + if !ctrl.userAc.CanCreateUser(ctx) { + err = fmt.Errorf("cannot import users: not allowed to create users") + return + } + if !ctrl.roleAc.CanCreateRole(ctx) { + err = fmt.Errorf("cannot import users: not allowed to create roles") + return + } + + // Parse inputs + f, err := r.Upload.Open() + if err != nil { + return + } + defer f.Close() + + mt, err := mimetype.DetectReader(f) + if err != nil { + return + } + _, err = f.Seek(0, 0) + if err != nil { + return + } + + if !mt.Is("application/zip") { + err = fmt.Errorf("cannot import users: unsupported file format") + return + } + + // un-archive + archive, err := zip.NewReader(f, r.Upload.Size) + if err != nil { + return + } + + // decode with Envoy + yd := yaml.Decoder() + nn := make([]resource.Interface, 0, 10) + var mm []resource.Interface + for _, archF := range archive.File { + if archF.FileInfo().IsDir() { + continue + } + var f io.ReadCloser + + f, err = archF.Open() + if err != nil { + return + } + defer f.Close() + + mm, err = yd.Decode(ctx, f, nil) + if err != nil { + return + } + nn = append(nn, mm...) + } + + // Validate + for _, n := range nn { + switch n.ResourceType() { + case types.UserResourceType, + types.RoleResourceType: + continue + + default: + err = fmt.Errorf("cannot import users: invalid resource provided: %s", n.ResourceType()) + return + } + } + + se := envoyStore.NewStoreEncoder(service.DefaultStore, &envoyStore.EncoderConfig{ + OnExisting: resource.Skip, + }) + + bld := envoy.NewBuilder(se) + g, err := bld.Build(ctx, nn...) + if err != nil { + return + } + + err = envoy.Encode(ctx, g, se) + return api.OK(), err +} + func (ctrl User) makeFilterPayload(ctx context.Context, uu types.UserSet, f types.UserFilter, err error) (*userSetPayload, error) { if err != nil { return nil, err @@ -267,3 +477,15 @@ func (ctrl User) makeFilterPayload(ctx context.Context, uu types.UserSet, f type return &userSetPayload{Filter: f, Set: uu}, nil } + +func (ctrl User) serve(ctx context.Context, fn string, archive io.ReadSeeker, err error) (interface{}, error) { + if err != nil { + return nil, err + } + + return func(w http.ResponseWriter, req *http.Request) { + w.Header().Add("Content-Disposition", "attachment; filename="+fn) + + http.ServeContent(w, req, fn, time.Now(), archive) + }, nil +}