3
0

Add support for user export/import

This commit is contained in:
Tomaž Jerman
2022-02-16 17:26:53 +01:00
parent d841aad13b
commit 9142c7b5a3
10 changed files with 558 additions and 13 deletions
+24 -1
View File
@@ -12,22 +12,45 @@ type (
User struct {
*base
Res *types.User
RoleMembership []Identifiers
RefRoles RefSet
}
)
func NewUser(u *types.User) *User {
func NewUser(u *types.User, roles ...string) *User {
r := &User{base: &base{}}
r.SetResourceType(types.UserResourceType)
r.Res = u
r.AddIdentifier(identifiers(u.Handle, u.Email, u.Name, u.ID)...)
// Role membership
for _, role := range roles {
rid := MakeIdentifiers(role)
r.RoleMembership = append(r.RoleMembership, rid)
r.RefRoles = append(r.RefRoles, r.AddRef(types.RoleResourceType, role))
}
// Initial timestamps
r.SetTimestamps(MakeTimestampsCUDAS(&u.CreatedAt, u.UpdatedAt, u.DeletedAt, nil, u.SuspendedAt))
return r
}
func (u *User) AddRoles(roles ...*types.Role) *User {
for _, r := range roles {
idf := firstOkString(r.Handle, strconv.FormatUint(r.ID, 10))
rid := MakeIdentifiers(idf)
u.RoleMembership = append(u.RoleMembership, rid)
u.RefRoles = append(u.RefRoles, u.AddRef(types.RoleResourceType, idf))
}
return u
}
func (r *User) Resource() interface{} {
return r.Res
}
+45 -5
View File
@@ -5,6 +5,7 @@ import (
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
"github.com/cortezaproject/corteza-server/store"
"github.com/cortezaproject/corteza-server/system/types"
)
func NewUserFromResource(res *resource.User, cfg *EncoderConfig) resourceState {
@@ -71,7 +72,10 @@ func (n *user) Encode(ctx context.Context, pl *payload) (err error) {
// Create a fresh user
if !exists {
return store.CreateUser(ctx, pl.s, res)
if err = store.CreateUser(ctx, pl.s, res); err != nil {
return
}
return n.membership(ctx, pl.s, res, pl.state.ParentResources, exists)
}
// Update existing user
@@ -86,11 +90,47 @@ func (n *user) Encode(ctx context.Context, pl *payload) (err error) {
res = mergeUsers(res, n.u)
}
err = store.UpdateUser(ctx, pl.s, res)
if err != nil {
if err = store.UpdateUser(ctx, pl.s, res); err != nil {
return err
}
n.res.Res = res
return nil
return n.membership(ctx, pl.s, res, pl.state.ParentResources, exists)
}
func (n *user) membership(ctx context.Context, s store.Storer, res *types.User, pp resource.InterfaceSet, exists bool) (err error) {
// find all roles
roles := make([]uint64, 0, 10)
var r *types.Role
for _, m := range n.res.RoleMembership {
r, err = findRole(ctx, s, pp, m)
if err != nil {
return
}
roles = append(roles, r.ID)
}
// update
// @todo some smarter diff calculations; should be fine for now but could be improved.
var mm types.RoleMemberSet
if exists {
mm, _, err = s.SearchRoleMembers(ctx, types.RoleMemberFilter{UserID: n.u.ID})
if err != nil {
return
}
}
for _, m := range mm {
if err = store.DeleteRoleMember(ctx, s, m); err != nil {
return
}
}
for _, r := range roles {
if err = store.CreateRoleMember(ctx, s, &types.RoleMember{UserID: res.ID, RoleID: r}); err != nil {
return
}
}
return
}
+3 -2
View File
@@ -7,8 +7,9 @@ import (
type (
user struct {
res *types.User
ts *resource.Timestamps
res *types.User
ts *resource.Timestamps
roles []string
envoyConfig *resource.EnvoyConfig
encoderConfig *EncoderConfig
+19
View File
@@ -9,8 +9,14 @@ import (
)
func userFromResource(r *resource.User, cfg *EncoderConfig) *user {
roles := make([]string, 0, len(r.RefRoles))
for _, r := range r.RefRoles {
roles = append(roles, r.Identifiers.First())
}
return &user{
res: r.Res,
roles: roles,
encoderConfig: cfg,
}
}
@@ -53,7 +59,20 @@ func (u *user) MarshalYAML() (interface{}, error) {
"name", u.res.Name,
"handle", u.res.Handle,
"kind", u.res.Kind,
)
if err != nil {
return nil, err
}
if len(u.roles) > 0 {
nsn, err = addMap(nsn,
"roles", u.roles,
)
if err != nil {
return nil, err
}
}
nsn, err = addMap(nsn,
"meta", u.res.Meta,
"labels", u.res.Labels,
+24 -1
View File
@@ -68,6 +68,10 @@ func (wrap *user) UnmarshalYAML(n *yaml.Node) (err error) {
return
}
if wrap.roles, err = decodeUserRoles(n); err != nil {
return
}
if wrap.ts, err = decodeTimestamps(n); err != nil {
return
}
@@ -75,6 +79,25 @@ func (wrap *user) UnmarshalYAML(n *yaml.Node) (err error) {
return nil
}
func decodeUserRoles(n *yaml.Node) (roles []string, err error) {
var ecNode *yaml.Node
for i, k := range n.Content {
if k.Value == "roles" {
ecNode = n.Content[i+1]
break
}
}
if ecNode == nil {
return
}
return roles, y7s.EachSeq(ecNode, func(v *yaml.Node) (err error) {
roles = append(roles, v.Value)
return nil
})
}
func (wset userSet) MarshalEnvoy() ([]resource.Interface, error) {
nn := make([]resource.Interface, 0, len(wset))
@@ -91,7 +114,7 @@ func (wset userSet) MarshalEnvoy() ([]resource.Interface, error) {
}
func (wrap user) MarshalEnvoy() ([]resource.Interface, error) {
rs := resource.NewUser(wrap.res)
rs := resource.NewUser(wrap.res, wrap.roles...)
rs.SetTimestamps(wrap.ts)
rs.SetConfig(wrap.envoyConfig)
+31
View File
@@ -670,6 +670,37 @@ endpoints:
name: userID
required: true
title: ID
- name: export
method: GET
title: Export users
path: "/export/{filename}.zip"
parameters:
path:
- name: filename
type: string
required: true
title: Output filename
get:
- name: inclRoleMembership
type: bool
required: false
title: Include role membership
- name: inclRoles
type: bool
required: false
title: Include roles
- name: import
method: POST
title: Import users
path: "/import"
parameters:
post:
- name: upload
type: "*multipart.FileHeader"
required: true
title: File import
- title: Applications
path: "/application"
entrypoint: application
+38
View File
@@ -34,6 +34,8 @@ type (
MembershipRemove(context.Context, *request.UserMembershipRemove) (interface{}, error)
TriggerScript(context.Context, *request.UserTriggerScript) (interface{}, error)
SessionsRemove(context.Context, *request.UserSessionsRemove) (interface{}, error)
Export(context.Context, *request.UserExport) (interface{}, error)
Import(context.Context, *request.UserImport) (interface{}, error)
}
// HTTP API interface
@@ -53,6 +55,8 @@ type (
MembershipRemove func(http.ResponseWriter, *http.Request)
TriggerScript func(http.ResponseWriter, *http.Request)
SessionsRemove func(http.ResponseWriter, *http.Request)
Export func(http.ResponseWriter, *http.Request)
Import func(http.ResponseWriter, *http.Request)
}
)
@@ -296,6 +300,38 @@ func NewUser(h UserAPI) *User {
return
}
api.Send(w, r, value)
},
Export: func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
params := request.NewUserExport()
if err := params.Fill(r); err != nil {
api.Send(w, r, err)
return
}
value, err := h.Export(r.Context(), params)
if err != nil {
api.Send(w, r, err)
return
}
api.Send(w, r, value)
},
Import: func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
params := request.NewUserImport()
if err := params.Fill(r); err != nil {
api.Send(w, r, err)
return
}
value, err := h.Import(r.Context(), params)
if err != nil {
api.Send(w, r, err)
return
}
api.Send(w, r, value)
},
}
@@ -319,5 +355,7 @@ func (h User) MountRoutes(r chi.Router, middlewares ...func(http.Handler) http.H
r.Delete("/users/{userID}/membership/{roleID}", h.MembershipRemove)
r.Post("/users/{userID}/trigger", h.TriggerScript)
r.Delete("/users/{userID}/sessions", h.SessionsRemove)
r.Get("/users/export/{filename}.zip", h.Export)
r.Post("/users/import", h.Import)
})
}
+147
View File
@@ -279,6 +279,30 @@ type (
// ID
UserID uint64 `json:",string"`
}
UserExport struct {
// Filename PATH parameter
//
// Output filename
Filename string
// InclRoleMembership GET parameter
//
// Include role membership
InclRoleMembership bool
// InclRoles GET parameter
//
// Include roles
InclRoles bool
}
UserImport struct {
// Upload POST parameter
//
// File import
Upload *multipart.FileHeader
}
)
// NewUserList request
@@ -1395,3 +1419,126 @@ func (r *UserSessionsRemove) Fill(req *http.Request) (err error) {
return err
}
// NewUserExport request
func NewUserExport() *UserExport {
return &UserExport{}
}
// Auditable returns all auditable/loggable parameters
func (r UserExport) Auditable() map[string]interface{} {
return map[string]interface{}{
"filename": r.Filename,
"inclRoleMembership": r.InclRoleMembership,
"inclRoles": r.InclRoles,
}
}
// Auditable returns all auditable/loggable parameters
func (r UserExport) GetFilename() string {
return r.Filename
}
// Auditable returns all auditable/loggable parameters
func (r UserExport) GetInclRoleMembership() bool {
return r.InclRoleMembership
}
// Auditable returns all auditable/loggable parameters
func (r UserExport) GetInclRoles() bool {
return r.InclRoles
}
// Fill processes request and fills internal variables
func (r *UserExport) Fill(req *http.Request) (err error) {
{
// GET params
tmp := req.URL.Query()
if val, ok := tmp["inclRoleMembership"]; ok && len(val) > 0 {
r.InclRoleMembership, err = payload.ParseBool(val[0]), nil
if err != nil {
return err
}
}
if val, ok := tmp["inclRoles"]; ok && len(val) > 0 {
r.InclRoles, err = payload.ParseBool(val[0]), nil
if err != nil {
return err
}
}
}
{
var val string
// path params
val = chi.URLParam(req, "filename")
r.Filename, err = val, nil
if err != nil {
return err
}
}
return err
}
// NewUserImport request
func NewUserImport() *UserImport {
return &UserImport{}
}
// Auditable returns all auditable/loggable parameters
func (r UserImport) Auditable() map[string]interface{} {
return map[string]interface{}{
"upload": r.Upload,
}
}
// Auditable returns all auditable/loggable parameters
func (r UserImport) GetUpload() *multipart.FileHeader {
return r.Upload
}
// Fill processes request and fills internal variables
func (r *UserImport) Fill(req *http.Request) (err error) {
if strings.HasPrefix(strings.ToLower(req.Header.Get("content-type")), "application/json") {
err = json.NewDecoder(req.Body).Decode(r)
switch {
case err == io.EOF:
err = nil
case err != nil:
return fmt.Errorf("error parsing http request body: %w", err)
}
}
{
// Caching 32MB to memory, the rest to disk
if err = req.ParseMultipartForm(32 << 20); err != nil && err != http.ErrNotMultipart {
return err
} else if err == nil {
// Multipart params
// Ignoring upload as its handled in the POST params section
}
}
{
if err = req.ParseForm(); err != nil {
return err
}
// POST params
if _, r.Upload, err = req.FormFile("upload"); err != nil {
return fmt.Errorf("error processing uploaded file: %w", err)
}
}
return err
}
+1
View File
@@ -26,6 +26,7 @@ type (
roleAccessController interface {
CanGrant(context.Context) bool
CanCreateRole(context.Context) bool
CanUpdateRole(context.Context, *types.Role) bool
CanDeleteRole(context.Context, *types.Role) bool
CanManageMembersOnRole(context.Context, *types.Role) bool
+226 -4
View File
@@ -1,19 +1,29 @@
package rest
import (
"archive/zip"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"time"
"github.com/cortezaproject/corteza-server/pkg/api"
"github.com/cortezaproject/corteza-server/pkg/corredor"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
envoyStore "github.com/cortezaproject/corteza-server/pkg/envoy/store"
"github.com/cortezaproject/corteza-server/pkg/envoy/yaml"
"github.com/cortezaproject/corteza-server/pkg/filter"
"github.com/cortezaproject/corteza-server/pkg/payload"
"github.com/cortezaproject/corteza-server/system/rest/request"
"github.com/cortezaproject/corteza-server/system/service"
"github.com/cortezaproject/corteza-server/system/service/event"
"github.com/cortezaproject/corteza-server/system/types"
"github.com/gabriel-vasile/mimetype"
"github.com/pkg/errors"
"github.com/spf13/cast"
)
@@ -24,19 +34,30 @@ type (
User struct {
user service.UserService
role service.RoleService
userAc userAccessController
roleAc roleAccessController
}
userSetPayload struct {
Filter types.UserFilter `json:"filter"`
Set types.UserSet `json:"set"`
}
userAccessController interface {
CanCreateUser(context.Context) bool
CanUpdateUser(context.Context, *types.User) bool
}
)
func (User) New() *User {
ctrl := &User{}
ctrl.user = service.DefaultUser
ctrl.role = service.DefaultRole
return ctrl
return &User{
user: service.DefaultUser,
role: service.DefaultRole,
userAc: service.DefaultAccessControl,
roleAc: service.DefaultAccessControl,
}
}
func (ctrl User) List(ctx context.Context, r *request.UserList) (interface{}, error) {
@@ -256,6 +277,195 @@ func (ctrl *User) SessionsRemove(ctx context.Context, r *request.UserSessionsRem
return
}
// Export exports users with optional role membership and related roles
//
// @note this is a temporary implementation; it will be reworked when we rework Envoy and related bits.
func (ctrl *User) Export(ctx context.Context, r *request.UserExport) (rsp interface{}, err error) {
// Users
uu, _, err := ctrl.user.Find(ctx, types.UserFilter{})
if err != nil {
return
}
// Roles
roleIndex := make(map[uint64]*types.Role)
roleResIndex := make(map[uint64]resource.Interface)
rr, _, err := ctrl.role.Find(ctx, types.RoleFilter{Paging: filter.Paging{Limit: 0}})
if err != nil {
return
}
for _, r := range rr {
roleIndex[r.ID] = r
}
// Membership
resources := make(resource.InterfaceSet, 0, len(uu))
var membership types.RoleMemberSet
for _, u := range uu {
usrRes := resource.NewUser(u)
if r.InclRoleMembership {
membership, err = ctrl.role.Membership(ctx, u.ID)
if err != nil {
return
}
aux := make(types.RoleSet, 0, 2)
for _, m := range membership {
if _, ok := roleResIndex[m.RoleID]; !ok {
roleResIndex[m.RoleID] = resource.NewRole(roleIndex[m.RoleID])
if r.InclRoles {
resources = append(resources, roleResIndex[m.RoleID])
}
}
aux = append(aux, roleIndex[m.RoleID])
}
usrRes.AddRoles(aux...)
}
resources = append(resources, usrRes)
}
// Encode
ye := yaml.NewYamlEncoder(&yaml.EncoderConfig{})
bld := envoy.NewBuilder(ye)
g, err := bld.Build(ctx, resources...)
if err != nil {
return nil, err
}
err = envoy.Encode(ctx, g, ye)
if err != nil {
return
}
// make archive
buf := bytes.NewBuffer(nil)
w := zip.NewWriter(buf)
var (
f io.Writer
bb []byte
)
for _, s := range ye.Stream() {
// @todo generalize when needed
f, err = w.Create(fmt.Sprintf("%s.yaml", s.Resource))
if err != nil {
return
}
bb, err = ioutil.ReadAll(s.Source)
if err != nil {
return
}
_, err = f.Write(bb)
if err != nil {
return
}
}
err = w.Close()
if err != nil {
return
}
return ctrl.serve(ctx, fmt.Sprintf("%s.zip", r.Filename), bytes.NewReader(buf.Bytes()), nil)
}
// Import imports users with optional role membership and related roles
//
// @note this is a temporary implementation; it will be reworked when we rework Envoy and related bits.
func (ctrl *User) Import(ctx context.Context, r *request.UserImport) (rsp interface{}, err error) {
// AC
// @todo refactor when we refactor this part of the sys
if !ctrl.userAc.CanCreateUser(ctx) {
err = fmt.Errorf("cannot import users: not allowed to create users")
return
}
if !ctrl.roleAc.CanCreateRole(ctx) {
err = fmt.Errorf("cannot import users: not allowed to create roles")
return
}
// Parse inputs
f, err := r.Upload.Open()
if err != nil {
return
}
defer f.Close()
mt, err := mimetype.DetectReader(f)
if err != nil {
return
}
_, err = f.Seek(0, 0)
if err != nil {
return
}
if !mt.Is("application/zip") {
err = fmt.Errorf("cannot import users: unsupported file format")
return
}
// un-archive
archive, err := zip.NewReader(f, r.Upload.Size)
if err != nil {
return
}
// decode with Envoy
yd := yaml.Decoder()
nn := make([]resource.Interface, 0, 10)
var mm []resource.Interface
for _, archF := range archive.File {
if archF.FileInfo().IsDir() {
continue
}
var f io.ReadCloser
f, err = archF.Open()
if err != nil {
return
}
defer f.Close()
mm, err = yd.Decode(ctx, f, nil)
if err != nil {
return
}
nn = append(nn, mm...)
}
// Validate
for _, n := range nn {
switch n.ResourceType() {
case types.UserResourceType,
types.RoleResourceType:
continue
default:
err = fmt.Errorf("cannot import users: invalid resource provided: %s", n.ResourceType())
return
}
}
se := envoyStore.NewStoreEncoder(service.DefaultStore, &envoyStore.EncoderConfig{
OnExisting: resource.Skip,
})
bld := envoy.NewBuilder(se)
g, err := bld.Build(ctx, nn...)
if err != nil {
return
}
err = envoy.Encode(ctx, g, se)
return api.OK(), err
}
func (ctrl User) makeFilterPayload(ctx context.Context, uu types.UserSet, f types.UserFilter, err error) (*userSetPayload, error) {
if err != nil {
return nil, err
@@ -267,3 +477,15 @@ func (ctrl User) makeFilterPayload(ctx context.Context, uu types.UserSet, f type
return &userSetPayload{Filter: f, Set: uu}, nil
}
func (ctrl User) serve(ctx context.Context, fn string, archive io.ReadSeeker, err error) (interface{}, error) {
if err != nil {
return nil, err
}
return func(w http.ResponseWriter, req *http.Request) {
w.Header().Add("Content-Disposition", "attachment; filename="+fn)
http.ServeContent(w, req, fn, time.Now(), archive)
}, nil
}