From 211a6da71c081d41b55c8be56b550493d39e6d40 Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Thu, 17 Nov 2022 07:05:40 +0100 Subject: [PATCH] Refactor synthetic user generation --- server/app/cli.go | 2 +- server/pkg/seeder/faker.go | 4 +- server/pkg/seeder/generator.go | 51 +++++++++++++ server/pkg/seeder/generator_test.go | 45 +++++++++++ server/system/commands/generator.go | 64 ++++++++++++++++ server/system/commands/seeder.go | 72 ------------------ server/system/service/user.go | 114 ++++++++++++++++++++++++++++ 7 files changed, 277 insertions(+), 75 deletions(-) create mode 100644 server/pkg/seeder/generator.go create mode 100644 server/pkg/seeder/generator_test.go create mode 100644 server/system/commands/generator.go delete mode 100644 server/system/commands/seeder.go diff --git a/server/app/cli.go b/server/app/cli.go index 0db86b67d..860b3917b 100644 --- a/server/app/cli.go +++ b/server/app/cli.go @@ -134,7 +134,7 @@ func (app *CortezaApp) InitCLI() { cli.EnvCommand(), cli.VersionCommand(), seeder.BaseCommand( - systemCommands.SeedUsers(ctx, app), + systemCommands.GenerateSyntheticUsers(ctx, app), composeCommands.SeedRecords(ctx, app), ), ) diff --git a/server/pkg/seeder/faker.go b/server/pkg/seeder/faker.go index 7f6821658..adf139afe 100644 --- a/server/pkg/seeder/faker.go +++ b/server/pkg/seeder/faker.go @@ -43,7 +43,7 @@ func (f faker) fakeValueByName(name string) (val string, ok bool) { // Ensure randomization on initial f.seed() - // Generate & return value from mapped methods + // Generator & return value from mapped methods method, ok := f.methods[name] if ok { return method(), ok @@ -54,7 +54,7 @@ func (f faker) fakeValueByName(name string) (val string, ok bool) { // @todo: currently its dependent on predefined methods but custom kind fake gen can be improved! // generateValue generate value based on name or given type func (f faker) fakeValue(name, kind string, opt valueOptions) (val string, err error) { - // Generate & return value from mapped methods + // Generator & return value from mapped methods val, ok := f.fakeValueByName(name) if ok { return diff --git a/server/pkg/seeder/generator.go b/server/pkg/seeder/generator.go new file mode 100644 index 000000000..d2ba2bf8a --- /dev/null +++ b/server/pkg/seeder/generator.go @@ -0,0 +1,51 @@ +package seeder + +import ( + "context" + "fmt" + "github.com/cortezaproject/corteza/server/pkg/errors" +) + +type ( + fn func(ctx context.Context) error +) + +const ( + maxRetry = 10 +) + +// Generator function generates synthetic data by calling generator function +// +// It will retry on error up to maxRetry times +func Generator(ctx context.Context, generator fn, total uint) (err error) { + var ( + retry uint + ) + + if generator == nil { + return fmt.Errorf("generator is nil") + } + + for total > 0 { + if retry > maxRetry { + return fmt.Errorf("max retry count (%d) reached", maxRetry) + } + + err = generator(ctx) + + if err == nil { + retry = 0 + total-- + continue + } + + if errors.IsDuplicateData(err) { + retry++ + continue + } + + return err + } + + return nil +} diff --git a/server/pkg/seeder/generator_test.go b/server/pkg/seeder/generator_test.go new file mode 100644 index 000000000..15b776a1c --- /dev/null +++ b/server/pkg/seeder/generator_test.go @@ -0,0 +1,45 @@ +package seeder + +import ( + "context" + "github.com/cortezaproject/corteza/server/pkg/errors" + "github.com/stretchr/testify/require" + "testing" +) + +func TestGenerate(t *testing.T) { + var ( + req = require.New(t) + ) + + req.Error(Generator( + context.Background(), + nil, + 0, + ), "generator is nil") + + count := 0 + req.NoError(Generator( + context.Background(), + func(ctx context.Context) error { + count++ + return nil + }, + 5, + )) + req.Equal(5, count) + + count = 0 + req.NoError(Generator( + context.Background(), + func(ctx context.Context) error { + count++ + if count%2 == 0 { + return nil + } + return errors.DuplicateData("foo") + }, + 5, + )) + req.Equal(10, count) +} diff --git a/server/system/commands/generator.go b/server/system/commands/generator.go new file mode 100644 index 000000000..7747ebff9 --- /dev/null +++ b/server/system/commands/generator.go @@ -0,0 +1,64 @@ +package commands + +import ( + "context" + "github.com/brianvoe/gofakeit/v6" + "github.com/cortezaproject/corteza/server/pkg/auth" + "github.com/cortezaproject/corteza/server/pkg/cli" + "github.com/cortezaproject/corteza/server/system/service" + "time" + + "github.com/spf13/cobra" +) + +func GenerateSyntheticUsers(ctx context.Context, app serviceInitializer) *cobra.Command { + var ( + total uint + faker = gofakeit.NewCrypto() + + base = &cobra.Command{ + Use: "users", + PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { + return app.InitServices(ctx) + }, + } + + gen = &cobra.Command{ + Use: "generate", + Aliases: []string{"gen"}, + Short: "Generate synthetic users", + Run: func(cmd *cobra.Command, args []string) { + cmd.Printf("Generating %d users ...", total) + bm := time.Now() + + ctx = auth.SetIdentityToContext(ctx, auth.ServiceUser()) + cli.HandleError(service.DefaultUser.CreateSynthetic(ctx, faker, total)) + + cmd.Printf("done in %s", time.Since(bm).Round(time.Millisecond)) + cmd.Println() + }, + } + + rem = &cobra.Command{ + Use: "remove", + Aliases: []string{"rm", "d", "delete", "del"}, + Short: "Remove synthetic users", + Run: func(cmd *cobra.Command, args []string) { + cmd.Printf("Removing all synthetic users ...") + bm := time.Now() + + ctx = auth.SetIdentityToContext(ctx, auth.ServiceUser()) + cli.HandleError(service.DefaultUser.RemoveSynthetic(ctx)) + + cmd.Printf("done in %s", time.Since(bm).Round(time.Millisecond)) + cmd.Println() + }, + } + ) + + gen.Flags().UintVarP(&total, "total", "t", 1, "Number of synthetic users generated") + + base.AddCommand(gen, rem) + + return base +} diff --git a/server/system/commands/seeder.go b/server/system/commands/seeder.go deleted file mode 100644 index 46cc2f539..000000000 --- a/server/system/commands/seeder.go +++ /dev/null @@ -1,72 +0,0 @@ -package commands - -import ( - "context" - "github.com/cortezaproject/corteza/server/pkg/cli" - "github.com/cortezaproject/corteza/server/pkg/dal" - "github.com/cortezaproject/corteza/server/pkg/seeder" - - "github.com/spf13/cobra" -) - -type ( - seederService interface { - CreateUser(seeder.Params) ([]uint64, error) - CreateRecord(seeder.RecordParams) ([]uint64, error) - DeleteAllUser() error - } -) - -var ( - svc seederService -) - -func SeedUsers(ctx context.Context, app serviceInitializer) (cmd *cobra.Command) { - var ( - limit int - ) - cmd = &cobra.Command{ - Use: "users", - Short: "Seed users", - Args: cobra.MaximumNArgs(0), - - PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { - if err = app.InitServices(cli.Context()); err != nil { - return err - } - svc = seeder.Seeder(ctx, seeder.DefaultStore, dal.Service(), seeder.Faker()) - return nil - }, - } - - cmd.AddCommand( - &cobra.Command{ - Use: "create", - Short: "Create users", - Args: cobra.MaximumNArgs(0), - Run: func(cmd *cobra.Command, args []string) { - userIDs, err := svc.CreateUser(seeder.Params{Limit: limit}) - cli.HandleError(err) - - cmd.Printf(" Created %d users", len(userIDs)) - cmd.Println() - }, - }, - &cobra.Command{ - Use: "delete", - Short: "Delete users", - Args: cobra.MaximumNArgs(0), - Run: func(cmd *cobra.Command, args []string) { - cli.HandleError(svc.DeleteAllUser()) - - cmd.Println(" Deleted all users") - - }, - }, - ) - - cmdCreate := cmd.Commands()[0] - cmdCreate.Flags().IntVarP(&limit, "limit", "l", 1, "How many users to be created") - - return cmd -} diff --git a/server/system/service/user.go b/server/system/service/user.go index 88dfa9c01..d1af268b4 100644 --- a/server/system/service/user.go +++ b/server/system/service/user.go @@ -7,6 +7,7 @@ import ( "regexp" "strconv" "strings" + "time" "github.com/cortezaproject/corteza/server/pkg/actionlog" internalAuth "github.com/cortezaproject/corteza/server/pkg/auth" @@ -46,6 +47,12 @@ type ( preloaded map[string]*types.User } + synteticUserDataGen interface { + Name() string + Username() string + Number(int, int) int + } + UserOptions struct { LimitUsers int } @@ -813,6 +820,111 @@ func (svc user) checkLimits(ctx context.Context) error { return nil } +// CreateSynthetic generates, saves and returns new user +// +// Generated users will have their handles prefixed with "synthetic_" and email domain "synthetic.tld" +// +// Function checks if user can create users but avoids all other checks (besides unique value) +func (svc user) CreateSynthetic(ctx context.Context, src synteticUserDataGen, total uint) error { + if !svc.ac.CanCreateUser(ctx) { + return UserErrNotAllowedToCreate() + } + + const maxRetries = 10 + + return store.Tx(ctx, svc.store, func(ctx context.Context, s store.Storer) (err error) { + var retry uint + for total > 0 || maxRetries < retry { + // even with pre-check for unique users this one + // still returns not unique error from time to time ?! + err = store.CreateUser(ctx, s, syntheticUser(src)) + if errors.IsDuplicateData(err) { + retry++ + continue + } + + if err != nil { + return + } + + retry = 0 + total-- + } + + return + }) +} + +func syntheticUser(src synteticUserDataGen) (r *types.User) { + r = &types.User{ + ID: nextID(), + Kind: types.NormalUser, + Name: src.Name(), + Handle: "synthetic_" + src.Username(), + EmailConfirmed: src.Number(0, 1) > 0, + + // Make sure all users are created in the past + CreatedAt: time.Now().Add(time.Hour * time.Duration(src.Number(100000, 1000000)*-1)), + } + + r.Email = strings.ToLower(strings.ReplaceAll(r.Name, " ", ".")) + "@synthetic.tld" + + if src.Number(0, 1) > 0 { + aux := time.Now().Add(time.Hour * time.Duration(src.Number(100, 100000)*-1)) + r.UpdatedAt = &aux + } + + return +} + +// CreateSynthetic generates, saves and returns new user +// +// Generated users will have their handles prefixed with "synthetic_" and email domain "synthetic.tld" +func (svc user) RemoveSynthetic(ctx context.Context) error { + // not a mistake, we do not need or want to check if user can be deleted + if !svc.ac.CanCreateUser(ctx) { + return UserErrNotAllowedToCreate() + } + + var ( + f = types.UserFilter{Query: "@synthetic.tld"} + uu types.UserSet + ) + + f.Limit = 1000 + + // @todo this should be optimized by using store.DeleteUserByFilter + return store.Tx(ctx, svc.store, func(ctx context.Context, s store.Storer) (err error) { + for { + uu, _, err = store.SearchUsers(ctx, s, f) + if len(uu) == 0 || err != nil { + // when nothing is fetch or error returned + // break out of the loop + return + } + + for _, u := range uu { + // check if handle starts with synthetic_ + if !strings.HasPrefix(u.Handle, "synthetic_") { + continue + } + + // check if email ends with synthetic.tld + if !strings.HasSuffix(u.Email, "@synthetic.tld") { + continue + } + + if err = store.DeleteUser(ctx, s, u); err != nil { + return + } + } + } + + return + }) + +} + func loadUser(ctx context.Context, s store.Users, ID uint64) (res *types.User, err error) { if ID == 0 { return nil, UserErrInvalidID() @@ -840,6 +952,8 @@ func uniqueUserCheck(ctx context.Context, s store.Storer, u *types.User) (err er Suspended: filter.StateInclusive, } + f.Limit = 1 + switch field { case "email": if u.Email == "" {