Refactor synthetic user generation

server/app/cli.go

@@ -134,7 +134,7 @@ func (app *CortezaApp) InitCLI() {
 		cli.EnvCommand(),
 		cli.VersionCommand(),
 		seeder.BaseCommand(
-			systemCommands.SeedUsers(ctx, app),
+			systemCommands.GenerateSyntheticUsers(ctx, app),
 			composeCommands.SeedRecords(ctx, app),
 		),
 	)

server/pkg/seeder/faker.go

@@ -43,7 +43,7 @@ func (f faker) fakeValueByName(name string) (val string, ok bool) {
 	// Ensure randomization on initial
 	f.seed()
 
-	// Generate & return value from mapped methods
+	// Generator & return value from mapped methods
 	method, ok := f.methods[name]
 	if ok {
 		return method(), ok
@@ -54,7 +54,7 @@ func (f faker) fakeValueByName(name string) (val string, ok bool) {
 // @todo: currently its dependent on predefined methods but custom kind fake gen can be improved!
 // generateValue generate value based on name or given type
 func (f faker) fakeValue(name, kind string, opt valueOptions) (val string, err error) {
-	// Generate & return value from mapped methods
+	// Generator & return value from mapped methods
 	val, ok := f.fakeValueByName(name)
 	if ok {
 		return

server/pkg/seeder/generator.go

@@ -0,0 +1,51 @@
+package seeder
+
+import (
+	"context"
+	"fmt"
+	"github.com/cortezaproject/corteza/server/pkg/errors"
+)
+
+type (
+	fn func(ctx context.Context) error
+)
+
+const (
+	maxRetry = 10
+)
+
+// Generator function generates synthetic data by calling generator function
+//
+// It will retry on error up to maxRetry times
+func Generator(ctx context.Context, generator fn, total uint) (err error) {
+	var (
+		retry uint
+	)
+
+	if generator == nil {
+		return fmt.Errorf("generator is nil")
+	}
+
+	for total > 0 {
+		if retry > maxRetry {
+			return fmt.Errorf("max retry count (%d) reached", maxRetry)
+		}
+
+		err = generator(ctx)
+
+		if err == nil {
+			retry = 0
+			total--
+			continue
+		}
+
+		if errors.IsDuplicateData(err) {
+			retry++
+			continue
+		}
+
+		return err
+	}
+
+	return nil
+}

server/pkg/seeder/generator_test.go

@@ -0,0 +1,45 @@
+package seeder
+
+import (
+	"context"
+	"github.com/cortezaproject/corteza/server/pkg/errors"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestGenerate(t *testing.T) {
+	var (
+		req = require.New(t)
+	)
+
+	req.Error(Generator(
+		context.Background(),
+		nil,
+		0,
+	), "generator is nil")
+
+	count := 0
+	req.NoError(Generator(
+		context.Background(),
+		func(ctx context.Context) error {
+			count++
+			return nil
+		},
+		5,
+	))
+	req.Equal(5, count)
+
+	count = 0
+	req.NoError(Generator(
+		context.Background(),
+		func(ctx context.Context) error {
+			count++
+			if count%2 == 0 {
+				return nil
+			}
+			return errors.DuplicateData("foo")
+		},
+		5,
+	))
+	req.Equal(10, count)
+}

server/system/commands/generator.go

@@ -0,0 +1,64 @@
+package commands
+
+import (
+	"context"
+	"github.com/brianvoe/gofakeit/v6"
+	"github.com/cortezaproject/corteza/server/pkg/auth"
+	"github.com/cortezaproject/corteza/server/pkg/cli"
+	"github.com/cortezaproject/corteza/server/system/service"
+	"time"
+
+	"github.com/spf13/cobra"
+)
+
+func GenerateSyntheticUsers(ctx context.Context, app serviceInitializer) *cobra.Command {
+	var (
+		total uint
+		faker = gofakeit.NewCrypto()
+
+		base = &cobra.Command{
+			Use: "users",
+			PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+				return app.InitServices(ctx)
+			},
+		}
+
+		gen = &cobra.Command{
+			Use:     "generate",
+			Aliases: []string{"gen"},
+			Short:   "Generate synthetic users",
+			Run: func(cmd *cobra.Command, args []string) {
+				cmd.Printf("Generating %d users ...", total)
+				bm := time.Now()
+
+				ctx = auth.SetIdentityToContext(ctx, auth.ServiceUser())
+				cli.HandleError(service.DefaultUser.CreateSynthetic(ctx, faker, total))
+
+				cmd.Printf("done in %s", time.Since(bm).Round(time.Millisecond))
+				cmd.Println()
+			},
+		}
+
+		rem = &cobra.Command{
+			Use:     "remove",
+			Aliases: []string{"rm", "d", "delete", "del"},
+			Short:   "Remove synthetic users",
+			Run: func(cmd *cobra.Command, args []string) {
+				cmd.Printf("Removing all synthetic users ...")
+				bm := time.Now()
+
+				ctx = auth.SetIdentityToContext(ctx, auth.ServiceUser())
+				cli.HandleError(service.DefaultUser.RemoveSynthetic(ctx))
+
+				cmd.Printf("done in %s", time.Since(bm).Round(time.Millisecond))
+				cmd.Println()
+			},
+		}
+	)
+
+	gen.Flags().UintVarP(&total, "total", "t", 1, "Number of synthetic users generated")
+
+	base.AddCommand(gen, rem)
+
+	return base
+}

server/system/commands/seeder.go

@@ -1,72 +0,0 @@
-package commands
-
-import (
-	"context"
-	"github.com/cortezaproject/corteza/server/pkg/cli"
-	"github.com/cortezaproject/corteza/server/pkg/dal"
-	"github.com/cortezaproject/corteza/server/pkg/seeder"
-
-	"github.com/spf13/cobra"
-)
-
-type (
-	seederService interface {
-		CreateUser(seeder.Params) ([]uint64, error)
-		CreateRecord(seeder.RecordParams) ([]uint64, error)
-		DeleteAllUser() error
-	}
-)
-
-var (
-	svc seederService
-)
-
-func SeedUsers(ctx context.Context, app serviceInitializer) (cmd *cobra.Command) {
-	var (
-		limit int
-	)
-	cmd = &cobra.Command{
-		Use:   "users",
-		Short: "Seed users",
-		Args:  cobra.MaximumNArgs(0),
-
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
-			if err = app.InitServices(cli.Context()); err != nil {
-				return err
-			}
-			svc = seeder.Seeder(ctx, seeder.DefaultStore, dal.Service(), seeder.Faker())
-			return nil
-		},
-	}
-
-	cmd.AddCommand(
-		&cobra.Command{
-			Use:   "create",
-			Short: "Create users",
-			Args:  cobra.MaximumNArgs(0),
-			Run: func(cmd *cobra.Command, args []string) {
-				userIDs, err := svc.CreateUser(seeder.Params{Limit: limit})
-				cli.HandleError(err)
-
-				cmd.Printf("                     Created    %d    users", len(userIDs))
-				cmd.Println()
-			},
-		},
-		&cobra.Command{
-			Use:   "delete",
-			Short: "Delete users",
-			Args:  cobra.MaximumNArgs(0),
-			Run: func(cmd *cobra.Command, args []string) {
-				cli.HandleError(svc.DeleteAllUser())
-
-				cmd.Println("                     Deleted    all    users")
-
-			},
-		},
-	)
-
-	cmdCreate := cmd.Commands()[0]
-	cmdCreate.Flags().IntVarP(&limit, "limit", "l", 1, "How many users to be created")
-
-	return cmd
-}

server/system/service/user.go

@@ -7,6 +7,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/cortezaproject/corteza/server/pkg/actionlog"
 	internalAuth "github.com/cortezaproject/corteza/server/pkg/auth"
@@ -46,6 +47,12 @@ type (
 		preloaded map[string]*types.User
 	}
 
+	synteticUserDataGen interface {
+		Name() string
+		Username() string
+		Number(int, int) int
+	}
+
 	UserOptions struct {
 		LimitUsers int
 	}
@@ -813,6 +820,111 @@ func (svc user) checkLimits(ctx context.Context) error {
 	return nil
 }
 
+// CreateSynthetic generates, saves and returns new user
+//
+// Generated users will have their handles prefixed with "synthetic_" and email domain "synthetic.tld"
+//
+// Function checks if user can create users but avoids all other checks (besides unique value)
+func (svc user) CreateSynthetic(ctx context.Context, src synteticUserDataGen, total uint) error {
+	if !svc.ac.CanCreateUser(ctx) {
+		return UserErrNotAllowedToCreate()
+	}
+
+	const maxRetries = 10
+
+	return store.Tx(ctx, svc.store, func(ctx context.Context, s store.Storer) (err error) {
+		var retry uint
+		for total > 0 || maxRetries < retry {
+			// even with pre-check for unique users this one
+			// still returns not unique error from time to time ?!
+			err = store.CreateUser(ctx, s, syntheticUser(src))
+			if errors.IsDuplicateData(err) {
+				retry++
+				continue
+			}
+
+			if err != nil {
+				return
+			}
+
+			retry = 0
+			total--
+		}
+
+		return
+	})
+}
+
+func syntheticUser(src synteticUserDataGen) (r *types.User) {
+	r = &types.User{
+		ID:             nextID(),
+		Kind:           types.NormalUser,
+		Name:           src.Name(),
+		Handle:         "synthetic_" + src.Username(),
+		EmailConfirmed: src.Number(0, 1) > 0,
+
+		// Make sure all users are created in the past
+		CreatedAt: time.Now().Add(time.Hour * time.Duration(src.Number(100000, 1000000)*-1)),
+	}
+
+	r.Email = strings.ToLower(strings.ReplaceAll(r.Name, " ", ".")) + "@synthetic.tld"
+
+	if src.Number(0, 1) > 0 {
+		aux := time.Now().Add(time.Hour * time.Duration(src.Number(100, 100000)*-1))
+		r.UpdatedAt = &aux
+	}
+
+	return
+}
+
+// CreateSynthetic generates, saves and returns new user
+//
+// Generated users will have their handles prefixed with "synthetic_" and email domain "synthetic.tld"
+func (svc user) RemoveSynthetic(ctx context.Context) error {
+	// not a mistake, we do not need or want to check if user can be deleted
+	if !svc.ac.CanCreateUser(ctx) {
+		return UserErrNotAllowedToCreate()
+	}
+
+	var (
+		f  = types.UserFilter{Query: "@synthetic.tld"}
+		uu types.UserSet
+	)
+
+	f.Limit = 1000
+
+	// @todo this should be optimized by using store.DeleteUserByFilter
+	return store.Tx(ctx, svc.store, func(ctx context.Context, s store.Storer) (err error) {
+		for {
+			uu, _, err = store.SearchUsers(ctx, s, f)
+			if len(uu) == 0 || err != nil {
+				// when nothing is fetch or error returned
+				// break out of the loop
+				return
+			}
+
+			for _, u := range uu {
+				// check if handle starts with synthetic_
+				if !strings.HasPrefix(u.Handle, "synthetic_") {
+					continue
+				}
+
+				// check if email ends with synthetic.tld
+				if !strings.HasSuffix(u.Email, "@synthetic.tld") {
+					continue
+				}
+
+				if err = store.DeleteUser(ctx, s, u); err != nil {
+					return
+				}
+			}
+		}
+
+		return
+	})
+
+}
+
 func loadUser(ctx context.Context, s store.Users, ID uint64) (res *types.User, err error) {
 	if ID == 0 {
 		return nil, UserErrInvalidID()
@@ -840,6 +952,8 @@ func uniqueUserCheck(ctx context.Context, s store.Storer, u *types.User) (err er
 			Suspended: filter.StateInclusive,
 		}
 
+		f.Limit = 1
+
 		switch field {
 		case "email":
 			if u.Email == "" {