diff --git a/server/compose/commands/generator.go b/server/compose/commands/generator.go new file mode 100644 index 000000000..cf56dd39a --- /dev/null +++ b/server/compose/commands/generator.go @@ -0,0 +1,135 @@ +package commands + +import ( + "context" + "fmt" + "github.com/brianvoe/gofakeit/v6" + "github.com/cortezaproject/corteza/server/compose/service" + "github.com/cortezaproject/corteza/server/compose/types" + "github.com/cortezaproject/corteza/server/pkg/auth" + "github.com/cortezaproject/corteza/server/pkg/cli" + "github.com/cortezaproject/corteza/server/pkg/seeder" + "time" + + "github.com/spf13/cobra" +) + +type ( + serviceInitializer interface { + InitServices(ctx context.Context) error + } + + seederService interface { + CreateUser(seeder.Params) ([]uint64, error) + CreateRecord(seeder.RecordParams) ([]uint64, error) + DeleteAllUser() error + DeleteAllRecord(*types.Module) error + DeleteAll(*seeder.RecordParams) (err error) + } +) + +var ( + svc seederService +) + +func SeedRecords(ctx context.Context, app serviceInitializer) (cmd *cobra.Command) { + var ( + total uint + faker = gofakeit.NewCrypto() + + namespace string + module string + + base = &cobra.Command{ + Use: "records", + Aliases: []string{"rec"}, + + PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { + if err = app.InitServices(ctx); err != nil { + return + } + + if err = service.DefaultModule.ReloadDALModels(ctx); err != nil { + return + } + + return + }, + } + + gen = &cobra.Command{ + Use: "generate", + Aliases: []string{"gen"}, + Short: "Create synthetic records", + Args: cobra.MaximumNArgs(0), + Run: func(cmd *cobra.Command, args []string) { + if len(namespace) == 0 || len(module) == 0 { + cli.HandleError(fmt.Errorf("specifiy ID and handle for both, module and namespace")) + } + + ctx = auth.SetIdentityToContext(ctx, auth.ServiceUser()) + ns, mod, err := resolveModule(ctx, service.DefaultNamespace, service.DefaultModule, namespace, module) + cli.HandleError(err) + + _ = faker + _ = ns + _ = mod + _ = err + cmd.Printf("Generating %d compose records (module: %s) ...", total, mod.Name) + bm := time.Now() + + cli.HandleError(service.DefaultRecord.CreateSynthetic(ctx, faker, mod, total)) + + cmd.Printf("done in %s", time.Since(bm).Round(time.Millisecond)) + cmd.Println() + }, + } + + rem = &cobra.Command{ + Use: "remove", + Aliases: []string{"rm", "d", "delete", "del"}, + Short: "Remove synthetic records", + Args: cobra.MaximumNArgs(0), + Run: func(cmd *cobra.Command, args []string) { + if len(namespace) == 0 || len(module) == 0 { + cli.HandleError(fmt.Errorf("specifiy ID and handle for both, module and namespace")) + } + + ctx = auth.SetIdentityToContext(ctx, auth.ServiceUser()) + ns, mod, err := resolveModule(ctx, service.DefaultNamespace, service.DefaultModule, namespace, module) + cli.HandleError(err) + + _ = ns + _ = mod + _ = err + + cmd.Printf("Removing all synthetic compose records (module: %s) ...", mod.Name) + bm := time.Now() + + cli.HandleError(service.DefaultRecord.RemoveSynthetic(ctx, mod)) + + cmd.Printf("done in %s", time.Since(bm).Round(time.Millisecond)) + cmd.Println() + }, + } + ) + + for _, cmd = range []*cobra.Command{gen, rem} { + cmd.Flags().StringVarP(&namespace, "namespace", "n", "", "namespace IS or handle for recode generation") + cmd.Flags().StringVarP(&module, "module", "m", "", "module IS or handle for recode generation") + } + + gen.Flags().UintVarP(&total, "total", "t", 1, "Number of synthetic records generated") + + base.AddCommand(gen, rem) + return base +} + +func resolveModule(ctx context.Context, nsSvc service.NamespaceService, modSvc service.ModuleService, nsIdent, modIdent string) (ns *types.Namespace, mod *types.Module, err error) { + if ns, err = nsSvc.FindByAny(ctx, nsIdent); err != nil { + return + } + + mod, err = modSvc.FindByAny(ctx, ns.ID, modIdent) + return +} diff --git a/server/compose/commands/seeder.go b/server/compose/commands/seeder.go deleted file mode 100644 index e46c07b7c..000000000 --- a/server/compose/commands/seeder.go +++ /dev/null @@ -1,96 +0,0 @@ -package commands - -import ( - "context" - "github.com/cortezaproject/corteza/server/compose/service" - "github.com/cortezaproject/corteza/server/compose/types" - "github.com/cortezaproject/corteza/server/pkg/cli" - "github.com/cortezaproject/corteza/server/pkg/dal" - "github.com/cortezaproject/corteza/server/pkg/seeder" - - "github.com/spf13/cobra" -) - -type ( - serviceInitializer interface { - InitServices(ctx context.Context) error - } - - seederService interface { - CreateUser(seeder.Params) ([]uint64, error) - CreateRecord(seeder.RecordParams) ([]uint64, error) - DeleteAllUser() error - DeleteAllRecord(*types.Module) error - DeleteAll(*seeder.RecordParams) (err error) - } -) - -var ( - svc seederService -) - -func SeedRecords(ctx context.Context, app serviceInitializer) (cmd *cobra.Command) { - var ( - namespaceID int - moduleID int - - params = seeder.RecordParams{} - ) - cmd = &cobra.Command{ - Use: "records", - Short: "Seed records", - - PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { - if err = app.InitServices(cli.Context()); err != nil { - return err - } - - cmd.Println("compose:PersistentPreRunE") - if err = service.Activate(ctx); err != nil { - return err - } - - svc = seeder.Seeder(ctx, seeder.DefaultStore, dal.Service(), seeder.Faker()) - return nil - }, - } - - cmd.AddCommand( - &cobra.Command{ - Use: "create", - Short: "Create users", - Args: cobra.MaximumNArgs(0), - Run: func(cmd *cobra.Command, args []string) { - params.NamespaceID = uint64(namespaceID) - params.ModuleID = uint64(moduleID) - - userIDs, err := svc.CreateRecord(params) - cli.HandleError(err) - - cmd.Printf(" Created %d records", len(userIDs)) - cmd.Println() - }, - }, - &cobra.Command{ - Use: "delete", - Short: "Delete records", - Args: cobra.MaximumNArgs(0), - Run: func(cmd *cobra.Command, args []string) { - cli.HandleError(svc.DeleteAllRecord(&types.Module{})) - - cmd.Println(" Deleted all records") - - }, - }, - ) - - cmdCreate := cmd.Commands()[0] - cmdCreate.Flags().IntVarP(¶ms.Limit, "limit", "l", 1, "How many records to be created") - // @todo: Can be improved by adding one flag which accept string(handle) or id(int) for namespace and module - cmdCreate.Flags().IntVarP(&namespaceID, "namespace-id", "n", 0, "namespace id for recode creation") - cmdCreate.Flags().StringVarP(¶ms.NamespaceHandle, "namespace-handle", "a", "", "namespace handle for recode creation") - cmdCreate.Flags().IntVarP(&moduleID, "module-id", "m", 0, "module id for recode creation") - cmdCreate.Flags().StringVarP(¶ms.ModuleHandle, "module-handle", "b", "", "module handle for recode creation") - - return cmd -} diff --git a/server/compose/service/record.go b/server/compose/service/record.go index 8db77cd88..fe1eca95c 100644 --- a/server/compose/service/record.go +++ b/server/compose/service/record.go @@ -28,6 +28,7 @@ import ( "github.com/cortezaproject/corteza/server/pkg/errors" "github.com/cortezaproject/corteza/server/pkg/eventbus" "github.com/cortezaproject/corteza/server/store" + systemTypes "github.com/cortezaproject/corteza/server/system/types" ) const ( @@ -135,6 +136,19 @@ type ( TriggerScript(ctx context.Context, namespaceID, moduleID, recordID uint64, rvs types.RecordValueSet, script string) (*types.Module, *types.Record, error) } + synteticRecordDataGen interface { + LoremIpsumSentence(int) string + IntRange(int, int) int + DigitN(uint) string + Email() string + URL() string + Date() time.Time + Number(int, int) int + Street() string + City() string + Country() string + } + recordImportSession struct { Name string `json:"-"` SessionID uint64 `json:"sessionID,string"` @@ -1708,6 +1722,240 @@ func (svc record) DupDetection(ctx context.Context, m *types.Module, rec *types. return } +func (svc record) CreateSynthetic(ctx context.Context, src synteticRecordDataGen, mod *types.Module, total uint) error { + if !svc.ac.CanCreateRecordOnModule(ctx, mod) { + return RecordErrNotAllowedToCreate() + } + + const ( + maxRetries = 10 + + preloadUsers = 100 + preloadRecords = 100 + ) + + return store.Tx(ctx, svc.store, func(ctx context.Context, s store.Storer) (err error) { + var ( + retry uint + synth *types.Record + + uu systemTypes.UserSet + rr types.RecordSet + + refUsers []uint64 + refRecords map[uint64][]uint64 + ) + + refRecords = make(map[uint64][]uint64) + + for _, f := range mod.Fields { + + switch strings.ToLower(f.Kind) { + case "user": + if len(refUsers) == 0 { + // preload 100 users + flt := systemTypes.UserFilter{} + flt.Limit = preloadUsers + if uu, _, err = store.SearchUsers(ctx, s, flt); err != nil { + return + } + refUsers = uu.IDs() + } + + case "record": + refModID := f.Options.Uint64("moduleID") + + if len(refRecords[refModID]) == 0 { + // preload 100 records + flt := types.RecordFilter{} + flt.Limit = preloadRecords + if rr, _, err = dalutils.ComposeRecordsList(ctx, svc.dal, mod, flt); err != nil { + return + } + + refRecords[refModID] = rr.IDs() + } + } + + } + + for total > 0 || maxRetries < retry { + synth = syntheticRecord(src, refUsers, refRecords, mod) + + if err = RecordValueSanitization(mod, synth.Values); err != nil { + return + } + + if err = dalutils.ComposeRecordCreate(ctx, svc.dal, mod, synth); err != nil { + return + } + + total-- + } + + return + }) +} + +func syntheticRecord(src synteticRecordDataGen, userRefs []uint64, recRefs map[uint64][]uint64, mod *types.Module) (r *types.Record) { + var ( + randUser = func() uint64 { + if len(userRefs) == 0 { + return 0 + } + + return userRefs[src.IntRange(0, len(userRefs)-1)] + } + ) + + r = &types.Record{ + ID: nextID(), + ModuleID: mod.ID, + NamespaceID: mod.NamespaceID, + + // Make sure all users are created in the past + CreatedAt: time.Now().Add(time.Hour * time.Duration(src.Number(100000, 1000000)*-1)), + CreatedBy: randUser(), + + Values: syntheticRecordValues(src, userRefs, recRefs, mod), + + Meta: map[string]any{"synthetic": true}, + } + + if src.Number(0, 1) > 0 { + aux := time.Now().Add(time.Hour * time.Duration(src.Number(100, 100000)*-1)) + r.UpdatedAt = &aux + r.UpdatedBy = randUser() + } + + return +} + +func syntheticRecordValues(src synteticRecordDataGen, userRefs []uint64, recRefs map[uint64][]uint64, mod *types.Module) (rvs types.RecordValueSet) { + var ( + fname string + valuesPerField uint + val *types.RecordValue + + pickRandomID = func(vals []uint64) string { + if len(vals) == 0 { + return "" + } + + // picks random value from vals + return strconv.FormatUint(vals[src.IntRange(0, len(vals)-1)], 10) + } + ) + +fields: + for _, f := range mod.Fields { + valuesPerField = 1 + fname = strings.ToLower(f.Name) + + if f.Multi { + valuesPerField = uint(src.Number(0, 10)) + } + + for p := uint(0); p < valuesPerField; p++ { + val = &types.RecordValue{ + Name: f.Name, + Place: p, + } + + switch strings.ToLower(f.Kind) { + + case "bool": + val.Value = fmt.Sprintf("%d", src.Number(0, 1)) + + case "datetime": + // @todo respect present/past + // @todo respect date/time + val.Value = src.Date().Format(time.RFC3339) + + case "email": + val.Value = src.Email() + + case "file": + continue fields + + case "string": + fname = strings.ToLower(f.Name) + switch { + case strings.Contains(fname, "name"), + strings.Contains(fname, "title"), + strings.Contains(fname, "label"): + val.Value = src.LoremIpsumSentence(src.Number(3, 5)) + case strings.Contains(fname, "street"): + val.Value = src.Street() + case strings.Contains(fname, "city"): + val.Value = src.Street() + case strings.Contains(fname, "country"): + val.Value = src.Country() + case strings.Contains(fname, "desc"), + strings.Contains(fname, "note"): + val.Value = src.LoremIpsumSentence(src.Number(10, 100)) + default: + val.Value = src.LoremIpsumSentence(src.Number(4, 40)) + } + + case "number": + val.Value = src.DigitN(5) + + case "record": + refModID := f.Options.Uint64("moduleID") + + if refModID == 0 && len(recRefs[refModID]) == 0 { + continue fields + } + + val.Value = pickRandomID(recRefs[refModID]) + + case "select": + //val.Value = src.Select(f.Options) + continue fields + + case "url": + val.Value = src.URL() + + case "user": + val.Value = pickRandomID(userRefs) + + default: + continue fields + } + + rvs = append(rvs, val) + } + } + + return +} + +func (svc record) RemoveSynthetic(ctx context.Context, mod *types.Module) (err error) { + // not a mistake, we do not need or want to check if user can be deleted + if !svc.ac.CanCreateRecordOnModule(ctx, mod) { + return RecordErrNotAllowedToCreate() + } + + var ( + f = types.RecordFilter{Meta: map[string]any{"synthetic": true}} + rr []*types.Record + ) + + f.Limit = 1000 + + for { + rr, _, err = dalutils.ComposeRecordsList(ctx, svc.dal, mod, f) + if err != nil || len(rr) == 0 { + return + } + + if err = dalutils.ComposeRecordDelete(ctx, svc.dal, mod, rr...); err != nil { + return + } + } +} + func ComposeRecordFilterChecker(ctx context.Context, ac recordAccessController, m *types.Module) func(*types.Record) (bool, error) { return func(rec *types.Record) (bool, error) { // Setting module right before we do access control