DeforaNetworks/corteza
DeforaNetworks/corteza
3
0
Fork 0
Code

Refactor synthetic compose record generation

Browse Source
This commit is contained in:
Denis Arh 2022-11-17 07:05:40 +01:00
parent 211a6da71c
commit a48dfc0261
3 changed files with 383 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

135
server/compose/commands/generator.go Normal file
View File

@ -0,0 +1,135 @@
package commands
import (
"context"
"fmt"
"github.com/brianvoe/gofakeit/v6"
"github.com/cortezaproject/corteza/server/compose/service"
"github.com/cortezaproject/corteza/server/compose/types"
"github.com/cortezaproject/corteza/server/pkg/auth"
"github.com/cortezaproject/corteza/server/pkg/cli"
"github.com/cortezaproject/corteza/server/pkg/seeder"
"time"
"github.com/spf13/cobra"
)
type (
serviceInitializer interface {
InitServices(ctx context.Context) error
}
seederService interface {
CreateUser(seeder.Params) ([]uint64, error)
CreateRecord(seeder.RecordParams) ([]uint64, error)
DeleteAllUser() error
DeleteAllRecord(*types.Module) error
DeleteAll(*seeder.RecordParams) (err error)
}
)
var (
svc seederService
)
func SeedRecords(ctx context.Context, app serviceInitializer) (cmd *cobra.Command) {
var (
total uint
faker = gofakeit.NewCrypto()
namespace string
module string
base = &cobra.Command{
Use: "records",
Aliases: []string{"rec"},
PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
if err = app.InitServices(ctx); err != nil {
return
}
if err = service.DefaultModule.ReloadDALModels(ctx); err != nil {
return
}
return
},
}
gen = &cobra.Command{
Use: "generate",
Aliases: []string{"gen"},
Short: "Create synthetic records",
Args: cobra.MaximumNArgs(0),
Run: func(cmd *cobra.Command, args []string) {
if len(namespace) == 0 || len(module) == 0 {
cli.HandleError(fmt.Errorf("specifiy ID and handle for both, module and namespace"))
}
ctx = auth.SetIdentityToContext(ctx, auth.ServiceUser())
ns, mod, err := resolveModule(ctx, service.DefaultNamespace, service.DefaultModule, namespace, module)
cli.HandleError(err)
_ = faker
_ = ns
_ = mod
_ = err
cmd.Printf("Generating %d compose records (module: %s) ...", total, mod.Name)
bm := time.Now()
cli.HandleError(service.DefaultRecord.CreateSynthetic(ctx, faker, mod, total))
cmd.Printf("done in %s", time.Since(bm).Round(time.Millisecond))
cmd.Println()
},
}
rem = &cobra.Command{
Use: "remove",
Aliases: []string{"rm", "d", "delete", "del"},
Short: "Remove synthetic records",
Args: cobra.MaximumNArgs(0),
Run: func(cmd *cobra.Command, args []string) {
if len(namespace) == 0 || len(module) == 0 {
cli.HandleError(fmt.Errorf("specifiy ID and handle for both, module and namespace"))
}
ctx = auth.SetIdentityToContext(ctx, auth.ServiceUser())
ns, mod, err := resolveModule(ctx, service.DefaultNamespace, service.DefaultModule, namespace, module)
cli.HandleError(err)
_ = ns
_ = mod
_ = err
cmd.Printf("Removing all synthetic compose records (module: %s) ...", mod.Name)
bm := time.Now()
cli.HandleError(service.DefaultRecord.RemoveSynthetic(ctx, mod))
cmd.Printf("done in %s", time.Since(bm).Round(time.Millisecond))
cmd.Println()
},
}
)
for _, cmd = range []*cobra.Command{gen, rem} {
cmd.Flags().StringVarP(&namespace, "namespace", "n", "", "namespace IS or handle for recode generation")
cmd.Flags().StringVarP(&module, "module", "m", "", "module IS or handle for recode generation")
}
gen.Flags().UintVarP(&total, "total", "t", 1, "Number of synthetic records generated")
base.AddCommand(gen, rem)
return base
}
func resolveModule(ctx context.Context, nsSvc service.NamespaceService, modSvc service.ModuleService, nsIdent, modIdent string) (ns *types.Namespace, mod *types.Module, err error) {
if ns, err = nsSvc.FindByAny(ctx, nsIdent); err != nil {
return
}
mod, err = modSvc.FindByAny(ctx, ns.ID, modIdent)
return
}

96
server/compose/commands/seeder.go
View File

@ -1,96 +0,0 @@
package commands
import (
"context"
"github.com/cortezaproject/corteza/server/compose/service"
"github.com/cortezaproject/corteza/server/compose/types"
"github.com/cortezaproject/corteza/server/pkg/cli"
"github.com/cortezaproject/corteza/server/pkg/dal"
"github.com/cortezaproject/corteza/server/pkg/seeder"
"github.com/spf13/cobra"
)
type (
serviceInitializer interface {
InitServices(ctx context.Context) error
}
seederService interface {
CreateUser(seeder.Params) ([]uint64, error)
CreateRecord(seeder.RecordParams) ([]uint64, error)
DeleteAllUser() error
DeleteAllRecord(*types.Module) error
DeleteAll(*seeder.RecordParams) (err error)
}
)
var (
svc seederService
)
func SeedRecords(ctx context.Context, app serviceInitializer) (cmd *cobra.Command) {
var (
namespaceID int
moduleID int
params = seeder.RecordParams{}
)
cmd = &cobra.Command{
Use: "records",
Short: "Seed records",
PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
if err = app.InitServices(cli.Context()); err != nil {
return err
}
cmd.Println("compose:PersistentPreRunE")
if err = service.Activate(ctx); err != nil {
return err
}
svc = seeder.Seeder(ctx, seeder.DefaultStore, dal.Service(), seeder.Faker())
return nil
},
}
cmd.AddCommand(
&cobra.Command{
Use: "create",
Short: "Create users",
Args: cobra.MaximumNArgs(0),
Run: func(cmd *cobra.Command, args []string) {
params.NamespaceID = uint64(namespaceID)
params.ModuleID = uint64(moduleID)
userIDs, err := svc.CreateRecord(params)
cli.HandleError(err)
cmd.Printf(" Created %d records", len(userIDs))
cmd.Println()
},
},
&cobra.Command{
Use: "delete",
Short: "Delete records",
Args: cobra.MaximumNArgs(0),
Run: func(cmd *cobra.Command, args []string) {
cli.HandleError(svc.DeleteAllRecord(&types.Module{}))
cmd.Println(" Deleted all records")
},
},
)
cmdCreate := cmd.Commands()[0]
cmdCreate.Flags().IntVarP(&params.Limit, "limit", "l", 1, "How many records to be created")
// @todo: Can be improved by adding one flag which accept string(handle) or id(int) for namespace and module
cmdCreate.Flags().IntVarP(&namespaceID, "namespace-id", "n", 0, "namespace id for recode creation")
cmdCreate.Flags().StringVarP(&params.NamespaceHandle, "namespace-handle", "a", "", "namespace handle for recode creation")
cmdCreate.Flags().IntVarP(&moduleID, "module-id", "m", 0, "module id for recode creation")
cmdCreate.Flags().StringVarP(&params.ModuleHandle, "module-handle", "b", "", "module handle for recode creation")
return cmd
}

248
server/compose/service/record.go
View File

@ -28,6 +28,7 @@ import (
"github.com/cortezaproject/corteza/server/pkg/errors"
"github.com/cortezaproject/corteza/server/pkg/eventbus"
"github.com/cortezaproject/corteza/server/store"
systemTypes "github.com/cortezaproject/corteza/server/system/types"
)
const (
@ -135,6 +136,19 @@ type (
TriggerScript(ctx context.Context, namespaceID, moduleID, recordID uint64, rvs types.RecordValueSet, script string) (*types.Module, *types.Record, error)
}
synteticRecordDataGen interface {
LoremIpsumSentence(int) string
IntRange(int, int) int
DigitN(uint) string
Email() string
URL() string
Date() time.Time
Number(int, int) int
Street() string
City() string
Country() string
}
recordImportSession struct {
Name string `json:"-"`
SessionID uint64 `json:"sessionID,string"`
@ -1708,6 +1722,240 @@ func (svc record) DupDetection(ctx context.Context, m *types.Module, rec *types.
return
}
func (svc record) CreateSynthetic(ctx context.Context, src synteticRecordDataGen, mod *types.Module, total uint) error {
if !svc.ac.CanCreateRecordOnModule(ctx, mod) {
return RecordErrNotAllowedToCreate()
}
const (
maxRetries = 10
preloadUsers = 100
preloadRecords = 100
)
return store.Tx(ctx, svc.store, func(ctx context.Context, s store.Storer) (err error) {
var (
retry uint
synth *types.Record
uu systemTypes.UserSet
rr types.RecordSet
refUsers []uint64
refRecords map[uint64][]uint64
)
refRecords = make(map[uint64][]uint64)
for _, f := range mod.Fields {
switch strings.ToLower(f.Kind) {
case "user":
if len(refUsers) == 0 {
// preload 100 users
flt := systemTypes.UserFilter{}
flt.Limit = preloadUsers
if uu, _, err = store.SearchUsers(ctx, s, flt); err != nil {
return
}
refUsers = uu.IDs()
}
case "record":
refModID := f.Options.Uint64("moduleID")
if len(refRecords[refModID]) == 0 {
// preload 100 records
flt := types.RecordFilter{}
flt.Limit = preloadRecords
if rr, _, err = dalutils.ComposeRecordsList(ctx, svc.dal, mod, flt); err != nil {
return
}
refRecords[refModID] = rr.IDs()
}
}
}
for total > 0 || maxRetries < retry {
synth = syntheticRecord(src, refUsers, refRecords, mod)
if err = RecordValueSanitization(mod, synth.Values); err != nil {
return
}
if err = dalutils.ComposeRecordCreate(ctx, svc.dal, mod, synth); err != nil {
return
}
total--
}
return
})
}
func syntheticRecord(src synteticRecordDataGen, userRefs []uint64, recRefs map[uint64][]uint64, mod *types.Module) (r *types.Record) {
var (
randUser = func() uint64 {
if len(userRefs) == 0 {
return 0
}
return userRefs[src.IntRange(0, len(userRefs)-1)]
}
)
r = &types.Record{
ID: nextID(),
ModuleID: mod.ID,
NamespaceID: mod.NamespaceID,
// Make sure all users are created in the past
CreatedAt: time.Now().Add(time.Hour * time.Duration(src.Number(100000, 1000000)*-1)),
CreatedBy: randUser(),
Values: syntheticRecordValues(src, userRefs, recRefs, mod),
Meta: map[string]any{"synthetic": true},
}
if src.Number(0, 1) > 0 {
aux := time.Now().Add(time.Hour * time.Duration(src.Number(100, 100000)*-1))
r.UpdatedAt = &aux
r.UpdatedBy = randUser()
}
return
}
func syntheticRecordValues(src synteticRecordDataGen, userRefs []uint64, recRefs map[uint64][]uint64, mod *types.Module) (rvs types.RecordValueSet) {
var (
fname string
valuesPerField uint
val *types.RecordValue
pickRandomID = func(vals []uint64) string {
if len(vals) == 0 {
return ""
}
// picks random value from vals
return strconv.FormatUint(vals[src.IntRange(0, len(vals)-1)], 10)
}
)
fields:
for _, f := range mod.Fields {
valuesPerField = 1
fname = strings.ToLower(f.Name)
if f.Multi {
valuesPerField = uint(src.Number(0, 10))
}
for p := uint(0); p < valuesPerField; p++ {
val = &types.RecordValue{
Name: f.Name,
Place: p,
}
switch strings.ToLower(f.Kind) {
case "bool":
val.Value = fmt.Sprintf("%d", src.Number(0, 1))
case "datetime":
// @todo respect present/past
// @todo respect date/time
val.Value = src.Date().Format(time.RFC3339)
case "email":
val.Value = src.Email()
case "file":
continue fields
case "string":
fname = strings.ToLower(f.Name)
switch {
case strings.Contains(fname, "name"),
strings.Contains(fname, "title"),
strings.Contains(fname, "label"):
val.Value = src.LoremIpsumSentence(src.Number(3, 5))
case strings.Contains(fname, "street"):
val.Value = src.Street()
case strings.Contains(fname, "city"):
val.Value = src.Street()
case strings.Contains(fname, "country"):
val.Value = src.Country()
case strings.Contains(fname, "desc"),
strings.Contains(fname, "note"):
val.Value = src.LoremIpsumSentence(src.Number(10, 100))
default:
val.Value = src.LoremIpsumSentence(src.Number(4, 40))
}
case "number":
val.Value = src.DigitN(5)
case "record":
refModID := f.Options.Uint64("moduleID")
if refModID == 0 && len(recRefs[refModID]) == 0 {
continue fields
}
val.Value = pickRandomID(recRefs[refModID])
case "select":
//val.Value = src.Select(f.Options)
continue fields
case "url":
val.Value = src.URL()
case "user":
val.Value = pickRandomID(userRefs)
default:
continue fields
}
rvs = append(rvs, val)
}
}
return
}
func (svc record) RemoveSynthetic(ctx context.Context, mod *types.Module) (err error) {
// not a mistake, we do not need or want to check if user can be deleted
if !svc.ac.CanCreateRecordOnModule(ctx, mod) {
return RecordErrNotAllowedToCreate()
}
var (
f = types.RecordFilter{Meta: map[string]any{"synthetic": true}}
rr []*types.Record
)
f.Limit = 1000
for {
rr, _, err = dalutils.ComposeRecordsList(ctx, svc.dal, mod, f)
if err != nil || len(rr) == 0 {
return
}
if err = dalutils.ComposeRecordDelete(ctx, svc.dal, mod, rr...); err != nil {
return
}
}
}
func ComposeRecordFilterChecker(ctx context.Context, ac recordAccessController, m *types.Module) func(*types.Record) (bool, error) {
return func(rec *types.Record) (bool, error) {
// Setting module right before we do access control