From 4d6cb13f70aab7e66b178bd7c5135668149f2f8f Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Mon, 6 Jan 2020 10:15:30 +0100 Subject: [PATCH] Add basic automation script exporting --- compose/app.go | 3 + pkg/automation/command.go | 253 +++++++++++++++++++++++++++ pkg/automation/scheduled.go | 4 +- pkg/automation/scheduled_test.go | 8 +- pkg/automation/script.go | 28 ++- pkg/automation/script_repository.go | 2 +- pkg/automation/service.go | 20 +-- pkg/automation/trigger.go | 2 +- pkg/automation/trigger_repository.go | 12 +- system/app.go | 3 + 10 files changed, 295 insertions(+), 40 deletions(-) create mode 100644 pkg/automation/command.go diff --git a/compose/app.go b/compose/app.go index 9a72009e5..832495009 100644 --- a/compose/app.go +++ b/compose/app.go @@ -2,6 +2,7 @@ package compose import ( "context" + "github.com/cortezaproject/corteza-server/pkg/automation" "github.com/go-chi/chi" _ "github.com/joho/godotenv/autoload" @@ -99,5 +100,7 @@ func (app *App) RegisterCliCommands(p *cobra.Command) { p.AddCommand( commands.Importer(), commands.Exporter(), + // temp command, will be removed in 2020.6 + automation.ScriptMigrator(SERVICE), ) } diff --git a/pkg/automation/command.go b/pkg/automation/command.go new file mode 100644 index 000000000..21263bb73 --- /dev/null +++ b/pkg/automation/command.go @@ -0,0 +1,253 @@ +package automation + +import ( + "fmt" + "github.com/Masterminds/squirrel" + "github.com/cortezaproject/corteza-server/pkg/cli" + "github.com/cortezaproject/corteza-server/pkg/rh" + "github.com/davecgh/go-spew/spew" + "github.com/spf13/cobra" + "github.com/titpetric/factory" + "os" + "path" + "regexp" + "strconv" + "strings" + "text/template" + "unicode" +) + +type ( + module struct { + ID uint64 + Handle string + Name string + } +) + +var ( + sanitizer = regexp.MustCompile(`[^a-zA-Z0-9_\-.]+`) + + modules []*module + + // Language=GoTemplate + scriptTemplateRaw string = `import trigger from '+Trigger' + +export default { + label: {{ quote $.Name }}, + desc: '...', + triggers: [ +{{- range $t := $.Triggers }} + // auto-migrated + // ID: {{ $t.ID }} + // Created: {{ $t.CreatedAt }} + // Updated: {{ $t.UpdatedAt }} +{{ if not $t.Enabled }}/* disabled {{ end }} + trigger + .on({{ quote $t.Event }}){{- if $.RunAs }} + .as({{ quote $t.RunAs }}){{ end }} + .for({{ quote $t.Resource }}) + {{- makeConditionFn $t -}} +{{ if not $t.Enabled }}*/{{ end }} +{{ end -}} + ], + + async handler ({ $namespace, $module, $record }, { log, ComposeUI, Compose }) { + {{ indent .Source 4 }} + } +} +` + + tpl *template.Template +) + +func init() { + var err error + tpl = template.New("").Funcs(map[string]interface{}{ + //"camelCase": camelCase, + //"makeEvents": makeEvents, + + "dump": func(s ...interface{}) string { + return spew.Sdump(s...) + }, + + "quote": func(s string) string { + return `'` + s + `'` + }, + + "makeConditionFn": func(t *Trigger) string { + isStdBeforeAfter := (strings.HasSuffix(t.Event, "Create") || + strings.HasSuffix(t.Event, "Update") || + strings.HasSuffix(t.Event, "Delete")) && + (strings.HasPrefix(t.Event, "before") || + strings.HasPrefix(t.Event, "after")) + + if t.Condition != "" { + if t.Resource == "compose:record" && (isStdBeforeAfter || t.Event == "manual") { + id, _ := strconv.ParseUint(t.Condition, 10, 64) + if id > 0 { + for _, m := range modules { + if m.ID == id { + cnd := m.Handle + if cnd == "" { + cnd = t.Condition + } + + return fmt.Sprintf("\n .where('module', '%s'), // module (%d) %s\n", cnd, m.ID, m.Name) + } + } + + return fmt.Sprintf("\n .where('module', '%s'), // module not found, could not translate ID to handle\n", t.Condition) + } + } else if t.Event == "deferred" { + return fmt.Sprintf("\n .where('timestamp', '%s'),\n", t.Condition) + } else if t.Event == "interval" { + return fmt.Sprintf("\n .where('interval', '%s'),\n", t.Condition) + } + return fmt.Sprintf(", // unresolvable condition - %s \n", t.Condition) + } + + return ",\n" + }, + + "makeEventFn": func(ev string) string { + tpl := ".%s('%s')" + + if strings.HasPrefix(ev, "before") { + return fmt.Sprintf(tpl, "before", strings.ToLower(ev)) + } + + if strings.HasPrefix(ev, "after") { + return fmt.Sprintf(tpl, "after", strings.ToLower(ev)) + } + + if ev == "deferred" { + ev = "timestamp" + } + + return fmt.Sprintf(tpl, "on", ev) + }, + + "indent": func(s string, spaces int) (o string) { + for _, l := range strings.Split(s, "\n") { + o = o + strings.Repeat(" ", spaces) + strings.TrimRightFunc(l, unicode.IsSpace) + "\n" + } + + return + }, + }) + + tpl, err = tpl.Parse(scriptTemplateRaw) + if err != nil { + panic(err) + } +} + +func ScriptMigrator(subsys string) *cobra.Command { + var ( + tblPrefix = subsys + isCompose = subsys == "compose" + isSystem = subsys == "system" + ) + + if isSystem { + tblPrefix = "sys" + } + + cmd := &cobra.Command{ + Use: "script-migrator", + Short: "Migrates automation scripts", + Long: "Scans system & compose automation tables for scripts & Triggers and creates script files", + + Run: func(cmd *cobra.Command, args []string) { + var ( + dstPath = cmd.Flags().Lookup("dst").Value.String() + + f *os.File + + err error + ss = ScriptSet{} + tt = TriggerSet{} + + ctx = cli.Context() + + db = factory.Database.MustGet(subsys, "default").With(ctx) //.Quiet() + + // Skip deleted, scripts, ones named test and those with empty source + scriptQuery = squirrel. + Select("*"). + From(tblPrefix+"_automation_script"). + Where("name <> ?", "test"). + Where("source <> ?", ""). + Where("deleted_at IS NULL"). + OrderBy("RAND()") + + // Skip deleted triggers + triggerQuery = squirrel. + Select("*"). + From(tblPrefix + "_automation_trigger"). + Where("deleted_at IS NULL") + + // (for compose) + // preload modules - we want to refer to them (if possible) by handle + moduleQuery = squirrel. + Select("id", "handle", "name"). + From(tblPrefix + "_module") + ) + + err = rh.FetchAll(db, scriptQuery, &ss) + cli.HandleError(err) + + err = rh.FetchAll(db, triggerQuery, &tt) + cli.HandleError(err) + + if isCompose { + err = rh.FetchAll(db, moduleQuery, &modules) + cli.HandleError(err) + } + + cmd.Printf("Found %d scripts and %d triggers\n", len(ss), len(tt)) + + if len(dstPath) == 0 { + // No destination, just output list of scripts + for _, s := range ss { + cmd.Printf("%s\n", s.Name) + } + } else { + done := make(map[string]bool) + + for _, s := range ss { + // sanitize script name into safe file name + sname := sanitizer.ReplaceAllString(strings.ReplaceAll(s.Name, " ", "_"), "") + fname := "" + names := []string{ + fmt.Sprintf("%s.js", sname), + fmt.Sprintf("%s_%d.js", sname, s.ID), + } + + for _, fname = range names { + if !done[fname] { + break + } + } + + s.Triggers, _ = tt.Filter(func(t *Trigger) (b bool, err error) { + return t.ScriptID == s.ID, nil + }) + + fullpath := path.Join(dstPath, fname) + + cmd.Printf(" exporting to %s\n", fullpath) + f, err = os.Create(fullpath) + cli.HandleError(err) + cli.HandleError(tpl.Execute(f, s)) + done[fname] = true + } + } + }, + } + + cmd.Flags().String("dst", "", "Where to export the scripts") + + return cmd +} diff --git a/pkg/automation/scheduled.go b/pkg/automation/scheduled.go index 62dff013b..5b9bd3fa7 100644 --- a/pkg/automation/scheduled.go +++ b/pkg/automation/scheduled.go @@ -33,7 +33,7 @@ func buildScheduleList(runables ScriptSet) scheduledSet { _ = runables.Walk(func(s *Script) error { sch := schedule{scriptID: s.ID} - for _, t := range s.triggers { + for _, t := range s.Triggers { if !t.IsDeferred() { // only interested in deferred scripts continue @@ -55,7 +55,7 @@ func buildScheduleList(runables ScriptSet) scheduledSet { } } - for _, t := range s.triggers { + for _, t := range s.Triggers { if !t.IsInterval() { // only interested in interval scripts continue diff --git a/pkg/automation/scheduled_test.go b/pkg/automation/scheduled_test.go index 0b0e93a24..67ea66a8e 100644 --- a/pkg/automation/scheduled_test.go +++ b/pkg/automation/scheduled_test.go @@ -32,11 +32,11 @@ func TestScheduleBuilder(t *testing.T) { }{ {name: "basics", ss: ScriptSet{ - &Script{ID: 1, Enabled: true, triggers: TriggerSet{ + &Script{ID: 1, Enabled: true, Triggers: TriggerSet{ &Trigger{Enabled: true, Event: EVENT_TYPE_DEFERRED, Condition: "2000-01-01T00:02:00+02:00"}, &Trigger{Enabled: true, Event: EVENT_TYPE_DEFERRED, Condition: "2000-01-01T00:03:00+02:00"}, }}, - &Script{ID: 2, Enabled: true, triggers: TriggerSet{ + &Script{ID: 2, Enabled: true, Triggers: TriggerSet{ &Trigger{Enabled: true, Event: EVENT_TYPE_DEFERRED, Condition: "2000-01-01T00:02:00+02:00"}, &Trigger{Enabled: true, Event: EVENT_TYPE_DEFERRED, Condition: "2000-01-01T00:03:00+02:00"}, }}, @@ -49,11 +49,11 @@ func TestScheduleBuilder(t *testing.T) { }, {name: "intervals", ss: ScriptSet{ - &Script{ID: 1, Enabled: true, triggers: TriggerSet{ + &Script{ID: 1, Enabled: true, Triggers: TriggerSet{ &Trigger{Enabled: true, Event: EVENT_TYPE_INTERVAL, Condition: "0 * * * * * *"}, &Trigger{Enabled: true, Event: EVENT_TYPE_INTERVAL, Condition: "invalid"}, }}, - &Script{ID: 2, Enabled: true, triggers: TriggerSet{ + &Script{ID: 2, Enabled: true, Triggers: TriggerSet{ &Trigger{Enabled: true, Event: EVENT_TYPE_INTERVAL, Condition: "0 0 * * * * *"}, &Trigger{Enabled: true, Event: EVENT_TYPE_INTERVAL, Condition: "invalid"}, }}, diff --git a/pkg/automation/script.go b/pkg/automation/script.go index df97a6c9e..2acfd3e85 100644 --- a/pkg/automation/script.go +++ b/pkg/automation/script.go @@ -51,12 +51,12 @@ type ( DeletedAt *time.Time `db:"deleted_at" json:"deletedAt,omitempty"` DeletedBy uint64 `db:"deleted_by" json:"deletedBy,string,omitempty" ` - // Serves as container for valid triggers for runnable scripts internal cache + // Serves as container for valid Triggers for runnable scripts internal cache // and for as a transport on create/update operations // // Node: on c/u op., we currently just merge current state with the given list, - // w/o updating the rest of the script's triggers - triggers TriggerSet + // w/o updating the rest of the script's Triggers + Triggers TriggerSet // How are we merging? tms triggersMergeStrategy @@ -86,10 +86,10 @@ type ( ) const ( - // Ignore the given triggers + // Ignore the given Triggers STMS_IGNORE triggersMergeStrategy = iota - // Create triggers, no pre-checks + // Create Triggers, no pre-checks STMS_FRESH // Update existing with new @@ -133,11 +133,11 @@ func (s *Script) CheckCompatibility(t *Trigger) error { if t.IsDeferred() || t.IsInterval() { if s.RunInUA { - return errors.New("deferred triggers are not compatible with user-agent scripts") + return errors.New("deferred Triggers are not compatible with user-agent scripts") } if s.RunAsInvoker() { - return errors.New("deferred triggers are not compatible with run-as-invoker scripts") + return errors.New("deferred Triggers are not compatible with run-as-invoker scripts") } } @@ -149,7 +149,7 @@ func (s *Script) CheckCompatibility(t *Trigger) error { // Filters non-UA scripts that match event and resource + all extra conditions func (set ScriptSet) FilterByTrigger(event, resource string, cc ...TriggerConditionChecker) (out ScriptSet) { out, _ = set.Filter(func(s *Script) (bool, error) { - return s.IsValid() && s.triggers.HasMatch(Trigger{Event: event, Resource: resource}, cc...), nil + return s.IsValid() && s.Triggers.HasMatch(Trigger{Event: event, Resource: resource}, cc...), nil }) return @@ -181,7 +181,7 @@ func (s Script) RunAsInvoker() bool { return s.RunAs == 0 } -// AddTrigger appends one or more triggers to internal list of triggers on script struct +// AddTrigger appends one or more Triggers to internal list of Triggers on script struct // // We do not do any compatibility check (See Script.CheckCompatibility()); // this is only an utility func that helps us pass data along @@ -193,23 +193,19 @@ func (s *Script) AddTrigger(strategy triggersMergeStrategy, tt ...*Trigger) { } if s.tms == STMS_REPLACE { - s.triggers = TriggerSet{} + s.Triggers = TriggerSet{} } // Make sure all your trigger belong to us (ref same script or no ref): for _, t := range tt { if t.ScriptID == 0 || t.ScriptID == s.ID { - s.triggers = append(s.triggers, t) + s.Triggers = append(s.Triggers, t) } } } -func (s *Script) Triggers() TriggerSet { - return s.triggers -} - func (s Script) HasEvent(event string) bool { - return s.triggers.HasMatch(Trigger{Event: event}) + return s.Triggers.HasMatch(Trigger{Event: event}) } func (s Script) Credentials() string { diff --git a/pkg/automation/script_repository.go b/pkg/automation/script_repository.go index 1a81b8040..93d000d93 100644 --- a/pkg/automation/script_repository.go +++ b/pkg/automation/script_repository.go @@ -96,7 +96,7 @@ func (r *scriptRepository) find(db *factory.DB, filter ScriptFilter) (set Script } if f.Resource != "" { - // Making partial trigger repo struct on the fly to help us calculate the name of the triggers table + // Making partial trigger repo struct on the fly to help us calculate the name of the Triggers table query = query.Where( fmt.Sprintf("id IN (SELECT rel_script FROM `%s` WHERE resource = ?", r.table()), f.Resource, diff --git a/pkg/automation/service.go b/pkg/automation/service.go index a6cf3c9f2..c1fe07ef7 100644 --- a/pkg/automation/service.go +++ b/pkg/automation/service.go @@ -25,7 +25,7 @@ type ( // service will flush values on TRUE or just reload on FALSE f chan bool - // internal list of runnable scripts (and their accompanying triggers) + // internal list of runnable scripts (and their accompanying Triggers) runnables ScriptSet // internal list of scheduled scripts @@ -68,7 +68,7 @@ const ( // Service initializes service{} struct // -// service{} struct handles scripts & triggers. It acts as a caching layer and +// service{} struct handles scripts & Triggers. It acts as a caching layer and // proxy to repository where it verifies and enriches payloads // func Service(c AutomationServiceConfig) (svc *service) { @@ -191,7 +191,7 @@ func (svc *service) reload(ctx context.Context) { }) tt, err = svc.trepo.findRunnable(db) - svc.logger.Info("triggers loaded", zap.Error(err), zap.Int("count", len(tt))) + svc.logger.Info("Triggers loaded", zap.Error(err), zap.Int("count", len(tt))) if err != nil { return } @@ -217,8 +217,8 @@ func (svc *service) reload(ctx context.Context) { _ = tt.Walk(func(t *Trigger) error { s := svc.runnables.FindByID(t.ScriptID) if s != nil && t.IsValid() && s.CheckCompatibility(t) == nil { - // Add only compatible triggers - s.triggers = append(s.triggers, t) + // Add only compatible Triggers + s.Triggers = append(s.Triggers, t) } return nil @@ -237,7 +237,7 @@ func (svc *service) reload(ctx context.Context) { // FindRunnableScripts finds runnable scripts in internal list // // It uses resource, event and extra condition checkers to filter out all scripts -// that have matching triggers +// that have matching Triggers func (svc service) FindRunnableScripts(resource, event string, cc ...TriggerConditionChecker) ScriptSet { svc.l.Lock() defer svc.l.Unlock() @@ -280,7 +280,7 @@ func (svc service) CreateScript(ctx context.Context, s *Script) error { return } - err = s.triggers.Walk(func(t *Trigger) error { + err = s.Triggers.Walk(func(t *Trigger) error { return svc.setNewTriggerInfo(ctx, s, t) }) @@ -289,7 +289,7 @@ func (svc service) CreateScript(ctx context.Context, s *Script) error { } // Force no-pre-check - if err = svc.trepo.mergeSet(db, STMS_FRESH, s.ID, s.triggers); err != nil { + if err = svc.trepo.mergeSet(db, STMS_FRESH, s.ID, s.Triggers); err != nil { return } @@ -322,7 +322,7 @@ func (svc service) UpdateScript(ctx context.Context, s *Script) error { return } - err = s.triggers.Walk(func(t *Trigger) error { + err = s.Triggers.Walk(func(t *Trigger) error { if t.ID == 0 { return svc.setNewTriggerInfo(ctx, s, t) } else { @@ -334,7 +334,7 @@ func (svc service) UpdateScript(ctx context.Context, s *Script) error { return } - if err = svc.trepo.mergeSet(db, s.tms, s.ID, s.triggers); err != nil { + if err = svc.trepo.mergeSet(db, s.tms, s.ID, s.Triggers); err != nil { return } diff --git a/pkg/automation/trigger.go b/pkg/automation/trigger.go index 3bfc410de..49fc0f617 100644 --- a/pkg/automation/trigger.go +++ b/pkg/automation/trigger.go @@ -97,7 +97,7 @@ func (t Trigger) Uint64Condition() (o uint64) { return } -// HasMatch checks if any og the triggers in a set matches the given parameters +// HasMatch checks if any og the Triggers in a set matches the given parameters func (set TriggerSet) HasMatch(m Trigger, ff ...TriggerConditionChecker) bool { withTriggers: for _, t := range set { diff --git a/pkg/automation/trigger_repository.go b/pkg/automation/trigger_repository.go index 2a2a66aa3..98fe00639 100644 --- a/pkg/automation/trigger_repository.go +++ b/pkg/automation/trigger_repository.go @@ -60,7 +60,7 @@ func (r *triggerRepository) findByID(db *factory.DB, triggerID uint64) (*Trigger return rval, rh.IsFound(rh.FetchOne(db, query, rval), rval.ID > 0, errors.New("trigger not found")) } -// Find - finds triggers using given filter +// Find - finds Triggers using given filter func (r *triggerRepository) find(db *factory.DB, filter TriggerFilter) (set TriggerSet, f TriggerFilter, err error) { f = filter @@ -95,7 +95,7 @@ func (r *triggerRepository) find(db *factory.DB, filter TriggerFilter) (set Trig return set, f, rh.FetchPaged(db, query, f.Page, f.PerPage, &set) } -// FindAllRunnable - loads and returns all runnable triggers +// FindAllRunnable - loads and returns all runnable Triggers func (r *triggerRepository) findRunnable(db *factory.DB) (TriggerSet, error) { rr := make([]*Trigger, 0) @@ -103,7 +103,7 @@ func (r *triggerRepository) findRunnable(db *factory.DB) (TriggerSet, error) { db, r.query().Where("enabled AND deleted_at IS NULL"), &rr, - ), "could not load runnable triggers") + ), "could not load runnable Triggers") } func (r *triggerRepository) replace(db *factory.DB, t *Trigger) (err error) { @@ -123,7 +123,7 @@ func (r *triggerRepository) deleteByScriptID(db *factory.DB, scriptID uint64) (e // Check for existing events func (r *triggerRepository) checkDuplicate(db *factory.DB, t *Trigger) (*Trigger, error) { if t.IsDeferred() && t.IsInterval() { - // deferred & interval triggers + // deferred & interval Triggers // can have duplicates return nil, nil } @@ -153,7 +153,7 @@ func (r *triggerRepository) mergeSet(db *factory.DB, tms triggersMergeStrategy, // Mark all existing as deleted // // here, we're assuming we have the entire - // trigger list present (in s.triggers) + // trigger list present (in s.Triggers) if err := r.deleteByScriptID(db, scriptID); err != nil { return err } @@ -164,7 +164,7 @@ func (r *triggerRepository) mergeSet(db *factory.DB, tms triggersMergeStrategy, return nil } - // Replace (upsert) all triggers we have + // Replace (upsert) all Triggers we have return r.replace(db, t) }) } diff --git a/system/app.go b/system/app.go index 2dbf92f1c..888172a25 100644 --- a/system/app.go +++ b/system/app.go @@ -2,6 +2,7 @@ package system import ( "context" + "github.com/cortezaproject/corteza-server/pkg/automation" "github.com/go-chi/chi" _ "github.com/joho/godotenv/autoload" @@ -143,5 +144,7 @@ func (app *App) RegisterCliCommands(p *cobra.Command) { commands.Users(), commands.Roles(), commands.Sink(), + // temp command, will be removed in 2020.6 + automation.ScriptMigrator(SERVICE), ) }