DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Add basic automation script exporting

Browse Source
This commit is contained in:
Denis Arh
2020-01-18 07:05:34 +01:00
parent ce1fc3402d
commit 4d6cb13f70
10 changed files with 295 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose/app.go
+3
View File
@@ -2,6 +2,7 @@ package compose
import (
"context"
"github.com/cortezaproject/corteza-server/pkg/automation"
"github.com/go-chi/chi"
_ "github.com/joho/godotenv/autoload"
@@ -99,5 +100,7 @@ func (app *App) RegisterCliCommands(p *cobra.Command) {
p.AddCommand(
commands.Importer(),
commands.Exporter(),
// temp command, will be removed in 2020.6
automation.ScriptMigrator(SERVICE),
)
}
pkg/automation/command.go
+253
View File
@@ -0,0 +1,253 @@
package automation
import (
"fmt"
"github.com/Masterminds/squirrel"
"github.com/cortezaproject/corteza-server/pkg/cli"
"github.com/cortezaproject/corteza-server/pkg/rh"
"github.com/davecgh/go-spew/spew"
"github.com/spf13/cobra"
"github.com/titpetric/factory"
"os"
"path"
"regexp"
"strconv"
"strings"
"text/template"
"unicode"
)
type (
module struct {
ID uint64
Handle string
Name string
}
)
var (
sanitizer = regexp.MustCompile(`[^a-zA-Z0-9_\-.]+`)
modules []*module
// Language=GoTemplate
scriptTemplateRaw string = `import trigger from '+Trigger'
export default {
label: {{ quote $.Name }},
desc: '...',
triggers: [
{{- range $t := $.Triggers }}
// auto-migrated
// ID: {{ $t.ID }}
// Created: {{ $t.CreatedAt }}
// Updated: {{ $t.UpdatedAt }}
{{ if not $t.Enabled }}/* disabled {{ end }}
trigger
.on({{ quote $t.Event }}){{- if $.RunAs }}
.as({{ quote $t.RunAs }}){{ end }}
.for({{ quote $t.Resource }})
{{- makeConditionFn $t -}}
{{ if not $t.Enabled }}*/{{ end }}
{{ end -}}
],
async handler ({ $namespace, $module, $record }, { log, ComposeUI, Compose }) {
{{ indent .Source 4 }}
}
}
`
tpl *template.Template
)
func init() {
var err error
tpl = template.New("").Funcs(map[string]interface{}{
//"camelCase": camelCase,
//"makeEvents": makeEvents,
"dump": func(s ...interface{}) string {
return spew.Sdump(s...)
},
"quote": func(s string) string {
return `'` + s + `'`
},
"makeConditionFn": func(t *Trigger) string {
isStdBeforeAfter := (strings.HasSuffix(t.Event, "Create") ||
strings.HasSuffix(t.Event, "Update") ||
strings.HasSuffix(t.Event, "Delete")) &&
(strings.HasPrefix(t.Event, "before") ||
strings.HasPrefix(t.Event, "after"))
if t.Condition != "" {
if t.Resource == "compose:record" && (isStdBeforeAfter || t.Event == "manual") {
id, _ := strconv.ParseUint(t.Condition, 10, 64)
if id > 0 {
for _, m := range modules {
if m.ID == id {
cnd := m.Handle
if cnd == "" {
cnd = t.Condition
}
return fmt.Sprintf("\n .where('module', '%s'), // module (%d) %s\n", cnd, m.ID, m.Name)
}
}
return fmt.Sprintf("\n .where('module', '%s'), // module not found, could not translate ID to handle\n", t.Condition)
}
} else if t.Event == "deferred" {
return fmt.Sprintf("\n .where('timestamp', '%s'),\n", t.Condition)
} else if t.Event == "interval" {
return fmt.Sprintf("\n .where('interval', '%s'),\n", t.Condition)
}
return fmt.Sprintf(", // unresolvable condition - %s \n", t.Condition)
}
return ",\n"
},
"makeEventFn": func(ev string) string {
tpl := ".%s('%s')"
if strings.HasPrefix(ev, "before") {
return fmt.Sprintf(tpl, "before", strings.ToLower(ev))
}
if strings.HasPrefix(ev, "after") {
return fmt.Sprintf(tpl, "after", strings.ToLower(ev))
}
if ev == "deferred" {
ev = "timestamp"
}
return fmt.Sprintf(tpl, "on", ev)
},
"indent": func(s string, spaces int) (o string) {
for _, l := range strings.Split(s, "\n") {
o = o + strings.Repeat(" ", spaces) + strings.TrimRightFunc(l, unicode.IsSpace) + "\n"
}
return
},
})
tpl, err = tpl.Parse(scriptTemplateRaw)
if err != nil {
panic(err)
}
}
func ScriptMigrator(subsys string) *cobra.Command {
var (
tblPrefix = subsys
isCompose = subsys == "compose"
isSystem = subsys == "system"
)
if isSystem {
tblPrefix = "sys"
}
cmd := &cobra.Command{
Use: "script-migrator",
Short: "Migrates automation scripts",
Long: "Scans system & compose automation tables for scripts & Triggers and creates script files",
Run: func(cmd *cobra.Command, args []string) {
var (
dstPath = cmd.Flags().Lookup("dst").Value.String()
f *os.File
err error
ss = ScriptSet{}
tt = TriggerSet{}
ctx = cli.Context()
db = factory.Database.MustGet(subsys, "default").With(ctx) //.Quiet()
// Skip deleted, scripts, ones named test and those with empty source
scriptQuery = squirrel.
Select("*").
From(tblPrefix+"_automation_script").
Where("name <> ?", "test").
Where("source <> ?", "").
Where("deleted_at IS NULL").
OrderBy("RAND()")
// Skip deleted triggers
triggerQuery = squirrel.
Select("*").
From(tblPrefix + "_automation_trigger").
Where("deleted_at IS NULL")
// (for compose)
// preload modules - we want to refer to them (if possible) by handle
moduleQuery = squirrel.
Select("id", "handle", "name").
From(tblPrefix + "_module")
)
err = rh.FetchAll(db, scriptQuery, &ss)
cli.HandleError(err)
err = rh.FetchAll(db, triggerQuery, &tt)
cli.HandleError(err)
if isCompose {
err = rh.FetchAll(db, moduleQuery, &modules)
cli.HandleError(err)
}
cmd.Printf("Found %d scripts and %d triggers\n", len(ss), len(tt))
if len(dstPath) == 0 {
// No destination, just output list of scripts
for _, s := range ss {
cmd.Printf("%s\n", s.Name)
}
} else {
done := make(map[string]bool)
for _, s := range ss {
// sanitize script name into safe file name
sname := sanitizer.ReplaceAllString(strings.ReplaceAll(s.Name, " ", "_"), "")
fname := ""
names := []string{
fmt.Sprintf("%s.js", sname),
fmt.Sprintf("%s_%d.js", sname, s.ID),
}
for _, fname = range names {
if !done[fname] {
break
}
}
s.Triggers, _ = tt.Filter(func(t *Trigger) (b bool, err error) {
return t.ScriptID == s.ID, nil
})
fullpath := path.Join(dstPath, fname)
cmd.Printf(" exporting to %s\n", fullpath)
f, err = os.Create(fullpath)
cli.HandleError(err)
cli.HandleError(tpl.Execute(f, s))
done[fname] = true
}
}
},
}
cmd.Flags().String("dst", "", "Where to export the scripts")
return cmd
}
pkg/automation/scheduled.go
+2 -2
View File
@@ -33,7 +33,7 @@ func buildScheduleList(runables ScriptSet) scheduledSet {
_ = runables.Walk(func(s *Script) error {
sch := schedule{scriptID: s.ID}
for _, t := range s.triggers {
for _, t := range s.Triggers {
if !t.IsDeferred() {
// only interested in deferred scripts
continue
@@ -55,7 +55,7 @@ func buildScheduleList(runables ScriptSet) scheduledSet {
}
}
for _, t := range s.triggers {
for _, t := range s.Triggers {
if !t.IsInterval() {
// only interested in interval scripts
continue
pkg/automation/scheduled_test.go
+4 -4
View File
@@ -32,11 +32,11 @@ func TestScheduleBuilder(t *testing.T) {
}{
{name: "basics",
ss: ScriptSet{
&Script{ID: 1, Enabled: true, triggers: TriggerSet{
&Script{ID: 1, Enabled: true, Triggers: TriggerSet{
&Trigger{Enabled: true, Event: EVENT_TYPE_DEFERRED, Condition: "2000-01-01T00:02:00+02:00"},
&Trigger{Enabled: true, Event: EVENT_TYPE_DEFERRED, Condition: "2000-01-01T00:03:00+02:00"},
}},
&Script{ID: 2, Enabled: true, triggers: TriggerSet{
&Script{ID: 2, Enabled: true, Triggers: TriggerSet{
&Trigger{Enabled: true, Event: EVENT_TYPE_DEFERRED, Condition: "2000-01-01T00:02:00+02:00"},
&Trigger{Enabled: true, Event: EVENT_TYPE_DEFERRED, Condition: "2000-01-01T00:03:00+02:00"},
}},
@@ -49,11 +49,11 @@ func TestScheduleBuilder(t *testing.T) {
},
{name: "intervals",
ss: ScriptSet{
&Script{ID: 1, Enabled: true, triggers: TriggerSet{
&Script{ID: 1, Enabled: true, Triggers: TriggerSet{
&Trigger{Enabled: true, Event: EVENT_TYPE_INTERVAL, Condition: "0 * * * * * *"},
&Trigger{Enabled: true, Event: EVENT_TYPE_INTERVAL, Condition: "invalid"},
}},
&Script{ID: 2, Enabled: true, triggers: TriggerSet{
&Script{ID: 2, Enabled: true, Triggers: TriggerSet{
&Trigger{Enabled: true, Event: EVENT_TYPE_INTERVAL, Condition: "0 0 * * * * *"},
&Trigger{Enabled: true, Event: EVENT_TYPE_INTERVAL, Condition: "invalid"},
}},
pkg/automation/script.go
+12 -16
View File
@@ -51,12 +51,12 @@ type (
DeletedAt *time.Time `db:"deleted_at" json:"deletedAt,omitempty"`
DeletedBy uint64 `db:"deleted_by" json:"deletedBy,string,omitempty" `
// Serves as container for valid triggers for runnable scripts internal cache
// Serves as container for valid Triggers for runnable scripts internal cache
// and for as a transport on create/update operations
//
// Node: on c/u op., we currently just merge current state with the given list,
// w/o updating the rest of the script's triggers
triggers TriggerSet
// w/o updating the rest of the script's Triggers
Triggers TriggerSet
// How are we merging?
tms triggersMergeStrategy
@@ -86,10 +86,10 @@ type (
)
const (
// Ignore the given triggers
// Ignore the given Triggers
STMS_IGNORE triggersMergeStrategy = iota
// Create triggers, no pre-checks
// Create Triggers, no pre-checks
STMS_FRESH
// Update existing with new
@@ -133,11 +133,11 @@ func (s *Script) CheckCompatibility(t *Trigger) error {
if t.IsDeferred() || t.IsInterval() {
if s.RunInUA {
return errors.New("deferred triggers are not compatible with user-agent scripts")
return errors.New("deferred Triggers are not compatible with user-agent scripts")
}
if s.RunAsInvoker() {
return errors.New("deferred triggers are not compatible with run-as-invoker scripts")
return errors.New("deferred Triggers are not compatible with run-as-invoker scripts")
}
}
@@ -149,7 +149,7 @@ func (s *Script) CheckCompatibility(t *Trigger) error {
// Filters non-UA scripts that match event and resource + all extra conditions
func (set ScriptSet) FilterByTrigger(event, resource string, cc ...TriggerConditionChecker) (out ScriptSet) {
out, _ = set.Filter(func(s *Script) (bool, error) {
return s.IsValid() && s.triggers.HasMatch(Trigger{Event: event, Resource: resource}, cc...), nil
return s.IsValid() && s.Triggers.HasMatch(Trigger{Event: event, Resource: resource}, cc...), nil
})
return
@@ -181,7 +181,7 @@ func (s Script) RunAsInvoker() bool {
return s.RunAs == 0
}
// AddTrigger appends one or more triggers to internal list of triggers on script struct
// AddTrigger appends one or more Triggers to internal list of Triggers on script struct
//
// We do not do any compatibility check (See Script.CheckCompatibility());
// this is only an utility func that helps us pass data along
@@ -193,23 +193,19 @@ func (s *Script) AddTrigger(strategy triggersMergeStrategy, tt ...*Trigger) {
}
if s.tms == STMS_REPLACE {
s.triggers = TriggerSet{}
s.Triggers = TriggerSet{}
}
// Make sure all your trigger belong to us (ref same script or no ref):
for _, t := range tt {
if t.ScriptID == 0 || t.ScriptID == s.ID {
s.triggers = append(s.triggers, t)
s.Triggers = append(s.Triggers, t)
}
}
}
func (s *Script) Triggers() TriggerSet {
return s.triggers
}
func (s Script) HasEvent(event string) bool {
return s.triggers.HasMatch(Trigger{Event: event})
return s.Triggers.HasMatch(Trigger{Event: event})
}
func (s Script) Credentials() string {
pkg/automation/script_repository.go
+1 -1
View File
@@ -96,7 +96,7 @@ func (r *scriptRepository) find(db *factory.DB, filter ScriptFilter) (set Script
}
if f.Resource != "" {
// Making partial trigger repo struct on the fly to help us calculate the name of the triggers table
// Making partial trigger repo struct on the fly to help us calculate the name of the Triggers table
query = query.Where(
fmt.Sprintf("id IN (SELECT rel_script FROM `%s` WHERE resource = ?", r.table()),
f.Resource,
pkg/automation/service.go
+10 -10
View File
@@ -25,7 +25,7 @@ type (
// service will flush values on TRUE or just reload on FALSE
f chan bool
// internal list of runnable scripts (and their accompanying triggers)
// internal list of runnable scripts (and their accompanying Triggers)
runnables ScriptSet
// internal list of scheduled scripts
@@ -68,7 +68,7 @@ const (
// Service initializes service{} struct
//
// service{} struct handles scripts & triggers. It acts as a caching layer and
// service{} struct handles scripts & Triggers. It acts as a caching layer and
// proxy to repository where it verifies and enriches payloads
//
func Service(c AutomationServiceConfig) (svc *service) {
@@ -191,7 +191,7 @@ func (svc *service) reload(ctx context.Context) {
})
tt, err = svc.trepo.findRunnable(db)
svc.logger.Info("triggers loaded", zap.Error(err), zap.Int("count", len(tt)))
svc.logger.Info("Triggers loaded", zap.Error(err), zap.Int("count", len(tt)))
if err != nil {
return
}
@@ -217,8 +217,8 @@ func (svc *service) reload(ctx context.Context) {
_ = tt.Walk(func(t *Trigger) error {
s := svc.runnables.FindByID(t.ScriptID)
if s != nil && t.IsValid() && s.CheckCompatibility(t) == nil {
// Add only compatible triggers
s.triggers = append(s.triggers, t)
// Add only compatible Triggers
s.Triggers = append(s.Triggers, t)
}
return nil
@@ -237,7 +237,7 @@ func (svc *service) reload(ctx context.Context) {
// FindRunnableScripts finds runnable scripts in internal list
//
// It uses resource, event and extra condition checkers to filter out all scripts
// that have matching triggers
// that have matching Triggers
func (svc service) FindRunnableScripts(resource, event string, cc ...TriggerConditionChecker) ScriptSet {
svc.l.Lock()
defer svc.l.Unlock()
@@ -280,7 +280,7 @@ func (svc service) CreateScript(ctx context.Context, s *Script) error {
return
}
err = s.triggers.Walk(func(t *Trigger) error {
err = s.Triggers.Walk(func(t *Trigger) error {
return svc.setNewTriggerInfo(ctx, s, t)
})
@@ -289,7 +289,7 @@ func (svc service) CreateScript(ctx context.Context, s *Script) error {
}
// Force no-pre-check
if err = svc.trepo.mergeSet(db, STMS_FRESH, s.ID, s.triggers); err != nil {
if err = svc.trepo.mergeSet(db, STMS_FRESH, s.ID, s.Triggers); err != nil {
return
}
@@ -322,7 +322,7 @@ func (svc service) UpdateScript(ctx context.Context, s *Script) error {
return
}
err = s.triggers.Walk(func(t *Trigger) error {
err = s.Triggers.Walk(func(t *Trigger) error {
if t.ID == 0 {
return svc.setNewTriggerInfo(ctx, s, t)
} else {
@@ -334,7 +334,7 @@ func (svc service) UpdateScript(ctx context.Context, s *Script) error {
return
}
if err = svc.trepo.mergeSet(db, s.tms, s.ID, s.triggers); err != nil {
if err = svc.trepo.mergeSet(db, s.tms, s.ID, s.Triggers); err != nil {
return
}
pkg/automation/trigger.go
+1 -1
View File
@@ -97,7 +97,7 @@ func (t Trigger) Uint64Condition() (o uint64) {
return
}
// HasMatch checks if any og the triggers in a set matches the given parameters
// HasMatch checks if any og the Triggers in a set matches the given parameters
func (set TriggerSet) HasMatch(m Trigger, ff ...TriggerConditionChecker) bool {
withTriggers:
for _, t := range set {
pkg/automation/trigger_repository.go
+6 -6
View File
@@ -60,7 +60,7 @@ func (r *triggerRepository) findByID(db *factory.DB, triggerID uint64) (*Trigger
return rval, rh.IsFound(rh.FetchOne(db, query, rval), rval.ID > 0, errors.New("trigger not found"))
}
// Find - finds triggers using given filter
// Find - finds Triggers using given filter
func (r *triggerRepository) find(db *factory.DB, filter TriggerFilter) (set TriggerSet, f TriggerFilter, err error) {
f = filter
@@ -95,7 +95,7 @@ func (r *triggerRepository) find(db *factory.DB, filter TriggerFilter) (set Trig
return set, f, rh.FetchPaged(db, query, f.Page, f.PerPage, &set)
}
// FindAllRunnable - loads and returns all runnable triggers
// FindAllRunnable - loads and returns all runnable Triggers
func (r *triggerRepository) findRunnable(db *factory.DB) (TriggerSet, error) {
rr := make([]*Trigger, 0)
@@ -103,7 +103,7 @@ func (r *triggerRepository) findRunnable(db *factory.DB) (TriggerSet, error) {
db,
r.query().Where("enabled AND deleted_at IS NULL"),
&rr,
), "could not load runnable triggers")
), "could not load runnable Triggers")
}
func (r *triggerRepository) replace(db *factory.DB, t *Trigger) (err error) {
@@ -123,7 +123,7 @@ func (r *triggerRepository) deleteByScriptID(db *factory.DB, scriptID uint64) (e
// Check for existing events
func (r *triggerRepository) checkDuplicate(db *factory.DB, t *Trigger) (*Trigger, error) {
if t.IsDeferred() && t.IsInterval() {
// deferred & interval triggers
// deferred & interval Triggers
// can have duplicates
return nil, nil
}
@@ -153,7 +153,7 @@ func (r *triggerRepository) mergeSet(db *factory.DB, tms triggersMergeStrategy,
// Mark all existing as deleted
//
// here, we're assuming we have the entire
// trigger list present (in s.triggers)
// trigger list present (in s.Triggers)
if err := r.deleteByScriptID(db, scriptID); err != nil {
return err
}
@@ -164,7 +164,7 @@ func (r *triggerRepository) mergeSet(db *factory.DB, tms triggersMergeStrategy,
return nil
}
// Replace (upsert) all triggers we have
// Replace (upsert) all Triggers we have
return r.replace(db, t)
})
}
system/app.go
+3
View File
@@ -2,6 +2,7 @@ package system
import (
"context"
"github.com/cortezaproject/corteza-server/pkg/automation"
"github.com/go-chi/chi"
_ "github.com/joho/godotenv/autoload"
@@ -143,5 +144,7 @@ func (app *App) RegisterCliCommands(p *cobra.Command) {
commands.Users(),
commands.Roles(),
commands.Sink(),
// temp command, will be removed in 2020.6
automation.ScriptMigrator(SERVICE),
)
}