From 751317d544eed370e285743166118239af10dff4 Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Tue, 13 Aug 2019 16:14:19 +0200 Subject: [PATCH] Support automation script testing and manual running --- api/compose/spec.json | 25 ++++- api/compose/spec/automation_script.json | 47 +++++--- compose/internal/service/automation_runner.go | 51 +++++++-- compose/rest/automation_script.go | 103 ++++++++++++------ compose/rest/handlers/automation_script.go | 25 ++++- compose/rest/record.go | 6 + compose/rest/request/automation_script.go | 84 +++++++++++--- docs/compose/README.md | 30 +++-- 8 files changed, 286 insertions(+), 85 deletions(-) diff --git a/api/compose/spec.json b/api/compose/spec.json index 4dcce078c..b7f034e2b 100644 --- a/api/compose/spec.json +++ b/api/compose/spec.json @@ -1500,15 +1500,28 @@ { "name": "run", "method": "POST", - "title": "Run a specific script or code at the backend. For testing and manual execution", - "path": "/run", + "title": "Run a specific script or code at the backend. Used for running script manually", + "path": "/{scriptID}/run", + "parameters": { + "path": [ + {"type": "uint64", "name": "scriptID", "required": true} + ], + "post": [ + {"name": "moduleID", "type": "uint64", "title": "ModuleID to be used"}, + {"name": "recordID", "type": "uint64", "title": "RecordID to be used (instead of record payload)"}, + {"name": "record", "type": "json.RawMessage", "title": "Record payload to be used (instead of specific record when using recordID)"} + ] + } + }, + { + "name": "test", + "method": "POST", + "title": "Run source code in corredor. Used for testing", + "path": "/test", "parameters": { "post": [ - {"name": "scriptID", "type": "uint64", "title": "Script ID (scripts with manual triggers)"}, - {"name": "source", "type": "string", "title": "Script's source code (overrides scriptID parameter)"}, + {"name": "source", "type": "string", "title": "Script's source code"}, {"name": "moduleID", "type": "uint64", "title": "Preload module and pass it to the automation script"}, - {"name": "recordID", "type": "uint64", "title": "Preload record and pass it to the automation script"}, - {"name": "module", "type": "json.RawMessage", "title": "Module to pass to the automation script"}, {"name": "record", "type": "json.RawMessage", "title": "Record to pass to the automation script"} ] } diff --git a/api/compose/spec/automation_script.json b/api/compose/spec/automation_script.json index 39cba81e7..eccadf119 100644 --- a/api/compose/spec/automation_script.json +++ b/api/compose/spec/automation_script.json @@ -244,18 +244,45 @@ { "Name": "run", "Method": "POST", - "Title": "Run a specific script or code at the backend. For testing and manual execution", - "Path": "/run", + "Title": "Run a specific script or code at the backend. Used for running script manually", + "Path": "/{scriptID}/run", "Parameters": { - "post": [ + "path": [ { "name": "scriptID", - "title": "Script ID (scripts with manual triggers)", + "required": true, + "type": "uint64" + } + ], + "post": [ + { + "name": "moduleID", + "title": "ModuleID to be used", "type": "uint64" }, + { + "name": "recordID", + "title": "RecordID to be used", + "type": "uint64" + }, + { + "name": "record", + "title": "Record payload to be used (instead of specific record when using recordID)", + "type": "json.RawMessage" + } + ] + } + }, + { + "Name": "test", + "Method": "POST", + "Title": "Run source code in corredor. Used for testing", + "Path": "/test", + "Parameters": { + "post": [ { "name": "source", - "title": "Script's source code (overrides scriptID parameter)", + "title": "Script's source code", "type": "string" }, { @@ -263,16 +290,6 @@ "title": "Preload module and pass it to the automation script", "type": "uint64" }, - { - "name": "recordID", - "title": "Preload record and pass it to the automation script", - "type": "uint64" - }, - { - "name": "module", - "title": "Module to pass to the automation script", - "type": "json.RawMessage" - }, { "name": "record", "title": "Record to pass to the automation script", diff --git a/compose/internal/service/automation_runner.go b/compose/internal/service/automation_runner.go index e08d2c4bd..ac834af54 100644 --- a/compose/internal/service/automation_runner.go +++ b/compose/internal/service/automation_runner.go @@ -185,6 +185,24 @@ func (svc automationRunner) RecordManual(ctx context.Context, scriptID uint64, n return runner(script) } +func (svc automationRunner) RecordScriptTester(ctx context.Context, source string, ns *types.Namespace, m *types.Module, r *types.Record) (err error) { + // Make record script runner and + runner := svc.makeRecordScriptRunner(ctx, ns, m, r, false) + + return runner(&automation.Script{ + ID: 0, + Name: "test", + SourceRef: "test", + Source: source, + Async: false, + RunAs: 0, + RunInUA: false, + Timeout: 0, + Critical: true, + Enabled: false, + }) +} + // Runs record script // // We set-up script-running environment: security (definer / invoker), async, critical @@ -201,14 +219,6 @@ func (svc automationRunner) makeRecordScriptRunner(ctx context.Context, ns *type svc.logger.Debug("executing script", zap.Any("record", r)) return func(script *automation.Script) error { - if !script.IsValid() { - return errors.New("refusing to run invalid script") - } - - if script.RunInUA { - return errors.New("refusing to run user-agent script") - } - if svc.runner == nil { return errors.New("can not run corredor script: not connected") } @@ -277,6 +287,31 @@ func (svc automationRunner) makeRecordScriptRunner(ctx context.Context, ns *type result := proto.ToRecord(rsp.Record) r.OwnedBy, r.Values = result.OwnedBy, result.Values + // Let's copy module, namespace and other values if they are missing + if m != nil { + if r.ModuleID == 0 { + r.ModuleID = m.ID + r.NamespaceID = m.NamespaceID + } + } + + var currentUserID = auth.GetIdentityFromContext(ctx).Identity() + if script.RunAs > 0 { + currentUserID = script.RunAs + } + + if r.OwnedBy == 0 { + r.OwnedBy = currentUserID + } + + if r.CreatedAt.IsZero() { + r.CreatedAt = time.Now() + } + + if r.CreatedBy == 0 { + r.CreatedBy = currentUserID + } + return nil } } diff --git a/compose/rest/automation_script.go b/compose/rest/automation_script.go index 8ffd8c62d..822e31417 100644 --- a/compose/rest/automation_script.go +++ b/compose/rest/automation_script.go @@ -46,6 +46,10 @@ type ( RunInUA bool `json:"runInUA"` } + automationScriptRun struct { + Record *types.Record `json:"record,omitempty"` + } + AutomationScript struct { scripts automationScriptService runner automationScriptRunner @@ -66,7 +70,8 @@ type ( automationScriptRunner interface { UserScripts(context.Context) automation.ScriptSet - RecordManual(context.Context, uint64, *types.Namespace, *types.Module, *types.Record) (err error) + RecordManual(context.Context, uint64, *types.Namespace, *types.Module, *types.Record) error + RecordScriptTester(context.Context, string, *types.Namespace, *types.Module, *types.Record) error } automationScriptAccessController interface { @@ -219,50 +224,82 @@ func (ctrl AutomationScript) Runnable(ctx context.Context, r *request.Automation func (ctrl AutomationScript) Run(ctx context.Context, r *request.AutomationScriptRun) (interface{}, error) { var ( - err error - ns *types.Namespace - - module = &types.Module{} - record = &types.Record{} + rval automationScriptRun + ns, m, record, err = ctrl.loadRecordScriptRunningCombo(r.NamespaceID, r.ModuleID, r.RecordID, r.Record) ) - // Load requested namespace - if ns, err = ctrl.namespace.FindByID(r.NamespaceID); err != nil { + if err != nil { return nil, err } - // Unmarshal given module or find existing one from ID - if r.Module != nil { - if err = json.Unmarshal(r.Module, &module); err != nil { - return nil, err - } - } else if module, err = ctrl.module.FindByID(ns.ID, r.ModuleID); err != nil { - return nil, err - } - - // Unmarshal given record or find existing one from ID - if r.Record != nil { - if err = json.Unmarshal(r.Record, &record); err != nil { - return nil, err - } - } else if record, err = ctrl.record.FindByID(ns.ID, r.RecordID); err != nil { - return nil, err - } - - if err = ctrl.runner.RecordManual(ctx, r.ScriptID, ns, module, record); err != nil { + if err = ctrl.runner.RecordManual(ctx, r.ScriptID, ns, m, record); err != nil { return nil, err } // When record was passed return it. if record != nil { - // (ab)user payload maker from record controller - // @todo find a way how to solve this more elegantly. - return (Record{ - ac: service.DefaultAccessControl, - }).makePayload(ctx, module, record, nil) + rval.Record = record + + if err != nil { + return nil, err + } } - return resputil.OK(), nil + return rval, err +} + +func (ctrl AutomationScript) Test(ctx context.Context, r *request.AutomationScriptTest) (interface{}, error) { + var ( + rval automationScriptRun + ns, m, record, err = ctrl.loadRecordScriptRunningCombo(r.NamespaceID, r.ModuleID, 0, r.Record) + ) + + if err != nil { + return nil, err + } + + if err = ctrl.runner.RecordScriptTester(ctx, r.Source, ns, m, record); err != nil { + return nil, err + } + + // When record was passed return it. + if record != nil { + rval.Record = record + + if err != nil { + return nil, err + } + } + + return rval, err +} + +func (ctrl AutomationScript) loadRecordScriptRunningCombo(namespaceID, moduleID, recordID uint64, record json.RawMessage) (ns *types.Namespace, m *types.Module, r *types.Record, err error) { + r = &types.Record{} + + // Load requested namespace + if ns, err = ctrl.namespace.FindByID(namespaceID); err != nil { + return + } + + if moduleID > 0 { + // Unmarshal given module or find existing one from ID + if m, err = ctrl.module.FindByID(ns.ID, moduleID); err != nil { + return + } + } + + // Unmarshal given record or find existing one from ID + if record != nil { + if err = json.Unmarshal(record, &r); err != nil { + err = errors.New("Could not parse record payload") + return + } + } else if r, err = ctrl.record.FindByID(ns.ID, recordID); err != nil { + return + } + + return } func (ctrl AutomationScript) makePayload(ctx context.Context, s *automation.Script, err error) (*automationScriptPayload, error) { diff --git a/compose/rest/handlers/automation_script.go b/compose/rest/handlers/automation_script.go index 6f7c2d414..05a7e44a3 100644 --- a/compose/rest/handlers/automation_script.go +++ b/compose/rest/handlers/automation_script.go @@ -36,6 +36,7 @@ type AutomationScriptAPI interface { Delete(context.Context, *request.AutomationScriptDelete) (interface{}, error) Runnable(context.Context, *request.AutomationScriptRunnable) (interface{}, error) Run(context.Context, *request.AutomationScriptRun) (interface{}, error) + Test(context.Context, *request.AutomationScriptTest) (interface{}, error) } // HTTP API interface @@ -47,6 +48,7 @@ type AutomationScript struct { Delete func(http.ResponseWriter, *http.Request) Runnable func(http.ResponseWriter, *http.Request) Run func(http.ResponseWriter, *http.Request) + Test func(http.ResponseWriter, *http.Request) } func NewAutomationScript(h AutomationScriptAPI) *AutomationScript { @@ -191,6 +193,26 @@ func NewAutomationScript(h AutomationScriptAPI) *AutomationScript { resputil.JSON(w, value) } }, + Test: func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + params := request.NewAutomationScriptTest() + if err := params.Fill(r); err != nil { + logger.LogParamError("AutomationScript.Test", r, err) + resputil.JSON(w, err) + return + } + + value, err := h.Test(r.Context(), params) + if err != nil { + logger.LogControllerError("AutomationScript.Test", r, err, params.Auditable()) + resputil.JSON(w, err) + return + } + logger.LogControllerCall("AutomationScript.Test", r, params.Auditable()) + if !serveHTTP(value, w, r) { + resputil.JSON(w, value) + } + }, } } @@ -203,6 +225,7 @@ func (h AutomationScript) MountRoutes(r chi.Router, middlewares ...func(http.Han r.Post("/namespace/{namespaceID}/automation/script/{scriptID}", h.Update) r.Delete("/namespace/{namespaceID}/automation/script/{scriptID}", h.Delete) r.Get("/namespace/{namespaceID}/automation/script/runnable", h.Runnable) - r.Post("/namespace/{namespaceID}/automation/script/run", h.Run) + r.Post("/namespace/{namespaceID}/automation/script/{scriptID}/run", h.Run) + r.Post("/namespace/{namespaceID}/automation/script/test", h.Test) }) } diff --git a/compose/rest/record.go b/compose/rest/record.go index b4a59c588..df57fe924 100644 --- a/compose/rest/record.go +++ b/compose/rest/record.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" "github.com/cortezaproject/corteza-server/compose/encoder" + "github.com/cortezaproject/corteza-server/compose/internal/repository" "github.com/cortezaproject/corteza-server/compose/internal/service" "github.com/cortezaproject/corteza-server/compose/rest/request" "github.com/cortezaproject/corteza-server/compose/types" @@ -94,6 +95,11 @@ func (ctrl *Record) Read(ctx context.Context, r *request.RecordRead) (interface{ record, err := ctrl.record.With(ctx).FindByID(r.NamespaceID, r.RecordID) + // Temp workaround until we do proper by-module filtering for record findByID + if record != nil && record.ModuleID != r.ModuleID { + return nil, repository.ErrRecordNotFound + } + return ctrl.makePayload(ctx, m, record, err) } diff --git a/compose/rest/request/automation_script.go b/compose/rest/request/automation_script.go index 9e5f889bc..52170c0f5 100644 --- a/compose/rest/request/automation_script.go +++ b/compose/rest/request/automation_script.go @@ -479,12 +479,10 @@ var _ RequestFiller = NewAutomationScriptRunnable() // AutomationScript run request parameters type AutomationScriptRun struct { ScriptID uint64 `json:",string"` - Source string + NamespaceID uint64 `json:",string"` ModuleID uint64 `json:",string"` RecordID uint64 `json:",string"` - Module json.RawMessage Record json.RawMessage - NamespaceID uint64 `json:",string"` } func NewAutomationScriptRun() *AutomationScriptRun { @@ -495,12 +493,10 @@ func (r AutomationScriptRun) Auditable() map[string]interface{} { var out = map[string]interface{}{} out["scriptID"] = r.ScriptID - out["source"] = r.Source + out["namespaceID"] = r.NamespaceID out["moduleID"] = r.ModuleID out["recordID"] = r.RecordID - out["module"] = r.Module out["record"] = r.Record - out["namespaceID"] = r.NamespaceID return out } @@ -532,20 +528,78 @@ func (r *AutomationScriptRun) Fill(req *http.Request) (err error) { post[name] = string(param[0]) } - if val, ok := post["scriptID"]; ok { - r.ScriptID = parseUInt64(val) - } - if val, ok := post["source"]; ok { - r.Source = val - } + r.ScriptID = parseUInt64(chi.URLParam(req, "scriptID")) + r.NamespaceID = parseUInt64(chi.URLParam(req, "namespaceID")) if val, ok := post["moduleID"]; ok { r.ModuleID = parseUInt64(val) } if val, ok := post["recordID"]; ok { r.RecordID = parseUInt64(val) } - if val, ok := post["module"]; ok { - r.Module = json.RawMessage(val) + if val, ok := post["record"]; ok { + r.Record = json.RawMessage(val) + } + + return err +} + +var _ RequestFiller = NewAutomationScriptRun() + +// AutomationScript test request parameters +type AutomationScriptTest struct { + Source string + ModuleID uint64 `json:",string"` + Record json.RawMessage + NamespaceID uint64 `json:",string"` +} + +func NewAutomationScriptTest() *AutomationScriptTest { + return &AutomationScriptTest{} +} + +func (r AutomationScriptTest) Auditable() map[string]interface{} { + var out = map[string]interface{}{} + + out["source"] = r.Source + out["moduleID"] = r.ModuleID + out["record"] = r.Record + out["namespaceID"] = r.NamespaceID + + return out +} + +func (r *AutomationScriptTest) Fill(req *http.Request) (err error) { + if strings.ToLower(req.Header.Get("content-type")) == "application/json" { + err = json.NewDecoder(req.Body).Decode(r) + + switch { + case err == io.EOF: + err = nil + case err != nil: + return errors.Wrap(err, "error parsing http request body") + } + } + + if err = req.ParseForm(); err != nil { + return err + } + + get := map[string]string{} + post := map[string]string{} + urlQuery := req.URL.Query() + for name, param := range urlQuery { + get[name] = string(param[0]) + } + postVars := req.Form + for name, param := range postVars { + post[name] = string(param[0]) + } + + if val, ok := post["source"]; ok { + r.Source = val + } + if val, ok := post["moduleID"]; ok { + r.ModuleID = parseUInt64(val) } if val, ok := post["record"]; ok { r.Record = json.RawMessage(val) @@ -555,4 +609,4 @@ func (r *AutomationScriptRun) Fill(req *http.Request) (err error) { return err } -var _ RequestFiller = NewAutomationScriptRun() +var _ RequestFiller = NewAutomationScriptTest() diff --git a/docs/compose/README.md b/docs/compose/README.md index a481b1e5e..7608a4d76 100644 --- a/docs/compose/README.md +++ b/docs/compose/README.md @@ -121,7 +121,8 @@ | `POST` | `/namespace/{namespaceID}/automation/script/{scriptID}` | Update automation script | | `DELETE` | `/namespace/{namespaceID}/automation/script/{scriptID}` | Delete script | | `GET` | `/namespace/{namespaceID}/automation/script/runnable` | List of runnable (event=manual) scripts (executable on the backend or from user-agent/browser) | -| `POST` | `/namespace/{namespaceID}/automation/script/run` | Run a specific script or code at the backend. For testing and manual execution | +| `POST` | `/namespace/{namespaceID}/automation/script/{scriptID}/run` | Run a specific script or code at the backend. Used for running script manually | +| `POST` | `/namespace/{namespaceID}/automation/script/test` | Run source code in corredor. Used for testing | ## List/read automation script @@ -237,23 +238,38 @@ | condition | string | GET | Trigger condition | N/A | NO | | namespaceID | uint64 | PATH | Namespace ID | N/A | YES | -## Run a specific script or code at the backend. For testing and manual execution +## Run a specific script or code at the backend. Used for running script manually #### Method | URI | Protocol | Method | Authentication | | --- | -------- | ------ | -------------- | -| `/namespace/{namespaceID}/automation/script/run` | HTTP/S | POST | Client ID, Session ID | +| `/namespace/{namespaceID}/automation/script/{scriptID}/run` | HTTP/S | POST | Client ID, Session ID | #### Request parameters | Parameter | Type | Method | Description | Default | Required? | | --------- | ---- | ------ | ----------- | ------- | --------- | -| scriptID | uint64 | POST | Script ID (scripts with manual triggers) | N/A | NO | -| source | string | POST | Script's source code (overrides scriptID parameter) | N/A | NO | +| scriptID | uint64 | PATH | | N/A | YES | +| namespaceID | uint64 | PATH | Namespace ID | N/A | YES | +| moduleID | uint64 | POST | ModuleID to be used | N/A | NO | +| recordID | uint64 | POST | RecordID to be used | N/A | NO | +| record | json.RawMessage | POST | Record payload to be used (instead of specific record when using recordID) | N/A | NO | + +## Run source code in corredor. Used for testing + +#### Method + +| URI | Protocol | Method | Authentication | +| --- | -------- | ------ | -------------- | +| `/namespace/{namespaceID}/automation/script/test` | HTTP/S | POST | Client ID, Session ID | + +#### Request parameters + +| Parameter | Type | Method | Description | Default | Required? | +| --------- | ---- | ------ | ----------- | ------- | --------- | +| source | string | POST | Script's source code | N/A | NO | | moduleID | uint64 | POST | Preload module and pass it to the automation script | N/A | NO | -| recordID | uint64 | POST | Preload record and pass it to the automation script | N/A | NO | -| module | json.RawMessage | POST | Module to pass to the automation script | N/A | NO | | record | json.RawMessage | POST | Record to pass to the automation script | N/A | NO | | namespaceID | uint64 | PATH | Namespace ID | N/A | YES |