3
0

Support recording and searching actionlogs

This commit is contained in:
Denis Arh 2021-10-26 17:13:12 +02:00
parent 84762e068b
commit 402006bebb
17 changed files with 838 additions and 11 deletions

View File

@ -6,8 +6,13 @@ func MakeDebugPolicy() policyMatcher {
func MakeProductionPolicy() policyMatcher {
return NewPolicyAll(
// Ignore debug actions
NewPolicyNegate(NewPolicyMatchSeverity(Debug, Info)),
NewPolicyAny(
// Match all actions from automation
NewPolicyMatchRequestOrigin(RequestOrigin_Automation),
// Ignore debug actions
NewPolicyNegate(NewPolicyMatchSeverity(Debug, Info)),
),
)
}

View File

@ -16,6 +16,7 @@ const (
RequestOrigin_API_REST = "api/rest"
RequestOrigin_API_GRPC = "api/grpc"
RequestOrigin_Auth = "auth"
RequestOrigin_Automation = "automation"
)
// RequestOriginKey is the key that holds th unique request ID in a request context.

View File

@ -2,10 +2,10 @@ package actionlog
import (
"context"
"github.com/cortezaproject/corteza-server/pkg/id"
"strings"
"time"
"github.com/cortezaproject/corteza-server/pkg/id"
"github.com/go-chi/chi/middleware"
"go.uber.org/zap"
@ -65,7 +65,6 @@ func (svc service) Record(ctx context.Context, a *Action) {
a.ID = id.Next()
svc.log(a)
if !svc.policy.Match(a) {
// policy does not allow us to record this
return

View File

@ -59,6 +59,7 @@ type (
BeforeActionID uint64 `json:"beforeActionID"`
ActorID []uint64 `json:"actorID"`
Origin string `json:"origin"`
Resource string `json:"resource"`
Action string `json:"action"`
Limit uint `json:"limit"`
@ -69,9 +70,11 @@ type (
}
)
// Severity constants
//
// not using log/syslog LOG_* constants as they are only
// available outside windows env.
const (
// Not using log/syslog LOG_* constants as they are only
// available outside windows env.
Emergency Severity = iota
Alert
Critical
@ -203,3 +206,24 @@ func (s Severity) String() string {
return ""
}
func NewSeverity(s string) Severity {
switch s {
case "emergency":
return Emergency
case "alert":
return Alert
case "critical":
return Critical
case "err":
return Error
case "warning":
return Warning
case "notice":
return Notice
case "info":
return Info
}
return Debug
}

View File

@ -10,16 +10,18 @@ package options
type (
ActionLogOpt struct {
Enabled bool `env:"ACTIONLOG_ENABLED"`
Debug bool `env:"ACTIONLOG_DEBUG"`
Enabled bool `env:"ACTIONLOG_ENABLED"`
Debug bool `env:"ACTIONLOG_DEBUG"`
WorkflowFunctionsEnabled bool `env:"ACTIONLOG_WORKFLOW_FUNCTIONS_ENABLED"`
}
)
// ActionLog initializes and returns a ActionLogOpt with default values
func ActionLog() (o *ActionLogOpt) {
o = &ActionLogOpt{
Enabled: true,
Debug: false,
Enabled: true,
Debug: false,
WorkflowFunctionsEnabled: false,
}
fill(o)

View File

@ -13,3 +13,9 @@ props:
default: false
docs:
description: Enable debug action logging.
- name: workflowFunctionsEnabled
type: bool
default: false
docs:
description: Enable workflow function for searching and recording actions

View File

@ -2,6 +2,7 @@ package rdbms
import (
"encoding/json"
"github.com/Masterminds/squirrel"
"github.com/cortezaproject/corteza-server/pkg/actionlog"
"github.com/cortezaproject/corteza-server/store"
@ -29,6 +30,10 @@ func (s Store) convertActionlogFilter(f actionlog.Filter) (query squirrel.Select
query = query.Where(squirrel.Eq{"actor_id": f.ActorID})
}
if f.Origin != "" {
query = query.Where(squirrel.Eq{"origin": f.Origin})
}
if f.Resource != "" {
query = query.Where(squirrel.Eq{"resource": f.Resource})
}

377
system/automation/actionlog_handler.gen.go generated Normal file
View File

@ -0,0 +1,377 @@
package automation
// This file is auto-generated.
//
// Changes to this file may cause incorrect behavior and will be lost if
// the code is regenerated.
//
// Definitions file that controls how this file is generated:
// system/automation/actionlog_handler.yaml
import (
"context"
atypes "github.com/cortezaproject/corteza-server/automation/types"
"github.com/cortezaproject/corteza-server/pkg/actionlog"
"github.com/cortezaproject/corteza-server/pkg/expr"
"github.com/cortezaproject/corteza-server/pkg/wfexec"
"time"
)
var _ wfexec.ExecResponse
type (
actionlogHandlerRegistry interface {
AddFunctions(ff ...*atypes.Function)
Type(ref string) expr.Type
}
)
func (h actionlogHandler) register() {
h.reg.AddFunctions(
h.Search(),
h.Each(),
h.Record(),
)
}
type (
actionlogSearchArgs struct {
hasFromTimestamp bool
FromTimestamp *time.Time
hasToTimestamp bool
ToTimestamp *time.Time
hasBeforeActionID bool
BeforeActionID uint64
hasActorID bool
ActorID uint64
hasOrigin bool
Origin string
hasResource bool
Resource string
hasAction bool
Action string
hasLimit bool
Limit uint64
}
actionlogSearchResults struct {
Actions []*actionlog.Action
}
)
// Search function Action log search
//
// expects implementation of search function:
// func (h actionlogHandler) search(ctx context.Context, args *actionlogSearchArgs) (results *actionlogSearchResults, err error) {
// return
// }
func (h actionlogHandler) Search() *atypes.Function {
return &atypes.Function{
Ref: "actionlogSearch",
Kind: "function",
Labels: map[string]string{"actionlog": "step,workflow"},
Meta: &atypes.FunctionMeta{
Short: "Action log search",
},
Parameters: []*atypes.Param{
{
Name: "fromTimestamp",
Types: []string{"DateTime"},
},
{
Name: "toTimestamp",
Types: []string{"DateTime"},
},
{
Name: "beforeActionID",
Types: []string{"ID"},
},
{
Name: "actorID",
Types: []string{"ID"},
},
{
Name: "origin",
Types: []string{"String"},
},
{
Name: "resource",
Types: []string{"String"},
},
{
Name: "action",
Types: []string{"String"},
},
{
Name: "limit",
Types: []string{"UnsignedInteger"},
},
},
Results: []*atypes.Param{
{
Name: "actions",
Types: []string{"Action"},
IsArray: true,
},
},
Handler: func(ctx context.Context, in *expr.Vars) (out *expr.Vars, err error) {
var (
args = &actionlogSearchArgs{
hasFromTimestamp: in.Has("fromTimestamp"),
hasToTimestamp: in.Has("toTimestamp"),
hasBeforeActionID: in.Has("beforeActionID"),
hasActorID: in.Has("actorID"),
hasOrigin: in.Has("origin"),
hasResource: in.Has("resource"),
hasAction: in.Has("action"),
hasLimit: in.Has("limit"),
}
)
if err = in.Decode(args); err != nil {
return
}
var results *actionlogSearchResults
if results, err = h.search(ctx, args); err != nil {
return
}
out = &expr.Vars{}
{
// converting results.Actions (*actionlog.Action) to Array (of Action)
var (
tval expr.TypedValue
tarr = make([]expr.TypedValue, len(results.Actions))
)
for i := range results.Actions {
if tarr[i], err = h.reg.Type("Action").Cast(results.Actions[i]); err != nil {
return
}
}
if tval, err = expr.NewArray(tarr); err != nil {
return
} else if err = expr.Assign(out, "actions", tval); err != nil {
return
}
}
return
},
}
}
type (
actionlogEachArgs struct {
hasFromTimestamp bool
FromTimestamp *time.Time
hasToTimestamp bool
ToTimestamp *time.Time
hasBeforeActionID bool
BeforeActionID uint64
hasActorID bool
ActorID uint64
hasOrigin bool
Origin string
hasResource bool
Resource string
hasAction bool
Action string
hasLimit bool
Limit uint64
}
actionlogEachResults struct {
Action *actionlog.Action
}
)
// Each function Action log
//
// expects implementation of each function:
// func (h actionlogHandler) each(ctx context.Context, args *actionlogEachArgs) (results *actionlogEachResults, err error) {
// return
// }
func (h actionlogHandler) Each() *atypes.Function {
return &atypes.Function{
Ref: "actionlogEach",
Kind: "iterator",
Labels: map[string]string{"actionlog": "step,workflow"},
Meta: &atypes.FunctionMeta{
Short: "Action log",
},
Parameters: []*atypes.Param{
{
Name: "fromTimestamp",
Types: []string{"DateTime"},
},
{
Name: "toTimestamp",
Types: []string{"DateTime"},
},
{
Name: "beforeActionID",
Types: []string{"ID"},
},
{
Name: "actorID",
Types: []string{"ID"},
},
{
Name: "origin",
Types: []string{"String"},
},
{
Name: "resource",
Types: []string{"String"},
},
{
Name: "action",
Types: []string{"String"},
},
{
Name: "limit",
Types: []string{"UnsignedInteger"},
},
},
Results: []*atypes.Param{
{
Name: "action",
Types: []string{"Action"},
},
},
Iterator: func(ctx context.Context, in *expr.Vars) (out wfexec.IteratorHandler, err error) {
var (
args = &actionlogEachArgs{
hasFromTimestamp: in.Has("fromTimestamp"),
hasToTimestamp: in.Has("toTimestamp"),
hasBeforeActionID: in.Has("beforeActionID"),
hasActorID: in.Has("actorID"),
hasOrigin: in.Has("origin"),
hasResource: in.Has("resource"),
hasAction: in.Has("action"),
hasLimit: in.Has("limit"),
}
)
if err = in.Decode(args); err != nil {
return
}
return h.each(ctx, args)
},
}
}
type (
actionlogRecordArgs struct {
hasAction bool
Action string
hasResource bool
Resource string
hasError bool
Error string
hasSeverity bool
Severity string
hasDescription bool
Description string
hasMeta bool
Meta *expr.Vars
}
)
// Record function Record action into action log
//
// expects implementation of record function:
// func (h actionlogHandler) record(ctx context.Context, args *actionlogRecordArgs) (err error) {
// return
// }
func (h actionlogHandler) Record() *atypes.Function {
return &atypes.Function{
Ref: "actionlogRecord",
Kind: "function",
Labels: map[string]string{"actionlog": "step,workflow"},
Meta: &atypes.FunctionMeta{
Short: "Record action into action log",
},
Parameters: []*atypes.Param{
{
Name: "action",
Types: []string{"String"},
},
{
Name: "resource",
Types: []string{"String"},
},
{
Name: "error",
Types: []string{"String"},
},
{
Name: "severity",
Types: []string{"String"},
Meta: &atypes.ParamMeta{
Visual: map[string]interface{}{"options": []interface{}{"emergency", "alert", "critical", "err", "warning", "notice", "info", "debug"}},
},
},
{
Name: "description",
Types: []string{"String"},
},
{
Name: "meta",
Types: []string{"Vars"},
},
},
Handler: func(ctx context.Context, in *expr.Vars) (out *expr.Vars, err error) {
var (
args = &actionlogRecordArgs{
hasAction: in.Has("action"),
hasResource: in.Has("resource"),
hasError: in.Has("error"),
hasSeverity: in.Has("severity"),
hasDescription: in.Has("description"),
hasMeta: in.Has("meta"),
}
)
if err = in.Decode(args); err != nil {
return
}
return out, h.record(ctx, args)
},
}
}

View File

@ -0,0 +1,151 @@
package automation
import (
"context"
"github.com/cortezaproject/corteza-server/pkg/actionlog"
. "github.com/cortezaproject/corteza-server/pkg/expr"
"github.com/cortezaproject/corteza-server/pkg/wfexec"
)
type (
actionlogHandler struct {
reg actionlogHandlerRegistry
svc actionlog.Recorder
}
actionSetIterator struct {
// Item buffer, current item pointer, and total items traversed
ptr uint
buffer actionlog.ActionSet
total uint
// When filter limit is set, this constraints it
iterLimit uint
useIterLimit bool
// Item loader for additional chunks
filter actionlog.Filter
loader func() error
}
)
func ActionlogHandler(reg actionlogHandlerRegistry, svc actionlog.Recorder) *actionlogHandler {
h := &actionlogHandler{
reg: reg,
svc: svc,
}
h.register()
return h
}
func (h actionlogHandler) search(ctx context.Context, args *actionlogSearchArgs) (results *actionlogSearchResults, err error) {
results = &actionlogSearchResults{}
var (
f = actionlog.Filter{
FromTimestamp: args.FromTimestamp,
ToTimestamp: args.ToTimestamp,
BeforeActionID: args.BeforeActionID,
Origin: args.Origin,
Resource: args.Resource,
Action: args.Action,
Limit: uint(args.Limit),
}
)
if args.ActorID > 0 {
f.ActorID = []uint64{args.ActorID}
}
results.Actions, _, err = h.svc.Find(ctx, f)
return
}
func (h actionlogHandler) each(ctx context.Context, args *actionlogEachArgs) (out wfexec.IteratorHandler, err error) {
var (
i = &actionSetIterator{}
f = actionlog.Filter{
FromTimestamp: args.FromTimestamp,
ToTimestamp: args.ToTimestamp,
BeforeActionID: args.BeforeActionID,
Origin: args.Origin,
Resource: args.Resource,
Action: args.Action,
}
)
if args.ActorID > 0 {
f.ActorID = []uint64{args.ActorID}
}
if args.hasLimit {
i.useIterLimit = true
i.iterLimit = uint(args.Limit)
if args.Limit > uint64(wfexec.MaxIteratorBufferSize) {
f.Limit = wfexec.MaxIteratorBufferSize
}
i.iterLimit = uint(args.Limit)
} else {
f.Limit = wfexec.MaxIteratorBufferSize
}
i.filter = f
i.loader = func() (err error) {
i.total += i.ptr
i.ptr = 0
if len(i.buffer) > 0 {
i.filter.BeforeActionID = i.buffer[len(i.buffer)-1].ID
}
i.buffer, i.filter, err = h.svc.Find(ctx, i.filter)
return
}
// Initial load
return i, i.loader()
}
func (h actionlogHandler) record(ctx context.Context, args *actionlogRecordArgs) (err error) {
a := &actionlog.Action{
Resource: args.Resource,
Action: args.Action,
Error: args.Error,
Severity: actionlog.NewSeverity(args.Severity),
Description: args.Description,
}
if args.Meta != nil {
a.Meta = args.Meta.Dict()
}
ctx = actionlog.RequestOriginToContext(ctx, actionlog.RequestOrigin_Automation)
h.svc.Record(ctx, a)
return nil
}
func (i *actionSetIterator) More(context.Context, *Vars) (bool, error) {
return wfexec.GenericResourceNextCheck(i.useIterLimit, i.ptr, uint(len(i.buffer)), i.total, i.iterLimit, i.iterLimit > uint(len(i.buffer))), nil
}
func (i *actionSetIterator) Start(context.Context, *Vars) error { i.ptr = 0; return nil }
func (i *actionSetIterator) Next(context.Context, *Vars) (out *Vars, err error) {
if len(i.buffer)-int(i.ptr) <= 0 {
if err = i.loader(); err != nil {
panic(err)
}
}
out = &Vars{}
out.Set("user", Must(NewUser(i.buffer[i.ptr])))
out.Set("index", Must(NewInteger(i.total+i.ptr)))
i.ptr++
return out, nil
}

View File

@ -0,0 +1,72 @@
imports:
- time
- github.com/cortezaproject/corteza-server/pkg/actionlog
snippets:
filterParams: &filterParams
fromTimestamp: { types: [ { wf: DateTime } ] }
toTimestamp: { types: [ { wf: DateTime } ] }
beforeActionID: { types: [ { wf: ID } ] }
actorID: { types: [ { wf: ID } ] }
origin: { types: [ { wf: String } ] }
resource: { types: [ { wf: String } ] }
action: { types: [ { wf: String } ] }
limit: { types: [ { wf: UnsignedInteger } ] }
rvAction: &rvAction
wf: Action
labels: &labels
actionlog: "step,workflow"
functions:
search:
meta:
short: Action log search
params: *filterParams
labels:
<<: *labels
results:
actions:
<<: *rvAction
isArray: true
each:
kind: iterator
meta:
short: Action log
params: *filterParams
labels:
<<: *labels
results:
action: *rvAction
record:
meta:
short: Record action into action log
labels:
<<: *labels
params:
action:
types: [ { wf: String } ]
resource:
types: [ { wf: String } ]
error:
types: [ { wf: String } ]
severity:
types: [ { wf: String } ]
meta:
visual:
options:
- "emergency"
- "alert"
- "critical"
- "err"
- "warning"
- "notice"
- "info"
- "debug"
description:
types: [ { wf: String } ]
meta:
types: [ { wf: Vars, go: '*expr.Vars' } ]

View File

@ -11,6 +11,7 @@ package automation
import (
"context"
"fmt"
"github.com/cortezaproject/corteza-server/pkg/actionlog"
. "github.com/cortezaproject/corteza-server/pkg/expr"
"github.com/cortezaproject/corteza-server/pkg/rbac"
"github.com/cortezaproject/corteza-server/system/types"
@ -20,6 +21,55 @@ import (
var _ = context.Background
var _ = fmt.Errorf
// Action is an expression type, wrapper for *actionlog.Action type
type Action struct {
value *actionlog.Action
mux sync.RWMutex
}
// NewAction creates new instance of Action expression type
func NewAction(val interface{}) (*Action, error) {
if c, err := CastToAction(val); err != nil {
return nil, fmt.Errorf("unable to create Action: %w", err)
} else {
return &Action{value: c}, nil
}
}
// Get return underlying value on Action
func (t *Action) Get() interface{} {
t.mux.RLock()
defer t.mux.RUnlock()
return t.value
}
// GetValue returns underlying value on Action
func (t *Action) GetValue() *actionlog.Action {
t.mux.RLock()
defer t.mux.RUnlock()
return t.value
}
// Type return type name
func (Action) Type() string { return "Action" }
// Cast converts value to *actionlog.Action
func (Action) Cast(val interface{}) (TypedValue, error) {
return NewAction(val)
}
// Assign new value to Action
//
// value is first passed through CastToAction
func (t *Action) Assign(val interface{}) error {
if c, err := CastToAction(val); err != nil {
return err
} else {
t.value = c
return nil
}
}
// DocumentType is an expression type, wrapper for types.DocumentType type
type DocumentType struct {
value types.DocumentType

View File

@ -6,6 +6,7 @@ import (
"io"
"io/ioutil"
"github.com/cortezaproject/corteza-server/pkg/actionlog"
"github.com/cortezaproject/corteza-server/pkg/expr"
"github.com/cortezaproject/corteza-server/pkg/rbac"
"github.com/cortezaproject/corteza-server/system/types"
@ -198,3 +199,12 @@ func CastToRbacResource(val interface{}) (out rbac.Resource, err error) {
return nil, fmt.Errorf("unable to cast type %T to %T", val, out)
}
}
func CastToAction(val interface{}) (out *actionlog.Action, err error) {
switch val := expr.UntypedValue(val).(type) {
case *actionlog.Action:
return val, nil
default:
return nil, fmt.Errorf("unable to cast type %T to %T", val, out)
}
}

View File

@ -2,6 +2,7 @@ package: automation
imports:
- github.com/cortezaproject/corteza-server/system/types
- github.com/cortezaproject/corteza-server/pkg/rbac
- github.com/cortezaproject/corteza-server/pkg/actionlog
types:
Template:
@ -73,3 +74,6 @@ types:
RbacResource:
as: 'rbac.Resource'
Action:
as: '*actionlog.Action'

View File

@ -217,6 +217,18 @@ func Initialize(ctx context.Context, log *zap.Logger, s store.Storer, ws websock
DefaultRole,
)
if c.ActionLog.WorkflowFunctionsEnabled {
// register action-log functions & types only when enabled
automation.ActionlogHandler(
automationService.Registry(),
DefaultActionlog,
)
automationService.Registry().AddTypes(
automation.Action{},
)
}
return
}

View File

@ -0,0 +1,49 @@
package workflows
import (
"context"
"testing"
automationService "github.com/cortezaproject/corteza-server/automation/service"
"github.com/cortezaproject/corteza-server/automation/types"
"github.com/cortezaproject/corteza-server/pkg/actionlog"
"github.com/cortezaproject/corteza-server/pkg/logger"
"github.com/cortezaproject/corteza-server/system/automation"
"github.com/stretchr/testify/require"
)
func Test_actionlog_functions(t *testing.T) {
var (
ctx = bypassRBAC(context.Background())
req = require.New(t)
)
req.NoError(defStore.TruncateActionlogs(ctx))
//// register action log with storage backend
automation.ActionlogHandler(
automationService.Registry(),
actionlog.NewService(defStore, logger.Default(), logger.Default(), actionlog.NewPolicyAll()),
)
loadNewScenario(ctx, t)
var (
aux = struct {
Actions actionlog.ActionSet
}{}
)
vars, _ := mustExecWorkflow(ctx, t, "logger", types.WorkflowExecParams{})
req.NoError(vars.Decode(&aux))
// Expecting both, invoker & runner to be same as invoker
req.Len(aux.Actions, 2)
//undo action log registration
automation.ActionlogHandler(
automationService.Registry(),
automationService.DefaultActionlog,
)
}

View File

@ -42,7 +42,8 @@ func TestMain(m *testing.M) {
ctx := context.Background()
defApp = helpers.NewIntegrationTestApp(ctx, func(app *app.CortezaApp) (err error) {
//app.Opt.Workflow.ExecDebug = true
// some test suites require action-log enabled
app.Opt.ActionLog.WorkflowFunctionsEnabled = true
defStore = app.Store
eventbus.Set(eventBus)
return nil

View File

@ -0,0 +1,59 @@
workflows:
logger:
enabled: true
trace: true
triggers:
- enabled: true
stepID: 10
steps:
- stepID: 10
kind: function
ref: actionlogRecord
arguments:
- { target: action, type: String, value: "action" }
- { target: resource, type: String, value: "resource" }
- { target: error, type: String, value: "error" }
- { target: severity, type: String, value: "severity" }
- { target: description, type: String, value: "description" }
- stepID: 11
kind: function
ref: actionlogRecord
arguments:
- { target: action, type: String, value: "action" }
- { target: resource, type: String, value: "resource" }
- { target: error, type: String, value: "error" }
- { target: severity, type: String, value: "severity" }
- { target: description, type: String, value: "description" }
- stepID: 12
kind: function
ref: actionlogRecord
arguments:
- { target: action, type: String, value: "action" }
- { target: resource, type: String, value: "find-me" }
- { target: error, type: String, value: "error" }
- { target: severity, type: String, value: "severity" }
- { target: description, type: String, value: "description" }
- stepID: 13
kind: function
ref: actionlogRecord
arguments:
- { target: action, type: String, value: "action" }
- { target: resource, type: String, value: "find-me" }
- { target: error, type: String, value: "error" }
- { target: severity, type: String, value: "severity" }
- { target: description, type: String, value: "description" }
- stepID: 20
kind: function
ref: actionlogSearch
arguments:
- { target: resource, type: String, value: "find-me" }
results:
- { target: actions, expr: "actions" }
paths:
- { parentID: 10, childID: 11 }
- { parentID: 11, childID: 12 }
- { parentID: 12, childID: 13 }
- { parentID: 13, childID: 20 }