DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Improve corredor auto. server error handling

Browse Source 
Errors are now typified, node.js stack from error is accepted and converted
This commit is contained in:
Denis Arh
2020-11-05 07:57:27 +01:00
parent 111beae959
commit 92ca66cade
6 changed files with 142 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/boot_levels.go
View File

@@ -174,9 +174,6 @@ func (app *CortezaApp) InitServices(ctx context.Context) (err error) {
return
}
corredor.Service().SetUserFinder(sysService.DefaultUser)
corredor.Service().SetRoleFinder(sysService.DefaultRole)
{
// Initialize RBAC subsystem
// and (re)load rules from the storage backend
@@ -227,6 +224,9 @@ func (app *CortezaApp) InitServices(ctx context.Context) (err error) {
return
}
corredor.Service().SetUserFinder(sysService.DefaultUser)
corredor.Service().SetRoleFinder(sysService.DefaultRole)
// Initialize external authentication (from default settings)
external.Init()

2
compose/service/record.go
View File

@@ -1154,7 +1154,7 @@ func (svc record) Iterator(f types.RecordFilter, fn eventbus.HandlerFn, action s
err = func() error {
if err = fn(svc.ctx, event.RecordOnIteration(rec, nil, m, ns, nil)); err != nil {
if err.Error() != "Aborted" {
if errors.Is(err, corredor.ScriptExecAborted) {
// When script was softly aborted (return false),
// proceed with iteration but do not clone, update or delete
// current record!

121
pkg/corredor/service.go
View File

@@ -3,24 +3,23 @@ package corredor
import (
"context"
"fmt"
"strings"
"sync"
"time"
"github.com/go-chi/chi/middleware"
"github.com/pkg/errors"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
"github.com/cortezaproject/corteza-server/pkg/auth"
"github.com/cortezaproject/corteza-server/pkg/errors"
"github.com/cortezaproject/corteza-server/pkg/eventbus"
"github.com/cortezaproject/corteza-server/pkg/options"
"github.com/cortezaproject/corteza-server/pkg/rbac"
"github.com/cortezaproject/corteza-server/pkg/sentry"
"github.com/cortezaproject/corteza-server/system/types"
"github.com/davecgh/go-spew/spew"
"github.com/go-chi/chi/middleware"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
"strings"
"sync"
"time"
)
type (
@@ -142,6 +141,8 @@ var (
onManualEventType,
onIterationEventType,
}
ScriptExecAborted = errors.Plain(errors.KindAutomation, "aborted")
)
const (
@@ -299,19 +300,19 @@ func (svc service) ExecIterator(ctx context.Context, scriptName string) error {
)
if script = svc.sScripts.FindByName(scriptName); script == nil {
return errors.Errorf("nonexistent script (%q)", scriptName)
return fmt.Errorf("nonexistent script (%q)", scriptName)
}
if !svc.canExec(ctx, scriptName) {
return errors.Errorf("permission to execute %s denied", scriptName)
return fmt.Errorf("permission to execute %s denied", scriptName)
}
if script.Iterator == nil {
return errors.Errorf("not an itrator script")
return fmt.Errorf("not an itrator script")
}
if finder, ok := svc.iteratorProviders[script.Iterator.ResourceType]; !ok {
return errors.Errorf("unknown resource finder: %s", script.Iterator.ResourceType)
return fmt.Errorf("unknown resource finder: %s", script.Iterator.ResourceType)
} else {
if script.Security != nil {
runAs = script.Security.RunAs
@@ -319,7 +320,7 @@ func (svc service) ExecIterator(ctx context.Context, scriptName string) error {
if runAs != "" {
if !svc.opt.RunAsEnabled {
return errors.New("could not make runner context, run-as disabled")
return fmt.Errorf("could not make runner context, run-as disabled")
}
// Run this iterator as defined user
@@ -365,23 +366,23 @@ func (svc service) Exec(ctx context.Context, scriptName string, args ScriptArgs)
)
if len(scriptName) == 0 {
return errors.Errorf("script name not provided (%q)", scriptName)
return fmt.Errorf("script name not provided (%q)", scriptName)
}
if _, ok = svc.explicit[scriptName]; !ok {
return errors.Errorf("unregistered explicit script %q", scriptName)
return fmt.Errorf("unregistered explicit script %q", scriptName)
}
if _, ok = svc.explicit[scriptName][res]; !ok {
return errors.Errorf("unregistered explicit script %q for resource %q", scriptName, res)
return fmt.Errorf("unregistered explicit script %q for resource %q", scriptName, res)
}
if script = svc.sScripts.FindByName(scriptName); script == nil {
return errors.Errorf("nonexistent script (%q)", scriptName)
return fmt.Errorf("nonexistent script (%q)", scriptName)
}
if !svc.canExec(ctx, scriptName) {
return errors.Errorf("permission to execute %s denied", scriptName)
return fmt.Errorf("permission to execute %s denied", scriptName)
}
if script.Security != nil {
@@ -543,7 +544,7 @@ func (svc *service) processIterator(script *Script) (ptr uintptr, err error) {
}
if i.ResourceType == "" {
return 0, errors.Errorf("iterator resourceType not defined")
return 0, fmt.Errorf("iterator resourceType not defined")
}
log.Info(
@@ -561,11 +562,11 @@ func (svc *service) processIterator(script *Script) (ptr uintptr, err error) {
return
case onIntervalEventType, onTimestampEventType:
if len(i.Deferred) == 0 {
return 0, errors.Errorf("missing specification for interval/timestamp events")
return 0, fmt.Errorf("missing specification for interval/timestamp events")
}
if script.Security == nil {
return 0, errors.Errorf("can not schedule iterator without security descriptor")
return 0, fmt.Errorf("can not schedule iterator without security descriptor")
}
if p := strings.Index(i.ResourceType, ":"); p > 0 {
@@ -585,7 +586,7 @@ func (svc *service) processIterator(script *Script) (ptr uintptr, err error) {
eventbus.Constraint(eventbus.MustMakeConstraint("", "", i.Deferred...)),
), nil
default:
return 0, errors.Errorf("incompatible event type (%s) for iterator", i.EventType)
return 0, fmt.Errorf("incompatible event type (%s) for iterator", i.EventType)
}
return
@@ -657,7 +658,7 @@ func (svc service) exec(ctx context.Context, script string, runAs string, args S
zap.String("runAs", runAs),
zap.String("args", args.EventType()),
zap.String("resource", args.ResourceType()),
)
).WithOptions(zap.AddStacktrace(zap.FatalLevel))
)
log.Debug("triggered")
@@ -681,6 +682,10 @@ func (svc service) exec(ctx context.Context, script string, runAs string, args S
// Resolve/expand invoker user details from the context (if present
if i := auth.GetIdentityFromContext(ctx); i.Valid() {
if svc.users == nil {
return fmt.Errorf("could not run automation script without configured user service")
}
invoker, err = svc.users.FindByAny(ctx, i)
if err != nil {
return err
@@ -695,7 +700,7 @@ func (svc service) exec(ctx context.Context, script string, runAs string, args S
if len(runAs) > 0 {
if !svc.opt.RunAsEnabled {
return errors.New("could not make runner context, run-as disabled")
return fmt.Errorf("could not make runner context, run-as disabled")
}
var definer auth.Identifiable
@@ -774,25 +779,61 @@ func (svc service) exec(ctx context.Context, script string, runAs string, args S
)
if err != nil {
// See if this was a "soft abort"
//
// This means, we do not make any logs of this just
// tell the caller that the call was aborted
s := status.Convert(err)
if s != nil && s.Code() == codes.Aborted {
// Special care for errors with Aborted code
if s != nil {
msg := s.Message()
if len(msg) == 0 {
// No extra message, fallback to "aborted"
msg = "Aborted"
}
switch s.Code() {
return errors.New(msg)
case codes.NotFound:
// When requested script is not found on automation server
return errors.NotFound(msg)
case codes.Aborted:
// Scripts can be (softly) aborted by terminating with:
// a) return false
// b) throw Error("Aborted")
//
// Both this terminations have the same result. In case when
// iterator script is aborted that will finalize the iterator
// without an error
return ScriptExecAborted.Apply(
errors.Meta("script", script),
)
case codes.Unknown:
// When script was aborted or an unknown (to gRPC proto) error occurred.
// This is always a hard error
return errors.Automation(msg).Apply(
errors.Meta("script", script),
errors.AddNodeStack(trailer.Get("stack")),
)
case codes.InvalidArgument:
// Automation server might yield INVALID_ARGUMENT status.
// This can be caused by JSON encoding and it is highly unlikely
// when arguments are prepared by the server
return errors.InvalidData(msg)
default:
// When script execution fails and it is not handled otherwise,
// automation server yields INTERNAL error status
//
// This error and any other one that do not fit the above rules
// are wrapped with an opaque error
return errors.Automation("automation server internal fault").Wrap(err)
}
}
log.Warn("corredor responded with error", zap.Error(err))
return errors.New("failed to execute corredor script")
return fmt.Errorf("internal automation server error: %w", err)
}
log.Info("executed", zap.Any("result", rsp.Result))
@@ -800,8 +841,8 @@ func (svc service) exec(ctx context.Context, script string, runAs string, args S
// //// //// //// //// //// //// //// //// //// //// //// //// //// //// //// //// //// //// //// //// //// ////
// @todo process metadata (log, errors, stacktrace)
// spew.Dump("grpc exec header", header)
// spew.Dump("grpc exec trailer", trailer)
//spew.Dump("grpc exec header", header)
//spew.Dump("grpc exec trailer", trailer)
if rsp.Result == nil {
// No results
@@ -899,7 +940,7 @@ func (svc *service) serverScriptSecurity(ctx context.Context, script *ServerScri
out := make([]*rbac.Rule, len(roles))
for i, role := range roles {
if r, err := svc.roles.FindByAny(ctx, role); err != nil {
return nil, errors.Wrapf(err, "could not load security role: %s", role)
return nil, fmt.Errorf("could not load security role: %s: %w", role, err)
} else {
out[i] = &rbac.Rule{
RoleID: r.ID,

23
pkg/errors/kind.go
View File

@@ -15,12 +15,6 @@ const (
// details unless in development environment
KindInternal kind = iota
// store error
KindStore
// object store (file upload/download)
KindObjStore
// Data validation error
KindInvalidData
@@ -42,6 +36,15 @@ const (
// External system failure
KindExternal
// store error
KindStore
// object store (file upload/download)
KindObjStore
// automation error
KindAutomation
)
// translates error kind into http status
@@ -110,6 +113,10 @@ func External(m string, aa ...interface{}) *Error {
return err(KindExternal, fmt.Sprintf(m, aa...))
}
func Automation(m string, aa ...interface{}) *Error {
return err(KindAutomation, fmt.Sprintf(m, aa...))
}
func IsKind(err error, k kind) bool {
t, ok := err.(*Error)
if !ok {
@@ -168,3 +175,7 @@ func IsUnauthenticated(err error) bool {
func IsExternal(err error) bool {
return IsKind(err, KindExternal)
}
func IsAutomation(err error) bool {
return IsKind(err, KindAutomation)
}

19
pkg/errors/modifiers.go
View File

@@ -5,7 +5,7 @@ type (
)
// Adds meta
func Meta(k, v interface{}) func(e *Error) {
func Meta(k, v interface{}) mfn {
return func(e *Error) {
if e.meta == nil {
e.meta = meta{}
@@ -16,7 +16,7 @@ func Meta(k, v interface{}) func(e *Error) {
}
// Trim all keys from meta
func MetaTrim(kk ...interface{}) func(e *Error) {
func MetaTrim(kk ...interface{}) mfn {
return func(e *Error) {
for _, k := range kk {
delete(e.meta, k)
@@ -26,7 +26,7 @@ func MetaTrim(kk ...interface{}) func(e *Error) {
// StackSkip skips n frames in the stack
//
func StackSkip(n int) func(e *Error) {
func StackSkip(n int) mfn {
return func(e *Error) {
if n > len(e.stack) {
e.stack = nil
@@ -37,7 +37,7 @@ func StackSkip(n int) func(e *Error) {
}
// StackTrim removes n frames from the end of the stack
func StackTrim(n int) func(e *Error) {
func StackTrim(n int) mfn {
return func(e *Error) {
if len(e.stack) < n {
e.stack = nil
@@ -48,7 +48,7 @@ func StackTrim(n int) func(e *Error) {
}
// StackTrimAtFn removes all frames after (inclusive) the (1st) function match
func StackTrimAtFn(fn string) func(e *Error) {
func StackTrimAtFn(fn string) mfn {
return func(e *Error) {
for i := range e.stack {
if i > 2 && e.stack[i].Func == fn {
@@ -60,8 +60,15 @@ func StackTrimAtFn(fn string) func(e *Error) {
}
// Wrap embeds error
func Wrap(w error) func(e *Error) {
func Wrap(w error) mfn {
return func(e *Error) {
e.wrap = w
}
}
// Converts and attaches node.js stack to error
func AddNodeStack(stack []string) mfn {
return func(e *Error) {
e.stack = convertNodeStack(stack)
}
}

27
pkg/errors/stack.go
View File

@@ -47,3 +47,30 @@ func collectStack(skip int) []*frame {
return stack
}
// Converts node stack trace (from Error().stack) to internal structure
//
// Node stack traces are received with errors from Corredor (node.js) automation server
func convertNodeStack(stack []string) []*frame {
// @todo
//(metadata.MD) (len=2) {
// (string) (len=12) "content-type": ([]string) (len=1 cap=1) {
// (string) (len=16) "application/grpc"
// },
// (string) (len=5) "stack": ([]string) (len=10 cap=10) {
// (string) (len=93) "Object.exec (/Users/darh/dev/corteza/server-corredor/usr/testing/server-scripts/foo.js:12:11)",
// (string) (len=127) "Object.<anonymous> (/Users/darh/dev/corteza/server-corredor/node_modules/@cortezaproject/corteza-js/src/corredor/exec.ts:26:35)",
// (string) (len=28) "Generator.next (<anonymous>)",
// (string) (len=100) "/Users/darh/dev/corteza/server-corredor/node_modules/@cortezaproject/corteza-js/dist/index.js:277:71",
// (string) (len=25) "new Promise (<anonymous>)",
// (string) (len=112) "__awaiter (/Users/darh/dev/corteza/server-corredor/node_modules/@cortezaproject/corteza-js/dist/index.js:273:12)",
// (string) (len=114) "Object.Exec (/Users/darh/dev/corteza/server-corredor/node_modules/@cortezaproject/corteza-js/dist/index.js:483:12)",
// (string) (len=97) "ServerScripts.exec (/Users/darh/dev/corteza/server-corredor/src/services/server-scripts.ts:86:17)",
// (string) (len=96) "Object.Exec (/Users/darh/dev/corteza/server-corredor/src/grpc-handlers/server-scripts.ts:142:11)",
// (string) (len=78) "/Users/darh/dev/corteza/server-corredor/node_modules/grpc/src/server.js:593:13"
// }
//}
return nil
}