diff --git a/compose/service/service.go b/compose/service/service.go index 0dae2245e..cca47fe2f 100644 --- a/compose/service/service.go +++ b/compose/service/service.go @@ -3,6 +3,7 @@ package service import ( "context" "errors" + "github.com/cortezaproject/corteza-server/pkg/healthcheck" "go.uber.org/zap" "time" @@ -78,6 +79,8 @@ var ( // Initializes compose-only services func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) { + hcd := healthcheck.Defaults() + var db = repository.DB(ctx) DefaultLogger = log.Named("service") @@ -145,8 +148,11 @@ func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) { log.Info("initializing store", zap.String("path", path), zap.Error(err)) + } + hcd.Add(store.Healthcheck(DefaultStore), "Store/Compose") + if err != nil { return err } diff --git a/corteza/corteza.go b/corteza/corteza.go index 5087e8341..60defc25e 100644 --- a/corteza/corteza.go +++ b/corteza/corteza.go @@ -2,6 +2,7 @@ package corteza import ( "context" + "github.com/cortezaproject/corteza-server/pkg/healthcheck" "time" "github.com/pkg/errors" @@ -30,6 +31,12 @@ type ( var _ app.Runnable = &App{} func (app *App) Setup(log *zap.Logger, opts *app.Options) (err error) { + hcd := healthcheck.Defaults() + hcd.Add(scheduler.Healthcheck, "Scheduler") + hcd.Add(mail.Healthcheck, "Mail") + hcd.Add(corredor.Healthcheck, "Corredor") + hcd.Add(db.Healthcheck(), "Database") + app.log = log app.opt = opts diff --git a/messaging/service/service.go b/messaging/service/service.go index f64a310b0..b1c62f788 100644 --- a/messaging/service/service.go +++ b/messaging/service/service.go @@ -2,6 +2,7 @@ package service import ( "context" + "github.com/cortezaproject/corteza-server/pkg/healthcheck" "time" "go.uber.org/zap" @@ -57,6 +58,7 @@ var ( ) func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) { + hcd := healthcheck.Defaults() DefaultLogger = log.Named("service") { @@ -130,6 +132,8 @@ func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) { } } + hcd.Add(store.Healthcheck(DefaultStore), "Store/Messaging") + DefaultEvent = Event(ctx) DefaultChannel = Channel(ctx) DefaultAttachment = Attachment(ctx, DefaultStore) diff --git a/pkg/api/server.go b/pkg/api/server.go index 7e42e8ebb..3c396f9db 100644 --- a/pkg/api/server.go +++ b/pkg/api/server.go @@ -2,6 +2,7 @@ package api import ( "context" + "github.com/cortezaproject/corteza-server/pkg/healthcheck" "net" "net/http" @@ -115,6 +116,10 @@ func (s server) Serve(ctx context.Context) { router.Get("/version", version.HttpHandler) } + if s.httpOpt.EnableHealthcheckRoute { + router.Get("/healthcheck", healthcheck.HttpHandler()) + } + go func() { srv := http.Server{ Handler: router, diff --git a/pkg/app/options/http.go b/pkg/app/options/http.go index 21441c283..82d8de3ca 100644 --- a/pkg/app/options/http.go +++ b/pkg/app/options/http.go @@ -11,8 +11,9 @@ type ( LogResponse bool `env:"HTTP_LOG_RESPONSE"` Tracing bool `env:"HTTP_ERROR_TRACING"` - EnableVersionRoute bool `env:"HTTP_ENABLE_VERSION_ROUTE"` - EnableDebugRoute bool `env:"HTTP_ENABLE_DEBUG_ROUTE"` + EnableHealthcheckRoute bool `env:"HTTP_ENABLE_HEALTHCHECK_ROUTE"` + EnableVersionRoute bool `env:"HTTP_ENABLE_VERSION_ROUTE"` + EnableDebugRoute bool `env:"HTTP_ENABLE_DEBUG_ROUTE"` EnableMetrics bool `env:"HTTP_METRICS"` MetricsServiceLabel string `env:"HTTP_METRICS_NAME"` @@ -33,15 +34,16 @@ type ( func HTTP(pfix string) (o *HTTPServerOpt) { o = &HTTPServerOpt{ - Addr: ":80", - LogRequest: false, - LogResponse: false, - Tracing: false, - EnableVersionRoute: true, - EnableDebugRoute: false, - EnableMetrics: false, - MetricsServiceLabel: "corteza", - MetricsUsername: "metrics", + Addr: ":80", + LogRequest: false, + LogResponse: false, + Tracing: false, + EnableHealthcheckRoute: true, + EnableVersionRoute: true, + EnableDebugRoute: false, + EnableMetrics: false, + MetricsServiceLabel: "corteza", + MetricsUsername: "metrics", // Reports panics to Sentry through HTTP middleware EnablePanicReporting: true, diff --git a/pkg/corredor/healthcheck.go b/pkg/corredor/healthcheck.go new file mode 100644 index 000000000..4ab7a2b59 --- /dev/null +++ b/pkg/corredor/healthcheck.go @@ -0,0 +1,24 @@ +package corredor + +import ( + "context" + "fmt" + "google.golang.org/grpc/connectivity" +) + +// Healtcheck for global +func Healthcheck(ctx context.Context) error { + if gCorredor == nil { + return fmt.Errorf("uninitialized") + } + + if !gCorredor.opt.Enabled { + return nil + } + + if state := gCorredor.conn.GetState(); state != connectivity.Ready { + return fmt.Errorf("connection is %s", state) + } + + return nil +} diff --git a/pkg/db/healthcheck.go b/pkg/db/healthcheck.go new file mode 100644 index 000000000..c1861e8fa --- /dev/null +++ b/pkg/db/healthcheck.go @@ -0,0 +1,23 @@ +package db + +import ( + "context" + "github.com/titpetric/factory" +) + +func Healthcheck() func(ctx context.Context) error { + const name = "default" + return func(ctx context.Context) error { + db, err := factory.Database.Get(name) + if err != nil { + return err + } + + err = db.PingContext(ctx) + if err != nil { + return err + } + + return nil + } +} diff --git a/pkg/healthcheck/check.go b/pkg/healthcheck/check.go new file mode 100644 index 000000000..4cd9d9b5a --- /dev/null +++ b/pkg/healthcheck/check.go @@ -0,0 +1,118 @@ +package healthcheck + +import ( + "bytes" + "context" + "fmt" + "io" + "strings" +) + +type ( + checkFn func(ctx context.Context) error + + Meta struct { + Label string + Description string + } + + check struct { + fn checkFn + *Meta + } + + result struct { + err error + *Meta + } + + results []*result + checks struct { + cc []*check + } +) + +var ( + defaults *checks +) + +func init() { + defaults = New() +} + +func Defaults() *checks { + return defaults +} + +func New() *checks { + return &checks{cc: []*check{}} +} + +// Add appends new check +func (c *checks) Add(fn checkFn, label string, description ...string) { + c.cc = append(c.cc, &check{fn, &Meta{Label: label, Description: strings.Join(description, "")}}) +} + +func (c checks) Run(ctx context.Context) results { + var rr = make([]*result, len(c.cc)) + + for i, c := range c.cc { + rr[i] = &result{c.fn(ctx), c.Meta} + } + + return rr +} + +func (rr results) Healthy() bool { + for _, c := range rr { + if c.err != nil { + return false + } + } + + return true +} + +func (rr results) String() string { + buf := &bytes.Buffer{} + + rr.WriteTo(buf) + + return buf.String() +} + +func (rr results) WriteTo(w io.Writer) { + var ( + p = func(f string, aa ...interface{}) { + _, _ = fmt.Fprintf(w, f, aa...) + } + ) + + for _, r := range rr { + if r.IsHealthy() { + p("PASS") + } else { + p("FAIL") + } + + p(" %s", r.Label) + + if !r.IsHealthy() { + p(": %v", r.Error()) + } + + p("\n") + } +} + +func (r *result) IsHealthy() bool { + return r != nil && r.err == nil +} + +func (r *result) Error() string { + if r == nil || r.err == nil { + return "" + } + + return r.err.Error() +} diff --git a/pkg/healthcheck/check_test.go b/pkg/healthcheck/check_test.go new file mode 100644 index 000000000..ab133dc98 --- /dev/null +++ b/pkg/healthcheck/check_test.go @@ -0,0 +1,68 @@ +package healthcheck + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_Healthy(t *testing.T) { + tests := []struct { + name string + checks []*check + healthy bool + string string + }{ + { + "should be healthy with handle for stringer output", + []*check{{func(ctx context.Context) error { return nil }, &Meta{Label: "check01"}}}, + true, + "PASS check01\n", + }, + { + "should handle multiple healthy checks", + []*check{ + {func(ctx context.Context) error { return nil }, &Meta{Label: "check01"}}, + {func(ctx context.Context) error { return nil }, &Meta{Label: "check02"}}, + }, + true, + "PASS check01\nPASS check02\n", + }, + { + "should handle healthy and unhealthy checks", + []*check{ + {func(ctx context.Context) error { return nil }, &Meta{Label: "check01"}}, + {func(ctx context.Context) error { return fmt.Errorf("x") }, &Meta{Label: "check02"}}, + {func(ctx context.Context) error { return nil }, &Meta{Label: "check03"}}, + }, + false, + "PASS check01\nFAIL check02: x\nPASS check03\n", + }, + { + "should handle labels", + []*check{ + {func(ctx context.Context) error { return nil }, &Meta{Label: "check01"}}, + {func(ctx context.Context) error { return nil }, &Meta{Label: "Pretty check"}}, + }, + true, + "PASS check01\nPASS Pretty check\n", + }, + { + "should handle empty check list", + []*check{}, + true, + "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := assert.New(t) + r := (&checks{cc: tt.checks}).Run(context.Background()) + a.Equal(tt.healthy, r.Healthy(), "healthy result failed") + a.Equal(tt.string, r.String(), "stringer output match failed") + }) + } +} diff --git a/pkg/healthcheck/http_handler.go b/pkg/healthcheck/http_handler.go new file mode 100644 index 000000000..d1675ca47 --- /dev/null +++ b/pkg/healthcheck/http_handler.go @@ -0,0 +1,16 @@ +package healthcheck + +import "net/http" + +func HttpHandler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + results := Defaults().Run(r.Context()) + if results.Healthy() { + w.WriteHeader(http.StatusOK) + } else { + w.WriteHeader(http.StatusInternalServerError) + } + + results.WriteTo(w) + } +} diff --git a/pkg/mail/healthcheck.go b/pkg/mail/healthcheck.go new file mode 100644 index 000000000..5db2a9a15 --- /dev/null +++ b/pkg/mail/healthcheck.go @@ -0,0 +1,19 @@ +package mail + +import ( + "context" + "fmt" +) + +// Healtcheck for (global) scheduler +func Healthcheck(ctx context.Context) error { + if defaultDialer == nil { + return fmt.Errorf("uninitialized") + } + + if defaultDialerError != nil { + return defaultDialerError + } + + return nil +} diff --git a/pkg/scheduler/healthcheck.go b/pkg/scheduler/healthcheck.go new file mode 100644 index 000000000..e2a3a50f0 --- /dev/null +++ b/pkg/scheduler/healthcheck.go @@ -0,0 +1,19 @@ +package scheduler + +import ( + "context" + "fmt" +) + +// Healtcheck for (global) scheduler +func Healthcheck(ctx context.Context) error { + if gScheduler == nil { + return fmt.Errorf("uninitialized") + } + + if gScheduler.ticker == nil { + return fmt.Errorf("stopped") + } + + return nil +} diff --git a/pkg/store/healthcheck.go b/pkg/store/healthcheck.go new file mode 100644 index 000000000..dcf1f54f5 --- /dev/null +++ b/pkg/store/healthcheck.go @@ -0,0 +1,11 @@ +package store + +import ( + "context" +) + +func Healthcheck(s Store) func(ctx context.Context) error { + return func(ctx context.Context) error { + return s.Healthcheck(ctx) + } +} diff --git a/pkg/store/interfaces.go b/pkg/store/interfaces.go index 05b72ce50..11c8c7ae3 100644 --- a/pkg/store/interfaces.go +++ b/pkg/store/interfaces.go @@ -1,6 +1,7 @@ package store import ( + "context" "io" ) @@ -19,4 +20,7 @@ type Store interface { // Open returns file handle Open(filename string) (io.ReadSeeker, error) + + // Healthcheck checks health status of the store + Healthcheck(ctx context.Context) error } diff --git a/pkg/store/minio/store.go b/pkg/store/minio/store.go index b0dc164f7..0e3a0ec52 100644 --- a/pkg/store/minio/store.go +++ b/pkg/store/minio/store.go @@ -1,6 +1,7 @@ package minio import ( + "context" "fmt" "io" @@ -119,3 +120,7 @@ func (s store) Open(name string) (io.ReadSeeker, error) { ServerSideEncryption: s.sse, }) } + +func (s *store) Healthcheck(ctx context.Context) error { + return nil +} diff --git a/pkg/store/plain/store.go b/pkg/store/plain/store.go index 161f91831..d6e8a54db 100644 --- a/pkg/store/plain/store.go +++ b/pkg/store/plain/store.go @@ -1,6 +1,8 @@ package plain import ( + "bytes" + "context" "fmt" "io" "path" @@ -96,3 +98,30 @@ func (s *store) Open(filename string) (io.ReadSeeker, error) { return s.fs.Open(filename) } + +func (s *store) Healthcheck(ctx context.Context) error { + var ( + fname = s.namespace + "/.healthcheck" + buf = &bytes.Buffer{} + ) + + if s == nil { + return fmt.Errorf("uninitialized") + } + + buf.Write([]byte("healthcheck")) + + if err := s.Save(fname, buf); err != nil { + return err + } + + if _, err := s.Open(fname); err != nil { + return err + } + + if err := s.Remove(fname); err != nil { + return err + } + + return nil +} diff --git a/system/service/service.go b/system/service/service.go index 2b7825016..5583d2db9 100644 --- a/system/service/service.go +++ b/system/service/service.go @@ -2,6 +2,7 @@ package service import ( "context" + "github.com/cortezaproject/corteza-server/pkg/healthcheck" "go.uber.org/zap" "github.com/cortezaproject/corteza-server/pkg/actionlog" @@ -92,6 +93,7 @@ var ( ) func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) { + hcd := healthcheck.Defaults() DefaultLogger = log.Named("service") { @@ -163,6 +165,8 @@ func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) { } } + hcd.Add(store.Healthcheck(DefaultStore), "Store/System") + DefaultAuthNotification = AuthNotification(ctx) DefaultAuth = Auth(ctx) DefaultUser = User(ctx)