Merge branch 'feature-healthcheck' into develop

compose/service/service.go

@@ -3,6 +3,7 @@ package service
 import (
 	"context"
 	"errors"
+	"github.com/cortezaproject/corteza-server/pkg/healthcheck"
 	"go.uber.org/zap"
 	"time"
 
@@ -78,6 +79,8 @@ var (
 
 // Initializes compose-only services
 func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) {
+	hcd := healthcheck.Defaults()
+
 	var db = repository.DB(ctx)
 
 	DefaultLogger = log.Named("service")
@@ -145,8 +148,11 @@ func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) {
 			log.Info("initializing store",
 				zap.String("path", path),
 				zap.Error(err))
+
 		}
 
+		hcd.Add(store.Healthcheck(DefaultStore), "Store/Compose")
+
 		if err != nil {
 			return err
 		}

corteza/corteza.go

@@ -2,6 +2,7 @@ package corteza
 
 import (
 	"context"
+	"github.com/cortezaproject/corteza-server/pkg/healthcheck"
 	"time"
 
 	"github.com/pkg/errors"
@@ -30,6 +31,12 @@ type (
 var _ app.Runnable = &App{}
 
 func (app *App) Setup(log *zap.Logger, opts *app.Options) (err error) {
+	hcd := healthcheck.Defaults()
+	hcd.Add(scheduler.Healthcheck, "Scheduler")
+	hcd.Add(mail.Healthcheck, "Mail")
+	hcd.Add(corredor.Healthcheck, "Corredor")
+	hcd.Add(db.Healthcheck(), "Database")
+
 	app.log = log
 	app.opt = opts
 

messaging/service/service.go

@@ -2,6 +2,7 @@ package service
 
 import (
 	"context"
+	"github.com/cortezaproject/corteza-server/pkg/healthcheck"
 	"time"
 
 	"go.uber.org/zap"
@@ -57,6 +58,7 @@ var (
 )
 
 func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) {
+	hcd := healthcheck.Defaults()
 	DefaultLogger = log.Named("service")
 
 	{
@@ -130,6 +132,8 @@ func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) {
 		}
 	}
 
+	hcd.Add(store.Healthcheck(DefaultStore), "Store/Messaging")
+
 	DefaultEvent = Event(ctx)
 	DefaultChannel = Channel(ctx)
 	DefaultAttachment = Attachment(ctx, DefaultStore)

pkg/api/server.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"context"
+	"github.com/cortezaproject/corteza-server/pkg/healthcheck"
 	"net"
 	"net/http"
 
@@ -115,6 +116,10 @@ func (s server) Serve(ctx context.Context) {
 		router.Get("/version", version.HttpHandler)
 	}
 
+	if s.httpOpt.EnableHealthcheckRoute {
+		router.Get("/healthcheck", healthcheck.HttpHandler())
+	}
+
 	go func() {
 		srv := http.Server{
 			Handler: router,

pkg/app/options/http.go

@@ -11,8 +11,9 @@ type (
 		LogResponse bool   `env:"HTTP_LOG_RESPONSE"`
 		Tracing     bool   `env:"HTTP_ERROR_TRACING"`
 
-		EnableVersionRoute bool `env:"HTTP_ENABLE_VERSION_ROUTE"`
-		EnableDebugRoute   bool `env:"HTTP_ENABLE_DEBUG_ROUTE"`
+		EnableHealthcheckRoute bool `env:"HTTP_ENABLE_HEALTHCHECK_ROUTE"`
+		EnableVersionRoute     bool `env:"HTTP_ENABLE_VERSION_ROUTE"`
+		EnableDebugRoute       bool `env:"HTTP_ENABLE_DEBUG_ROUTE"`
 
 		EnableMetrics       bool   `env:"HTTP_METRICS"`
 		MetricsServiceLabel string `env:"HTTP_METRICS_NAME"`
@@ -33,15 +34,16 @@ type (
 
 func HTTP(pfix string) (o *HTTPServerOpt) {
 	o = &HTTPServerOpt{
-		Addr:                ":80",
-		LogRequest:          false,
-		LogResponse:         false,
-		Tracing:             false,
-		EnableVersionRoute:  true,
-		EnableDebugRoute:    false,
-		EnableMetrics:       false,
-		MetricsServiceLabel: "corteza",
-		MetricsUsername:     "metrics",
+		Addr:                   ":80",
+		LogRequest:             false,
+		LogResponse:            false,
+		Tracing:                false,
+		EnableHealthcheckRoute: true,
+		EnableVersionRoute:     true,
+		EnableDebugRoute:       false,
+		EnableMetrics:          false,
+		MetricsServiceLabel:    "corteza",
+		MetricsUsername:        "metrics",
 
 		// Reports panics to Sentry through HTTP middleware
 		EnablePanicReporting: true,

pkg/corredor/healthcheck.go

@@ -0,0 +1,24 @@
+package corredor
+
+import (
+	"context"
+	"fmt"
+	"google.golang.org/grpc/connectivity"
+)
+
+// Healtcheck for global
+func Healthcheck(ctx context.Context) error {
+	if gCorredor == nil {
+		return fmt.Errorf("uninitialized")
+	}
+
+	if !gCorredor.opt.Enabled {
+		return nil
+	}
+
+	if state := gCorredor.conn.GetState(); state != connectivity.Ready {
+		return fmt.Errorf("connection is %s", state)
+	}
+
+	return nil
+}

pkg/db/healthcheck.go

@@ -0,0 +1,23 @@
+package db
+
+import (
+	"context"
+	"github.com/titpetric/factory"
+)
+
+func Healthcheck() func(ctx context.Context) error {
+	const name = "default"
+	return func(ctx context.Context) error {
+		db, err := factory.Database.Get(name)
+		if err != nil {
+			return err
+		}
+
+		err = db.PingContext(ctx)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}

pkg/healthcheck/check.go

@@ -0,0 +1,118 @@
+package healthcheck
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"strings"
+)
+
+type (
+	checkFn func(ctx context.Context) error
+
+	Meta struct {
+		Label       string
+		Description string
+	}
+
+	check struct {
+		fn checkFn
+		*Meta
+	}
+
+	result struct {
+		err error
+		*Meta
+	}
+
+	results []*result
+	checks  struct {
+		cc []*check
+	}
+)
+
+var (
+	defaults *checks
+)
+
+func init() {
+	defaults = New()
+}
+
+func Defaults() *checks {
+	return defaults
+}
+
+func New() *checks {
+	return &checks{cc: []*check{}}
+}
+
+// Add appends new check
+func (c *checks) Add(fn checkFn, label string, description ...string) {
+	c.cc = append(c.cc, &check{fn, &Meta{Label: label, Description: strings.Join(description, "")}})
+}
+
+func (c checks) Run(ctx context.Context) results {
+	var rr = make([]*result, len(c.cc))
+
+	for i, c := range c.cc {
+		rr[i] = &result{c.fn(ctx), c.Meta}
+	}
+
+	return rr
+}
+
+func (rr results) Healthy() bool {
+	for _, c := range rr {
+		if c.err != nil {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (rr results) String() string {
+	buf := &bytes.Buffer{}
+
+	rr.WriteTo(buf)
+
+	return buf.String()
+}
+
+func (rr results) WriteTo(w io.Writer) {
+	var (
+		p = func(f string, aa ...interface{}) {
+			_, _ = fmt.Fprintf(w, f, aa...)
+		}
+	)
+
+	for _, r := range rr {
+		if r.IsHealthy() {
+			p("PASS")
+		} else {
+			p("FAIL")
+		}
+
+		p(" %s", r.Label)
+
+		if !r.IsHealthy() {
+			p(": %v", r.Error())
+		}
+
+		p("\n")
+	}
+}
+
+func (r *result) IsHealthy() bool {
+	return r != nil && r.err == nil
+}
+
+func (r *result) Error() string {
+	if r == nil || r.err == nil {
+		return ""
+	}
+
+	return r.err.Error()
+}

pkg/healthcheck/check_test.go

@@ -0,0 +1,68 @@
+package healthcheck
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_Healthy(t *testing.T) {
+	tests := []struct {
+		name    string
+		checks  []*check
+		healthy bool
+		string  string
+	}{
+		{
+			"should be healthy with handle for stringer output",
+			[]*check{{func(ctx context.Context) error { return nil }, &Meta{Label: "check01"}}},
+			true,
+			"PASS check01\n",
+		},
+		{
+			"should handle multiple healthy checks",
+			[]*check{
+				{func(ctx context.Context) error { return nil }, &Meta{Label: "check01"}},
+				{func(ctx context.Context) error { return nil }, &Meta{Label: "check02"}},
+			},
+			true,
+			"PASS check01\nPASS check02\n",
+		},
+		{
+			"should handle healthy and unhealthy checks",
+			[]*check{
+				{func(ctx context.Context) error { return nil }, &Meta{Label: "check01"}},
+				{func(ctx context.Context) error { return fmt.Errorf("x") }, &Meta{Label: "check02"}},
+				{func(ctx context.Context) error { return nil }, &Meta{Label: "check03"}},
+			},
+			false,
+			"PASS check01\nFAIL check02: x\nPASS check03\n",
+		},
+		{
+			"should handle labels",
+			[]*check{
+				{func(ctx context.Context) error { return nil }, &Meta{Label: "check01"}},
+				{func(ctx context.Context) error { return nil }, &Meta{Label: "Pretty check"}},
+			},
+			true,
+			"PASS check01\nPASS Pretty check\n",
+		},
+		{
+			"should handle empty check list",
+			[]*check{},
+			true,
+			"",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			a := assert.New(t)
+			r := (&checks{cc: tt.checks}).Run(context.Background())
+			a.Equal(tt.healthy, r.Healthy(), "healthy result failed")
+			a.Equal(tt.string, r.String(), "stringer output match failed")
+		})
+	}
+}

pkg/healthcheck/http_handler.go

@@ -0,0 +1,16 @@
+package healthcheck
+
+import "net/http"
+
+func HttpHandler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		results := Defaults().Run(r.Context())
+		if results.Healthy() {
+			w.WriteHeader(http.StatusOK)
+		} else {
+			w.WriteHeader(http.StatusInternalServerError)
+		}
+
+		results.WriteTo(w)
+	}
+}

pkg/mail/healthcheck.go

@@ -0,0 +1,19 @@
+package mail
+
+import (
+	"context"
+	"fmt"
+)
+
+// Healtcheck for (global) scheduler
+func Healthcheck(ctx context.Context) error {
+	if defaultDialer == nil {
+		return fmt.Errorf("uninitialized")
+	}
+
+	if defaultDialerError != nil {
+		return defaultDialerError
+	}
+
+	return nil
+}

pkg/scheduler/healthcheck.go

@@ -0,0 +1,19 @@
+package scheduler
+
+import (
+	"context"
+	"fmt"
+)
+
+// Healtcheck for (global) scheduler
+func Healthcheck(ctx context.Context) error {
+	if gScheduler == nil {
+		return fmt.Errorf("uninitialized")
+	}
+
+	if gScheduler.ticker == nil {
+		return fmt.Errorf("stopped")
+	}
+
+	return nil
+}

pkg/store/healthcheck.go

@@ -0,0 +1,11 @@
+package store
+
+import (
+	"context"
+)
+
+func Healthcheck(s Store) func(ctx context.Context) error {
+	return func(ctx context.Context) error {
+		return s.Healthcheck(ctx)
+	}
+}

pkg/store/interfaces.go

@@ -1,6 +1,7 @@
 package store
 
 import (
+	"context"
 	"io"
 )
 
@@ -19,4 +20,7 @@ type Store interface {
 
 	// Open returns file handle
 	Open(filename string) (io.ReadSeeker, error)
+
+	// Healthcheck checks health status of the store
+	Healthcheck(ctx context.Context) error
 }

pkg/store/minio/store.go

@@ -1,6 +1,7 @@
 package minio
 
 import (
+	"context"
 	"fmt"
 	"io"
 
@@ -119,3 +120,7 @@ func (s store) Open(name string) (io.ReadSeeker, error) {
 		ServerSideEncryption: s.sse,
 	})
 }
+
+func (s *store) Healthcheck(ctx context.Context) error {
+	return nil
+}

pkg/store/plain/store.go

@@ -1,6 +1,8 @@
 package plain
 
 import (
+	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"path"
@@ -96,3 +98,30 @@ func (s *store) Open(filename string) (io.ReadSeeker, error) {
 
 	return s.fs.Open(filename)
 }
+
+func (s *store) Healthcheck(ctx context.Context) error {
+	var (
+		fname = s.namespace + "/.healthcheck"
+		buf   = &bytes.Buffer{}
+	)
+
+	if s == nil {
+		return fmt.Errorf("uninitialized")
+	}
+
+	buf.Write([]byte("healthcheck"))
+
+	if err := s.Save(fname, buf); err != nil {
+		return err
+	}
+
+	if _, err := s.Open(fname); err != nil {
+		return err
+	}
+
+	if err := s.Remove(fname); err != nil {
+		return err
+	}
+
+	return nil
+}

system/service/service.go

@@ -2,6 +2,7 @@ package service
 
 import (
 	"context"
+	"github.com/cortezaproject/corteza-server/pkg/healthcheck"
 	"go.uber.org/zap"
 
 	"github.com/cortezaproject/corteza-server/pkg/actionlog"
@@ -92,6 +93,7 @@ var (
 )
 
 func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) {
+	hcd := healthcheck.Defaults()
 	DefaultLogger = log.Named("service")
 
 	{
@@ -163,6 +165,8 @@ func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) {
 		}
 	}
 
+	hcd.Add(store.Healthcheck(DefaultStore), "Store/System")
+
 	DefaultAuthNotification = AuthNotification(ctx)
 	DefaultAuth = Auth(ctx)
 	DefaultUser = User(ctx)