Add healthchecks for db, mail, scheduler, corredor, store

compose/service/service.go

@@ -3,6 +3,7 @@ package service
 import (
 	"context"
 	"errors"
+	"github.com/cortezaproject/corteza-server/pkg/healthcheck"
 	"go.uber.org/zap"
 	"time"
 
@@ -78,6 +79,8 @@ var (
 
 // Initializes compose-only services
 func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) {
+	hcd := healthcheck.Defaults()
+
 	var db = repository.DB(ctx)
 
 	DefaultLogger = log.Named("service")
@@ -145,8 +148,11 @@ func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) {
 			log.Info("initializing store",
 				zap.String("path", path),
 				zap.Error(err))
+
 		}
 
+		hcd.Add(store.Healthcheck(DefaultStore), "Store/Compose")
+
 		if err != nil {
 			return err
 		}

corteza/corteza.go

@@ -2,6 +2,7 @@ package corteza
 
 import (
 	"context"
+	"github.com/cortezaproject/corteza-server/pkg/healthcheck"
 	"time"
 
 	"github.com/pkg/errors"
@@ -30,6 +31,12 @@ type (
 var _ app.Runnable = &App{}
 
 func (app *App) Setup(log *zap.Logger, opts *app.Options) (err error) {
+	hcd := healthcheck.Defaults()
+	hcd.Add(scheduler.Healthcheck, "Scheduler")
+	hcd.Add(mail.Healthcheck, "Mail")
+	hcd.Add(corredor.Healthcheck, "Corredor")
+	hcd.Add(db.Healthcheck(), "Database")
+
 	app.log = log
 	app.opt = opts
 

messaging/service/service.go

@@ -2,6 +2,7 @@ package service
 
 import (
 	"context"
+	"github.com/cortezaproject/corteza-server/pkg/healthcheck"
 	"time"
 
 	"go.uber.org/zap"
@@ -57,6 +58,7 @@ var (
 )
 
 func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) {
+	hcd := healthcheck.Defaults()
 	DefaultLogger = log.Named("service")
 
 	{
@@ -130,6 +132,8 @@ func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) {
 		}
 	}
 
+	hcd.Add(store.Healthcheck(DefaultStore), "Store/Messaging")
+
 	DefaultEvent = Event(ctx)
 	DefaultChannel = Channel(ctx)
 	DefaultAttachment = Attachment(ctx, DefaultStore)

pkg/corredor/healthcheck.go

@@ -0,0 +1,24 @@
+package corredor
+
+import (
+	"context"
+	"fmt"
+	"google.golang.org/grpc/connectivity"
+)
+
+// Healtcheck for global
+func Healthcheck(ctx context.Context) error {
+	if gCorredor == nil {
+		return fmt.Errorf("uninitialized")
+	}
+
+	if !gCorredor.opt.Enabled {
+		return nil
+	}
+
+	if state := gCorredor.conn.GetState(); state != connectivity.Ready {
+		return fmt.Errorf("connection is %s", state)
+	}
+
+	return nil
+}

pkg/db/healthcheck.go

@@ -0,0 +1,23 @@
+package db
+
+import (
+	"context"
+	"github.com/titpetric/factory"
+)
+
+func Healthcheck() func(ctx context.Context) error {
+	const name = "default"
+	return func(ctx context.Context) error {
+		db, err := factory.Database.Get(name)
+		if err != nil {
+			return err
+		}
+
+		err = db.PingContext(ctx)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}

pkg/mail/healthcheck.go

@@ -0,0 +1,19 @@
+package mail
+
+import (
+	"context"
+	"fmt"
+)
+
+// Healtcheck for (global) scheduler
+func Healthcheck(ctx context.Context) error {
+	if defaultDialer == nil {
+		return fmt.Errorf("uninitialized")
+	}
+
+	if defaultDialerError != nil {
+		return defaultDialerError
+	}
+
+	return nil
+}

pkg/scheduler/healthcheck.go

@@ -0,0 +1,19 @@
+package scheduler
+
+import (
+	"context"
+	"fmt"
+)
+
+// Healtcheck for (global) scheduler
+func Healthcheck(ctx context.Context) error {
+	if gScheduler == nil {
+		return fmt.Errorf("uninitialized")
+	}
+
+	if gScheduler.ticker == nil {
+		return fmt.Errorf("stopped")
+	}
+
+	return nil
+}

pkg/store/healthcheck.go

@@ -0,0 +1,11 @@
+package store
+
+import (
+	"context"
+)
+
+func Healthcheck(s Store) func(ctx context.Context) error {
+	return func(ctx context.Context) error {
+		return s.Healthcheck(ctx)
+	}
+}

pkg/store/interfaces.go

@@ -1,6 +1,7 @@
 package store
 
 import (
+	"context"
 	"io"
 )
 
@@ -19,4 +20,7 @@ type Store interface {
 
 	// Open returns file handle
 	Open(filename string) (io.ReadSeeker, error)
+
+	// Healthcheck checks health status of the store
+	Healthcheck(ctx context.Context) error
 }

pkg/store/minio/store.go

@@ -1,6 +1,7 @@
 package minio
 
 import (
+	"context"
 	"fmt"
 	"io"
 
@@ -119,3 +120,7 @@ func (s store) Open(name string) (io.ReadSeeker, error) {
 		ServerSideEncryption: s.sse,
 	})
 }
+
+func (s *store) Healthcheck(ctx context.Context) error {
+	return nil
+}

pkg/store/plain/store.go

@@ -1,6 +1,8 @@
 package plain
 
 import (
+	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"path"
@@ -96,3 +98,30 @@ func (s *store) Open(filename string) (io.ReadSeeker, error) {
 
 	return s.fs.Open(filename)
 }
+
+func (s *store) Healthcheck(ctx context.Context) error {
+	var (
+		fname = s.namespace + "/.healthcheck"
+		buf   = &bytes.Buffer{}
+	)
+
+	if s == nil {
+		return fmt.Errorf("uninitialized")
+	}
+
+	buf.Write([]byte("healthcheck"))
+
+	if err := s.Save(fname, buf); err != nil {
+		return err
+	}
+
+	if _, err := s.Open(fname); err != nil {
+		return err
+	}
+
+	if err := s.Remove(fname); err != nil {
+		return err
+	}
+
+	return nil
+}

system/service/service.go

@@ -2,6 +2,7 @@ package service
 
 import (
 	"context"
+	"github.com/cortezaproject/corteza-server/pkg/healthcheck"
 	"go.uber.org/zap"
 
 	"github.com/cortezaproject/corteza-server/pkg/actionlog"
@@ -92,6 +93,7 @@ var (
 )
 
 func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) {
+	hcd := healthcheck.Defaults()
 	DefaultLogger = log.Named("service")
 
 	{
@@ -163,6 +165,8 @@ func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) {
 		}
 	}
 
+	hcd.Add(store.Healthcheck(DefaultStore), "Store/System")
+
 	DefaultAuthNotification = AuthNotification(ctx)
 	DefaultAuth = Auth(ctx)
 	DefaultUser = User(ctx)