Add db connection retrying on startup

crm/start.go

@@ -10,10 +10,10 @@ import (
 	"github.com/SentimensRG/ctx"
 	"github.com/SentimensRG/ctx/sigctx"
 	"github.com/pkg/errors"
-	"github.com/titpetric/factory"
 	"github.com/titpetric/factory/resputil"
 
 	migrate "github.com/crusttech/crust/crm/db"
+	"github.com/crusttech/crust/internal/db"
 	systemService "github.com/crusttech/crust/system/service"
 
 	"github.com/crusttech/crust/crm/service"
@@ -46,15 +46,9 @@ func Init() error {
 	mail.SetupDialer(flags.smtp)
 
 	// start/configure database connection
-	factory.Database.Add("default", flags.db.DSN)
-	db := factory.Database.MustGet()
-
-	// @todo: profiling as an external service?
-	switch flags.db.Profiler {
-	case "stdout":
-		db.Profiler = &factory.Database.ProfilerStdout
-	default:
-		log.Println("No database query profiler selected")
+	db, err := db.TryToConnect(flags.db.DSN, flags.db.Profiler)
+	if err != nil {
+		return errors.Wrap(err, "could not connect to database")
 	}
 
 	// migrate database schema

internal/db/connector.go

@@ -0,0 +1,34 @@
+package db
+
+import (
+	"log"
+	"time"
+
+	"github.com/titpetric/factory"
+)
+
+func TryToConnect(dsn, profiler string) (db *factory.DB, err error) {
+	factory.Database.Add("default", dsn)
+	var try = 0
+
+connect:
+	try++
+	db, err = factory.Database.Get()
+
+	if err != nil {
+		delay := time.Second * 5
+		log.Printf("Failed to connect, try %d, error: %v, retry in %.0fs", try, err, delay.Seconds())
+		time.Sleep(delay)
+		goto connect
+	}
+
+	// @todo: profiling as an external service?
+	switch profiler {
+	case "stdout":
+		db.Profiler = &factory.Database.ProfilerStdout
+	default:
+		log.Println("No database query profiler selected")
+	}
+
+	return
+}

sam/start.go

@@ -10,9 +10,9 @@ import (
 	"github.com/SentimensRG/ctx"
 	"github.com/SentimensRG/ctx/sigctx"
 	"github.com/pkg/errors"
-	"github.com/titpetric/factory"
 	"github.com/titpetric/factory/resputil"
 
+	"github.com/crusttech/crust/internal/db"
 	migrate "github.com/crusttech/crust/sam/db"
 	systemService "github.com/crusttech/crust/system/service"
 
@@ -46,15 +46,9 @@ func Init() error {
 	mail.SetupDialer(flags.smtp)
 
 	// start/configure database connection
-	factory.Database.Add("default", flags.db.DSN)
-	db := factory.Database.MustGet()
-
-	// @todo: profiling as an external service?
-	switch flags.db.Profiler {
-	case "stdout":
-		db.Profiler = &factory.Database.ProfilerStdout
-	default:
-		log.Println("No database query profiler selected")
+	db, err := db.TryToConnect(flags.db.DSN, flags.db.Profiler)
+	if err != nil {
+		return errors.Wrap(err, "could not connect to database")
 	}
 
 	// migrate database schema

system/start.go

@@ -10,9 +10,9 @@ import (
 	"github.com/SentimensRG/ctx"
 	"github.com/SentimensRG/ctx/sigctx"
 	"github.com/pkg/errors"
-	"github.com/titpetric/factory"
 	"github.com/titpetric/factory/resputil"
 
+	"github.com/crusttech/crust/internal/db"
 	migrate "github.com/crusttech/crust/system/db"
 	systemService "github.com/crusttech/crust/system/service"
 
@@ -45,15 +45,9 @@ func Init() error {
 	mail.SetupDialer(flags.smtp)
 
 	// start/configure database connection
-	factory.Database.Add("default", flags.db.DSN)
-	db := factory.Database.MustGet()
-
-	// @todo: profiling as an external service?
-	switch flags.db.Profiler {
-	case "stdout":
-		db.Profiler = &factory.Database.ProfilerStdout
-	default:
-		log.Println("No database query profiler selected")
+	db, err := db.TryToConnect(flags.db.DSN, flags.db.Profiler)
+	if err != nil {
+		return errors.Wrap(err, "could not connect to database")
 	}
 
 	// migrate database schema