Refactor bootstraping procedure

pkg/options/actionlog.go

@@ -0,0 +1,19 @@
+package options
+
+type (
+	ActionLogOpt struct {
+		Enabled bool `env:"ACTIONLOG_ENABLED"`
+		Debug   bool `env:"ACTIONLOG_DEBUG"`
+	}
+)
+
+func ActionLog() (o *ActionLogOpt) {
+	o = &ActionLogOpt{
+		Enabled: true,
+		Debug:   false,
+	}
+
+	fill(o)
+
+	return
+}

pkg/options/corredor.go

@@ -0,0 +1,61 @@
+package options
+
+import (
+	"path"
+	"time"
+)
+
+type (
+	CorredorOpt struct {
+		Enabled bool `env:"CORREDOR_ENABLED"`
+
+		// Also used by corredor service to configure gRPC server
+		Addr string `env:"CORREDOR_ADDR"`
+
+		MaxBackoffDelay time.Duration `env:"CORREDOR_MAX_BACKOFF_DELAY"`
+
+		MaxReceiveMessageSize int `env:"CORREDOR_MAX_RECEIVE_MESSAGE_SIZE"`
+
+		DefaultExecTimeout time.Duration `env:"CORREDOR_DEFAULT_EXEC_TIMEOUT"`
+
+		ListTimeout time.Duration `env:"CORREDOR_LIST_TIMEOUT"`
+		ListRefresh time.Duration `env:"CORREDOR_LIST_REFRESH"`
+
+		// Allow scripts to have runner explicitly defined
+		RunAsEnabled bool `env:"CORREDOR_RUN_AS_ENABLED"`
+
+		TlsCertEnabled bool   `env:"CORREDOR_CLIENT_CERTIFICATES_ENABLED"`
+		TlsCertPath    string `env:"CORREDOR_CLIENT_CERTIFICATES_PATH"`
+		TlsCertCA      string `env:"CORREDOR_CLIENT_CERTIFICATES_CA"`
+		TlsCertPrivate string `env:"CORREDOR_CLIENT_CERTIFICATES_PUBLIC"`
+		TlsCertPublic  string `env:"CORREDOR_CLIENT_CERTIFICATES_PRIVATE"`
+		TlsServerName  string `env:"CORREDOR_CLIENT_CERTIFICATES_SERVER_NAME"`
+	}
+)
+
+func Corredor() (o *CorredorOpt) {
+	o = &CorredorOpt{
+		Enabled:               true,
+		RunAsEnabled:          true,
+		Addr:                  "localhost:50051",
+		MaxBackoffDelay:       time.Minute,
+		MaxReceiveMessageSize: 2 << 23, // 16MB
+		DefaultExecTimeout:    time.Minute,
+		ListTimeout:           time.Second * 2,
+		ListRefresh:           time.Second * 5,
+
+		TlsCertEnabled: false,
+		TlsCertPath:    "/certs/corredor/client",
+		TlsCertCA:      "ca.crt",
+		TlsCertPublic:  "public.crt",
+		TlsCertPrivate: "private.key",
+	}
+
+	fill(o)
+
+	o.TlsCertCA = path.Join(o.TlsCertPath, o.TlsCertCA)
+	o.TlsCertPrivate = path.Join(o.TlsCertPath, o.TlsCertPrivate)
+	o.TlsCertPublic = path.Join(o.TlsCertPath, o.TlsCertPublic)
+
+	return
+}

pkg/options/db.go

@@ -0,0 +1,38 @@
+package options
+
+import (
+	"strings"
+	"time"
+)
+
+type (
+	DBOpt struct {
+		DSN      string        `env:"DB_DSN"`
+		Logger   bool          `env:"DB_LOGGER"`
+		MaxTries int           `env:"DB_MAX_TRIES"`
+		Delay    time.Duration `env:"DB_CONN_ERR_DELAY"`
+		Timeout  time.Duration `env:"DB_CONN_TIMEOUT"`
+	}
+)
+
+func DB(pfix string) (o *DBOpt) {
+	const delay = 15 * time.Second
+	const maxTries = 100
+
+	o = &DBOpt{
+		DSN:      "mysql://corteza:corteza@tcp(db:3306)/corteza?collation=utf8mb4_general_ci",
+		Logger:   false,
+		MaxTries: maxTries,
+		Delay:    delay,
+		Timeout:  maxTries * delay,
+	}
+
+	fill(o)
+
+	if !strings.Contains(o.DSN, "://") {
+		// Make sure DSN is compatible with new requirements
+		o.DSN = "mysql://" + o.DSN
+	}
+
+	return
+}

pkg/options/helpers.go

@@ -0,0 +1,104 @@
+package options
+
+import (
+	"os"
+	"reflect"
+	"time"
+
+	"github.com/spf13/cast"
+)
+
+func fill(opt interface{}) {
+	v := reflect.ValueOf(opt)
+	if v.Kind() != reflect.Ptr {
+		panic("expecting a pointer, not a value")
+	}
+
+	if v.IsNil() {
+		panic("nil pointer passed")
+	}
+
+	v = v.Elem()
+
+	length := v.NumField()
+	for i := 0; i < length; i++ {
+		f := v.Field(i)
+		t := v.Type().Field(i)
+		if tag := t.Tag.Get("env"); tag != "" {
+			if !f.CanSet() {
+				panic("unexpected pointer for field " + t.Name)
+			}
+
+			if f.Type() == reflect.TypeOf(time.Duration(1)) {
+				v.FieldByName(t.Name).SetInt(int64(EnvDuration(tag, time.Duration(f.Int()))))
+				continue
+			}
+
+			if f.Kind() == reflect.String {
+				v.FieldByName(t.Name).SetString(EnvString(tag, f.String()))
+				continue
+			}
+
+			if f.Kind() == reflect.Bool {
+				v.FieldByName(t.Name).SetBool(EnvBool(tag, f.Bool()))
+				continue
+			}
+
+			if f.Kind() == reflect.Int {
+				v.FieldByName(t.Name).SetInt(int64(EnvInt(tag, int(f.Int()))))
+				continue
+			}
+
+			if f.Kind() == reflect.Float32 {
+				v.FieldByName(t.Name).SetFloat(float64(EnvFloat32(tag, float32(f.Float()))))
+				continue
+			}
+
+			panic("unsupported type/kind for field " + t.Name)
+
+		}
+	}
+}
+
+func EnvString(key string, def string) string {
+	if val, has := os.LookupEnv(key); has {
+		return val
+	}
+	return def
+}
+
+func EnvBool(key string, def bool) bool {
+	if val, has := os.LookupEnv(key); has {
+		if b, err := cast.ToBoolE(val); err == nil {
+			return b
+		}
+	}
+	return def
+}
+
+func EnvInt(key string, def int) int {
+	if val, has := os.LookupEnv(key); has {
+		if i, err := cast.ToIntE(val); err == nil {
+			return i
+		}
+	}
+	return def
+}
+
+func EnvFloat32(key string, def float32) float32 {
+	if val, has := os.LookupEnv(key); has {
+		if i, err := cast.ToFloat32E(val); err == nil {
+			return i
+		}
+	}
+	return def
+}
+
+func EnvDuration(key string, def time.Duration) time.Duration {
+	if val, has := os.LookupEnv(key); has {
+		if d, err := cast.ToDurationE(val); err == nil {
+			return d
+		}
+	}
+	return def
+}

pkg/options/http.go

@@ -0,0 +1,66 @@
+package options
+
+import (
+	"github.com/cortezaproject/corteza-server/pkg/rand"
+)
+
+type (
+	HTTPServerOpt struct {
+		Addr        string `env:"HTTP_ADDR"`
+		LogRequest  bool   `env:"HTTP_LOG_REQUEST"`
+		LogResponse bool   `env:"HTTP_LOG_RESPONSE"`
+		Tracing     bool   `env:"HTTP_ERROR_TRACING"`
+
+		EnableHealthcheckRoute bool `env:"HTTP_ENABLE_HEALTHCHECK_ROUTE"`
+		EnableVersionRoute     bool `env:"HTTP_ENABLE_VERSION_ROUTE"`
+		EnableDebugRoute       bool `env:"HTTP_ENABLE_DEBUG_ROUTE"`
+
+		EnableMetrics       bool   `env:"HTTP_METRICS"`
+		MetricsServiceLabel string `env:"HTTP_METRICS_NAME"`
+		MetricsUsername     string `env:"HTTP_METRICS_USERNAME"`
+		MetricsPassword     string `env:"HTTP_METRICS_PASSWORD"`
+
+		EnablePanicReporting bool `env:"HTTP_REPORT_PANIC"`
+
+		ApiEnabled bool   `env:"HTTP_API_ENABLED"`
+		ApiBaseUrl string `env:"HTTP_API_BASE_URL"`
+
+		WebappEnabled bool   `env:"HTTP_WEBAPP_ENABLED"`
+		WebappBaseUrl string `env:"HTTP_WEBAPP_BASE_URL"`
+		WebappBaseDir string `env:"HTTP_WEBAPP_BASE_DIR"`
+		WebappList    string `env:"HTTP_WEBAPP_LIST"`
+	}
+)
+
+func HTTP(pfix string) (o *HTTPServerOpt) {
+	o = &HTTPServerOpt{
+		Addr:                   ":80",
+		LogRequest:             false,
+		LogResponse:            false,
+		Tracing:                false,
+		EnableHealthcheckRoute: true,
+		EnableVersionRoute:     true,
+		EnableDebugRoute:       false,
+		EnableMetrics:          false,
+		MetricsServiceLabel:    "corteza",
+		MetricsUsername:        "metrics",
+
+		// Reports panics to Sentry through HTTP middleware
+		EnablePanicReporting: true,
+
+		// Setting metrics password to random string to prevent security accidents...
+		MetricsPassword: string(rand.Bytes(5)),
+
+		ApiEnabled: true,
+		ApiBaseUrl: "",
+
+		WebappEnabled: false,
+		WebappBaseUrl: "/",
+		WebappBaseDir: "/webapp",
+		WebappList:    "admin,auth,messaging,compose",
+	}
+
+	fill(o)
+
+	return
+}

pkg/options/http_client.go

@@ -0,0 +1,23 @@
+package options
+
+import (
+	"time"
+)
+
+type (
+	HTTPClientOpt struct {
+		ClientTSLInsecure bool          `env:"HTTP_CLIENT_TSL_INSECURE"`
+		HttpClientTimeout time.Duration `env:"HTTP_CLIENT_TIMEOUT"`
+	}
+)
+
+func HttpClient(pfix string) (o *HTTPClientOpt) {
+	o = &HTTPClientOpt{
+		ClientTSLInsecure: false,
+		HttpClientTimeout: 30 * time.Second,
+	}
+
+	fill(o)
+
+	return
+}

pkg/options/jwt.go

@@ -0,0 +1,33 @@
+package options
+
+import (
+	"time"
+
+	"github.com/cortezaproject/corteza-server/pkg/rand"
+)
+
+type (
+	AuthOpt struct {
+		Secret string        `env:"AUTH_JWT_SECRET"`
+		Expiry time.Duration `env:"AUTH_JWT_EXPIRY"`
+	}
+)
+
+func Auth() (o *AuthOpt) {
+	o = &AuthOpt{
+		Expiry: time.Hour * 24 * 30,
+	}
+
+	fill(o)
+
+	// Setting JWT secret to random string to prevent security accidents...
+	//
+	// @todo check if this is a monolith system
+	//       on microservice setup we can not afford to autogenerate secret:
+	//       each subsystem will get it's own
+	if o.Secret == "" {
+		o.Secret = string(rand.Bytes(32))
+	}
+
+	return
+}

pkg/options/monitor.go

@@ -0,0 +1,20 @@
+package options
+
+import (
+	"time"
+)
+
+type (
+	MonitorOpt struct {
+		Interval time.Duration `env:"MONITOR_INTERVAL"`
+	}
+)
+
+func Monitor(pfix string) (o *MonitorOpt) {
+	o = &MonitorOpt{
+		Interval: 300 * time.Second,
+	}
+	fill(o)
+
+	return
+}

pkg/options/provision.go

@@ -0,0 +1,17 @@
+package options
+
+type (
+	ProvisionOpt struct {
+		Always bool `env:"PROVISION_ALWAYS"`
+	}
+)
+
+func Provision(pfix string) (o *ProvisionOpt) {
+	o = &ProvisionOpt{
+		Always: true,
+	}
+
+	fill(o)
+
+	return
+}

pkg/options/pubsub.go

@@ -0,0 +1,41 @@
+package options
+
+import (
+	"time"
+)
+
+type (
+	PubSubOpt struct {
+		Mode string `env:"PUBSUB_MODE"`
+
+		// Mode
+		PollingInterval time.Duration `env:"PUBSUB_POLLING_INTERVAL"`
+
+		// Redis
+		RedisAddr        string        `env:"PUBSUB_REDIS_ADDR"`
+		RedisTimeout     time.Duration `env:"PUBSUB_REDIS_TIMEOUT"`
+		RedisPingTimeout time.Duration `env:"PUBSUB_REDIS_PING_TIMEOUT"`
+		RedisPingPeriod  time.Duration `env:"PUBSUB_REDIS_PING_PERIOD"`
+	}
+)
+
+func PubSub(pfix string) (o *PubSubOpt) {
+	const (
+		timeout     = 15 * time.Second
+		pingTimeout = 120 * time.Second
+		pingPeriod  = (pingTimeout * 9) / 10
+	)
+
+	o = &PubSubOpt{
+		Mode:             "poll",
+		PollingInterval:  timeout,
+		RedisAddr:        "redis:6379",
+		RedisTimeout:     timeout,
+		RedisPingTimeout: pingTimeout,
+		RedisPingPeriod:  pingPeriod,
+	}
+
+	fill(o)
+
+	return
+}

pkg/options/sentry.go

@@ -0,0 +1,34 @@
+package options
+
+import (
+	"github.com/cortezaproject/corteza-server/pkg/version"
+)
+
+type (
+	SentryOpt struct {
+		DSN string `env:"SENTRY_DSN"`
+
+		Debug            bool    `env:"SENTRY_DEBUG"`
+		AttachStacktrace bool    `env:"SENTRY_ATTACH_STACKTRACE"`
+		SampleRate       float32 `env:"SENTRY_SAMPLE_RATE"`
+		MaxBreadcrumbs   int     `env:"SENTRY_MAX_BREADCRUMBS"`
+
+		ServerName  string `env:"SENTRY_SERVERNAME"`
+		Release     string `env:"SENTRY_RELEASE"`
+		Dist        string `env:"SENTRY_DIST"`
+		Environment string `env:"SENTRY_ENVIRONMENT"`
+	}
+)
+
+func Sentry(pfix string) (o *SentryOpt) {
+	o = &SentryOpt{
+		AttachStacktrace: true,
+		MaxBreadcrumbs:   0,
+
+		Release: version.Version,
+	}
+
+	fill(o)
+
+	return
+}

pkg/options/smtp.go

@@ -0,0 +1,25 @@
+package options
+
+type (
+	SMTPOpt struct {
+		Host string `env:"SMTP_HOST"`
+		Port int    `env:"SMTP_PORT"`
+		User string `env:"SMTP_USER"`
+		Pass string `env:"SMTP_PASS"`
+		From string `env:"SMTP_FROM"`
+	}
+)
+
+func SMTP(pfix string) (o *SMTPOpt) {
+	o = &SMTPOpt{
+		Host: "localhost:25",
+		Port: 25,
+		User: "",
+		Pass: "",
+		From: "",
+	}
+
+	fill(o)
+
+	return
+}

pkg/options/storage.go

@@ -0,0 +1,32 @@
+package options
+
+type (
+	StorageOpt struct {
+		Path string `env:"STORAGE_PATH"`
+
+		MinioEndpoint  string `env:"MINIO_ENDPOINT"`
+		MinioSecure    bool   `env:"MINIO_SECURE"`
+		MinioAccessKey string `env:"MINIO_ACCESS_KEY"`
+		MinioSecretKey string `env:"MINIO_SECRET_KEY"`
+		MinioSSECKey   string `env:"MINIO_SSEC_KEY"`
+		MinioBucket    string `env:"MINIO_BUCKET"`
+		MinioStrict    bool   `env:"MINIO_STRICT"`
+	}
+)
+
+func Storage(pfix string) (o *StorageOpt) {
+	o = &StorageOpt{
+		Path: "var/store",
+
+		// Make minio secure by default
+		MinioSecure: true,
+
+		// Run in struct mode:
+		//  - do not create un-existing buckets
+		MinioStrict: false,
+	}
+
+	fill(o)
+
+	return
+}

pkg/options/upgrade.go

@@ -0,0 +1,19 @@
+package options
+
+type (
+	UpgradeOpt struct {
+		Debug  bool `env:"UPGRADE_DEBUG"`
+		Always bool `env:"UPGRADE_ALWAYS"`
+	}
+)
+
+func Upgrade(pfix string) (o *UpgradeOpt) {
+	o = &UpgradeOpt{
+		Debug:  false,
+		Always: true,
+	}
+
+	fill(o)
+
+	return
+}

pkg/options/wait_for.go

@@ -0,0 +1,41 @@
+package options
+
+import (
+	"strings"
+	"time"
+)
+
+type (
+	WaitForOpt struct {
+		Delay                 time.Duration `env:"WAIT_FOR"`
+		StatusPage            bool          `env:"WAIT_FOR_STATUS_PAGE"`
+		Services              string        `env:"WAIT_FOR_SERVICES"`
+		ServicesTimeout       time.Duration `env:"WAIT_FOR_SERVICES_TIMEOUT"`
+		ServicesProbeTimeout  time.Duration `env:"WAIT_FOR_SERVICES_PROBE_TIMEOUT"`
+		ServicesProbeInterval time.Duration `env:"WAIT_FOR_SERVICES_PROBE_INTERVAL"`
+	}
+)
+
+func WaitFor(pfix string) (o *WaitForOpt) {
+	o = &WaitForOpt{
+		Delay:                 0,
+		StatusPage:            true,
+		Services:              "",
+		ServicesTimeout:       time.Minute,
+		ServicesProbeTimeout:  time.Second * 30,
+		ServicesProbeInterval: time.Second * 5,
+	}
+
+	fill(o)
+
+	return
+}
+
+// Parses hosts and return slice of strings, one per host
+func (o WaitForOpt) GetServices() []string {
+	if len(o.Services) == 0 {
+		return []string{}
+	}
+
+	return strings.Split(o.Services, " ")
+}

pkg/options/websocket.go

@@ -0,0 +1,31 @@
+package options
+
+import (
+	"time"
+)
+
+type (
+	WebsocketOpt struct {
+		Timeout     time.Duration `env:"WEBSOCKET_TIMEOUT"`
+		PingTimeout time.Duration `env:"WEBSOCKET_PING_TIMEOUT"`
+		PingPeriod  time.Duration `env:"WEBSOCKET_PING_PERIOD"`
+	}
+)
+
+func Websocket(pfix string) (o *WebsocketOpt) {
+	const (
+		timeout     = 15 * time.Second
+		pingTimeout = 120 * time.Second
+		pingPeriod  = (pingTimeout * 9) / 10
+	)
+
+	o = &WebsocketOpt{
+		Timeout:     timeout,
+		PingTimeout: pingTimeout,
+		PingPeriod:  pingPeriod,
+	}
+
+	fill(o)
+
+	return
+}