Cleanup internal, vendors, cleanup cmd/*

Introduces /pkg for non-intenral packages

pkg/cli/context.go

@@ -0,0 +1,16 @@
+package cli
+
+import (
+	"context"
+
+	ctxwrap "github.com/SentimensRG/ctx"
+	"github.com/SentimensRG/ctx/sigctx"
+)
+
+// Context is  small wrapper that returns sig-term bound context
+//
+// This can be used as (proper) background context that properly terminates
+// all subroutines.
+func Context() context.Context {
+	return ctxwrap.AsContext(sigctx.New())
+}

pkg/cli/flags/db.go

@@ -0,0 +1,26 @@
+package flags
+
+import (
+	"github.com/spf13/cobra"
+)
+
+type (
+	DBOpt struct {
+		DSN      string
+		Profiler string
+	}
+)
+
+func DB(cmd *cobra.Command, pfix string) (o *DBOpt) {
+	o = &DBOpt{}
+
+	bindString(cmd, &o.DSN,
+		pFlag(pfix, "db-dsn"), "corteza:corteza@tcp(db:3306)/corteza?collation=utf8mb4_general_ci",
+		"DSN for database connection")
+
+	bindString(cmd, &o.Profiler,
+		pFlag(pfix, "db-profiler"), "none",
+		"Profiler for DB queries (none, stdout, logger)")
+
+	return
+}

pkg/cli/flags/flags.go

@@ -0,0 +1,56 @@
+package flags
+
+import (
+	"os"
+	"strings"
+	"time"
+
+	"github.com/spf13/cast"
+	"github.com/spf13/cobra"
+)
+
+// Prefixes flag
+func pFlag(pfix, name string) string {
+	if pfix != "" {
+		name = pfix + "-" + name
+	}
+
+	return name
+}
+
+// Converts input (flag-name) into ENVIRONMENTAL_VARIABLE_KEY
+func envKey(s string) string {
+	return strings.ToUpper(strings.ReplaceAll(s, "-", "_"))
+}
+
+func bindString(cmd *cobra.Command, v *string, flag, def string, desc string) {
+	if env, has := os.LookupEnv(envKey(flag)); has {
+		def = cast.ToString(env)
+	}
+
+	cmd.Flags().StringVar(v, flag, def, desc)
+}
+
+func bindBool(cmd *cobra.Command, v *bool, flag string, def bool, desc string) {
+	if env, has := os.LookupEnv(envKey(flag)); has {
+		def = cast.ToBool(env)
+	}
+
+	cmd.Flags().BoolVar(v, flag, def, desc)
+}
+
+func bindInt(cmd *cobra.Command, v *int, flag string, def int, desc string) {
+	if env, has := os.LookupEnv(envKey(flag)); has {
+		def = cast.ToInt(env)
+	}
+
+	cmd.Flags().IntVar(v, flag, def, desc)
+}
+
+func bindDuration(cmd *cobra.Command, v *time.Duration, flag string, def time.Duration, desc string) {
+	if env, has := os.LookupEnv(envKey(flag)); has {
+		def = cast.ToDuration(env)
+	}
+
+	cmd.Flags().DurationVar(v, flag, def, desc)
+}

pkg/cli/flags/http.go

@@ -0,0 +1,71 @@
+package flags
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/cortezaproject/corteza-server/internal/rand"
+)
+
+type (
+	HTTPOpt struct {
+		Addr    string
+		Logging bool
+		Pretty  bool
+		Tracing bool
+
+		EnableVersionRoute bool
+		EnableDebugRoute   bool
+
+		EnableMetrics       bool
+		MetricsServiceLabel string
+		MetricsUsername     string
+		MetricsPassword     string
+	}
+)
+
+func HTTP(cmd *cobra.Command, pfix string) (o *HTTPOpt) {
+	o = &HTTPOpt{}
+
+	bindString(cmd, &o.Addr,
+		pFlag(pfix, "http-addr"), ":80",
+		"Listen address for HTTP server")
+
+	bindBool(cmd, &o.Logging,
+		pFlag(pfix, "http-log"), true,
+		"Enable/disable HTTP request log")
+
+	bindBool(cmd, &o.Pretty,
+		pFlag(pfix, "http-pretty-json"), false,
+		"Prettify returned JSON output")
+
+	bindBool(cmd, &o.Tracing,
+		pFlag(pfix, "http-error-tracing"), false,
+		"Return error stack frame")
+
+	bindBool(cmd, &o.EnableVersionRoute,
+		pFlag(pfix, "http-enable-version-route"), true,
+		"Enable /version route")
+
+	bindBool(cmd, &o.EnableDebugRoute,
+		pFlag(pfix, "http-enable-debug-route"), false,
+		"Enable /debug route with pprof data")
+
+	bindBool(cmd, &o.EnableMetrics,
+		pFlag(pfix, "http-metrics"), false,
+		"Enable metrics")
+
+	bindString(cmd, &o.MetricsServiceLabel,
+		pFlag(pfix, "http-metrics-name"), "corteza",
+		"Provide metrics service label for Prometheus")
+
+	bindString(cmd, &o.MetricsUsername,
+		pFlag(pfix, "http-metrics-username"), "metrics",
+		"Provide metrics username for Prometheus")
+
+	// Setting metrics password to random string to prevent security accidents...
+	bindString(cmd, &o.MetricsPassword,
+		pFlag(pfix, "http-metrics-password"), string(rand.Bytes(5)),
+		"Provide metrics password for Prometheus")
+
+	return
+}

pkg/cli/flags/http_client.go

@@ -0,0 +1,28 @@
+package flags
+
+import (
+	"time"
+
+	"github.com/spf13/cobra"
+)
+
+type (
+	HttpClientOpt struct {
+		ClientTSLInsecure bool
+		HttpClientTimeout time.Duration
+	}
+)
+
+func HttpClient(cmd *cobra.Command) (o *HttpClientOpt) {
+	o = &HttpClientOpt{}
+
+	bindBool(cmd, &o.ClientTSLInsecure,
+		"http-client-tsl-insecure", false,
+		"Skip insecure TSL verification on outbound HTTP requests (allow invalid/self-signed certificates")
+
+	bindDuration(cmd, &o.HttpClientTimeout,
+		"http-client-timeout", 30*time.Second,
+		"Default HTTP client timeout")
+
+	return
+}

pkg/cli/flags/jwt.go

@@ -0,0 +1,29 @@
+package flags
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/cortezaproject/corteza-server/internal/rand"
+)
+
+type (
+	JWTOpt struct {
+		Secret string
+		Expiry int
+	}
+)
+
+func JWT(cmd *cobra.Command) (o *JWTOpt) {
+	o = &JWTOpt{}
+
+	// Setting JWT secret to random string to prevent security accidents...
+	bindString(cmd, &o.Secret,
+		"auth-jwt-secret", string(rand.Bytes(32)),
+		"JWT Secret")
+
+	bindInt(cmd, &o.Expiry,
+		"auth-jwt-expiry", 60*24*30,
+		"JWT Expiration in minutes")
+
+	return
+}

pkg/cli/flags/log.go

@@ -0,0 +1,26 @@
+package flags
+
+import (
+	"github.com/spf13/cobra"
+)
+
+type (
+	LogOpt struct {
+		Level string
+		JSON  bool
+	}
+)
+
+func Log(cmd *cobra.Command) (o *LogOpt) {
+	o = &LogOpt{}
+
+	bindString(cmd, &o.Level,
+		"log-level", "info",
+		"Log level (debug, info, warn, error, panic, fatal)")
+
+	bindBool(cmd, &o.JSON,
+		"log-json", true,
+		"Log in JSON format")
+
+	return
+}

pkg/cli/flags/monitor.go

@@ -0,0 +1,21 @@
+package flags
+
+import (
+	"github.com/spf13/cobra"
+)
+
+type (
+	MonitorOpt struct {
+		Interval int
+	}
+)
+
+func Monitor(cmd *cobra.Command, pfix string) (o *MonitorOpt) {
+	o = &MonitorOpt{}
+
+	bindInt(cmd, &o.Interval,
+		pFlag(pfix, "monitor-interval"), 300,
+		"Monitor interval (seconds, 0 = disable)")
+
+	return
+}

pkg/cli/flags/provision.go

@@ -0,0 +1,21 @@
+package flags
+
+import (
+	"github.com/spf13/cobra"
+)
+
+type (
+	ProvisionOpt struct {
+		Database bool
+	}
+)
+
+func Provision(cmd *cobra.Command, pfix string) (o *ProvisionOpt) {
+	o = &ProvisionOpt{}
+
+	bindBool(cmd, &o.Database,
+		pFlag(pfix, "provision-database"), true,
+		"Run database migration scripts")
+
+	return
+}

pkg/cli/flags/pubsub.go

@@ -0,0 +1,58 @@
+package flags
+
+import (
+	"time"
+
+	"github.com/spf13/cobra"
+)
+
+type (
+	PubSubOpt struct {
+		Mode string
+
+		// Mode
+		PollingInterval time.Duration
+
+		// Redis
+		RedisAddr        string
+		RedisTimeout     time.Duration
+		RedisPingTimeout time.Duration
+		RedisPingPeriod  time.Duration
+	}
+)
+
+func PubSub(cmd *cobra.Command, pfix string) (o *PubSubOpt) {
+	o = &PubSubOpt{}
+
+	const (
+		timeout     = 15 * time.Second
+		pingTimeout = 120 * time.Second
+		pingPeriod  = (pingTimeout * 9) / 10
+	)
+
+	bindString(cmd, &o.Mode,
+		pFlag(pfix, "pubsub-mode"), "poll",
+		"Pub/Sub mode (poll, redis")
+
+	bindDuration(cmd, &o.RedisPingTimeout,
+		pFlag(pfix, "pubsub-polling-interval"), timeout,
+		"Sub/Sub polling interval")
+
+	bindString(cmd, &o.RedisAddr,
+		pFlag(pfix, "pubsub-redis-addr"), "redis:6379",
+		"Pub/Sub mode (poll, redis")
+
+	bindDuration(cmd, &o.RedisTimeout,
+		pFlag(pfix, "pubsub-redis-timeout"), timeout,
+		"Websocket connection timeout")
+
+	bindDuration(cmd, &o.RedisPingTimeout,
+		pFlag(pfix, "pubsub-redis-ping-timeout"), pingTimeout,
+		"Pub/Sub connection ping timeout")
+
+	bindDuration(cmd, &o.RedisPingPeriod,
+		pFlag(pfix, "pubsub-redis-ping-period"), pingPeriod,
+		"Pub/Sub connection ping period (should be lower than timeout)")
+
+	return
+}

pkg/cli/flags/smtp.go

@@ -0,0 +1,41 @@
+package flags
+
+import (
+	"github.com/spf13/cobra"
+)
+
+type (
+	SMTPOpt struct {
+		Host string
+		Port int
+		User string
+		Pass string
+		From string
+	}
+)
+
+func SMTP(cmd *cobra.Command) (o *SMTPOpt) {
+	o = &SMTPOpt{}
+
+	bindString(cmd, &o.Host,
+		"smtp-host", "localhost:25",
+		"SMTP hostname")
+
+	bindString(cmd, &o.User,
+		"smtp-username", "",
+		"SMTP server username")
+
+	bindString(cmd, &o.Pass,
+		"smtp-pass", "",
+		"SMTP server password")
+
+	bindString(cmd, &o.From,
+		"smtp-from", "",
+		"Sender's email address")
+
+	bindInt(cmd, &o.Port,
+		"smtp-port", 25,
+		"SMTP port number")
+
+	return
+}

pkg/cli/flags/websocket.go

@@ -0,0 +1,39 @@
+package flags
+
+import (
+	"time"
+
+	"github.com/spf13/cobra"
+)
+
+type (
+	WebsocketOpt struct {
+		Timeout     time.Duration
+		PingTimeout time.Duration
+		PingPeriod  time.Duration
+	}
+)
+
+func Websocket(cmd *cobra.Command, pfix string) (o *WebsocketOpt) {
+	o = &WebsocketOpt{}
+
+	const (
+		timeout     = 15 * time.Second
+		pingTimeout = 120 * time.Second
+		pingPeriod  = (pingTimeout * 9) / 10
+	)
+
+	bindDuration(cmd, &o.Timeout,
+		pFlag(pfix, "websocket-timeout"), timeout,
+		"Websocket connection timeout")
+
+	bindDuration(cmd, &o.PingTimeout,
+		pFlag(pfix, "websocket-ping-timeout"), pingTimeout,
+		"Websocket connection ping timeout")
+
+	bindDuration(cmd, &o.PingPeriod,
+		pFlag(pfix, "websocket-ping-period"), pingPeriod,
+		"Websocket connection ping period (should be lower than timeout)")
+
+	return
+}

pkg/cli/helpers.go

@@ -0,0 +1,107 @@
+package cli
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+
+	"github.com/cortezaproject/corteza-server/internal/auth"
+	"github.com/cortezaproject/corteza-server/internal/http"
+	"github.com/cortezaproject/corteza-server/internal/logger"
+	"github.com/cortezaproject/corteza-server/internal/mail"
+	"github.com/cortezaproject/corteza-server/pkg/cli/flags"
+)
+
+// SetupProvisionCommands sets-up standard provision commands
+// Deprecated: use SetupProvisionSubCommands
+func SetupProvisionCommands(ac func() error, md func() error) *cobra.Command {
+	var (
+		cmd = &cobra.Command{
+			Use:   "provision",
+			Short: "Provision tasks",
+		}
+	)
+
+	// Add only commands with defined callbacks
+	if ac != nil {
+		cmd.AddCommand(&cobra.Command{
+			Use:   "access-control-rules",
+			Short: "Reset access control rules & roles",
+
+			RunE: func(cmd *cobra.Command, args []string) error {
+				return ac()
+			},
+		})
+	}
+
+	// Add only commands with defined callbacks
+	if md != nil {
+		cmd.AddCommand(&cobra.Command{
+			Use:   "migrate-database",
+			Short: "Run database migration scripts",
+
+			RunE: func(cmd *cobra.Command, args []string) error {
+				return md()
+			},
+		})
+	}
+
+	return cmd
+}
+
+type (
+	provisioner interface {
+		ProvisionMigrateDatabase(ctx context.Context) error
+		ProvisionAccessControl(ctx context.Context) error
+	}
+)
+
+func SetupProvisionSubcommands(ctx context.Context, p provisioner) *cobra.Command {
+	var (
+		cmd = &cobra.Command{
+			Use:   "provision",
+			Short: "Provision tasks",
+		}
+	)
+
+	// Add only commands with defined callbacks
+	cmd.AddCommand(&cobra.Command{
+		Use:   "access-control-rules",
+		Short: "Reset access control rules & roles",
+
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return p.ProvisionAccessControl(ctx)
+		},
+	})
+
+	// Add only commands with defined callbacks
+	cmd.AddCommand(&cobra.Command{
+		Use:   "migrate-database",
+		Short: "Run database migration scripts",
+
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return p.ProvisionMigrateDatabase(ctx)
+		},
+	})
+
+	return cmd
+}
+
+func InitGeneralServices(logOpt *flags.LogOpt, smtpOpt *flags.SMTPOpt, jwtOpt *flags.JWTOpt, httpClientOpt *flags.HttpClientOpt) {
+	var logLevel = zap.InfoLevel
+	_ = logLevel.Set(logOpt.Level)
+
+	if logger.Default() == nil {
+		logger.Init(logLevel)
+	} else {
+		logger.DefaultLevel.SetLevel(logLevel)
+	}
+
+	auth.SetupDefault(jwtOpt.Secret, jwtOpt.Expiry)
+	mail.SetupDialer(smtpOpt.Host, smtpOpt.Port, smtpOpt.User, smtpOpt.Pass, smtpOpt.From)
+	http.SetupDefaults(
+		httpClientOpt.HttpClientTimeout,
+		httpClientOpt.ClientTSLInsecure,
+	)
+}