DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Improve cli env setup & running

Browse Source
This commit is contained in:
Denis Arh
2019-05-26 08:41:20 +02:00
parent 36b32fa2fe
commit 22476933f4
89 changed files with 945 additions and 907 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cli/flags/db.go
+2 -2
View File
@@ -14,11 +14,11 @@ type (
func DB(cmd *cobra.Command, pfix string) (o *DBOpt) {
o = &DBOpt{}
bindString(cmd, &o.DSN,
BindString(cmd, &o.DSN,
pFlag(pfix, "db-dsn"), "corteza:corteza@tcp(db:3306)/corteza?collation=utf8mb4_general_ci",
"DSN for database connection")
bindString(cmd, &o.Profiler,
BindString(cmd, &o.Profiler,
pFlag(pfix, "db-profiler"), "none",
"Profiler for DB queries (none, stdout, logger)")
pkg/cli/flags/flags.go
+4 -4
View File
@@ -23,7 +23,7 @@ func envKey(s string) string {
return strings.ToUpper(strings.ReplaceAll(s, "-", "_"))
}
func bindString(cmd *cobra.Command, v *string, flag, def string, desc string) {
func BindString(cmd *cobra.Command, v *string, flag, def string, desc string) {
if env, has := os.LookupEnv(envKey(flag)); has {
def = cast.ToString(env)
}
@@ -31,7 +31,7 @@ func bindString(cmd *cobra.Command, v *string, flag, def string, desc string) {
cmd.Flags().StringVar(v, flag, def, desc)
}
func bindBool(cmd *cobra.Command, v *bool, flag string, def bool, desc string) {
func BindBool(cmd *cobra.Command, v *bool, flag string, def bool, desc string) {
if env, has := os.LookupEnv(envKey(flag)); has {
def = cast.ToBool(env)
}
@@ -39,7 +39,7 @@ func bindBool(cmd *cobra.Command, v *bool, flag string, def bool, desc string) {
cmd.Flags().BoolVar(v, flag, def, desc)
}
func bindInt(cmd *cobra.Command, v *int, flag string, def int, desc string) {
func BindInt(cmd *cobra.Command, v *int, flag string, def int, desc string) {
if env, has := os.LookupEnv(envKey(flag)); has {
def = cast.ToInt(env)
}
@@ -47,7 +47,7 @@ func bindInt(cmd *cobra.Command, v *int, flag string, def int, desc string) {
cmd.Flags().IntVar(v, flag, def, desc)
}
func bindDuration(cmd *cobra.Command, v *time.Duration, flag string, def time.Duration, desc string) {
func BindDuration(cmd *cobra.Command, v *time.Duration, flag string, def time.Duration, desc string) {
if env, has := os.LookupEnv(envKey(flag)); has {
def = cast.ToDuration(env)
}
pkg/cli/flags/http.go
+10 -10
View File
@@ -26,44 +26,44 @@ type (
func HTTP(cmd *cobra.Command, pfix string) (o *HTTPOpt) {
o = &HTTPOpt{}
bindString(cmd, &o.Addr,
BindString(cmd, &o.Addr,
pFlag(pfix, "http-addr"), ":80",
"Listen address for HTTP server")
bindBool(cmd, &o.Logging,
BindBool(cmd, &o.Logging,
pFlag(pfix, "http-log"), true,
"Enable/disable HTTP request log")
bindBool(cmd, &o.Pretty,
BindBool(cmd, &o.Pretty,
pFlag(pfix, "http-pretty-json"), false,
"Prettify returned JSON output")
bindBool(cmd, &o.Tracing,
BindBool(cmd, &o.Tracing,
pFlag(pfix, "http-error-tracing"), false,
"Return error stack frame")
bindBool(cmd, &o.EnableVersionRoute,
BindBool(cmd, &o.EnableVersionRoute,
pFlag(pfix, "http-enable-version-route"), true,
"Enable /version route")
bindBool(cmd, &o.EnableDebugRoute,
BindBool(cmd, &o.EnableDebugRoute,
pFlag(pfix, "http-enable-debug-route"), false,
"Enable /debug route with pprof data")
bindBool(cmd, &o.EnableMetrics,
BindBool(cmd, &o.EnableMetrics,
pFlag(pfix, "http-metrics"), false,
"Enable metrics")
bindString(cmd, &o.MetricsServiceLabel,
BindString(cmd, &o.MetricsServiceLabel,
pFlag(pfix, "http-metrics-name"), "corteza",
"Provide metrics service label for Prometheus")
bindString(cmd, &o.MetricsUsername,
BindString(cmd, &o.MetricsUsername,
pFlag(pfix, "http-metrics-username"), "metrics",
"Provide metrics username for Prometheus")
// Setting metrics password to random string to prevent security accidents...
bindString(cmd, &o.MetricsPassword,
BindString(cmd, &o.MetricsPassword,
pFlag(pfix, "http-metrics-password"), string(rand.Bytes(5)),
"Provide metrics password for Prometheus")
pkg/cli/flags/http_client.go
+2 -2
View File
@@ -16,11 +16,11 @@ type (
func HttpClient(cmd *cobra.Command) (o *HttpClientOpt) {
o = &HttpClientOpt{}
bindBool(cmd, &o.ClientTSLInsecure,
BindBool(cmd, &o.ClientTSLInsecure,
"http-client-tsl-insecure", false,
"Skip insecure TSL verification on outbound HTTP requests (allow invalid/self-signed certificates")
bindDuration(cmd, &o.HttpClientTimeout,
BindDuration(cmd, &o.HttpClientTimeout,
"http-client-timeout", 30*time.Second,
"Default HTTP client timeout")
pkg/cli/flags/jwt.go
+2 -2
View File
@@ -17,11 +17,11 @@ func JWT(cmd *cobra.Command) (o *JWTOpt) {
o = &JWTOpt{}
// Setting JWT secret to random string to prevent security accidents...
bindString(cmd, &o.Secret,
BindString(cmd, &o.Secret,
"auth-jwt-secret", string(rand.Bytes(32)),
"JWT Secret")
bindInt(cmd, &o.Expiry,
BindInt(cmd, &o.Expiry,
"auth-jwt-expiry", 60*24*30,
"JWT Expiration in minutes")
pkg/cli/flags/log.go
+3 -6
View File
@@ -5,22 +5,19 @@ import (
)
type (
// Logger's output leve is configured here, but
// dev/prod configuration happens earlier
LogOpt struct {
Level string
JSON bool
}
)
func Log(cmd *cobra.Command) (o *LogOpt) {
o = &LogOpt{}
bindString(cmd, &o.Level,
BindString(cmd, &o.Level,
"log-level", "info",
"Log level (debug, info, warn, error, panic, fatal)")
bindBool(cmd, &o.JSON,
"log-json", true,
"Log in JSON format")
return
}
pkg/cli/flags/monitor.go
+6 -4
View File
@@ -1,21 +1,23 @@
package flags
import (
"time"
"github.com/spf13/cobra"
)
type (
MonitorOpt struct {
Interval int
Interval time.Duration
}
)
func Monitor(cmd *cobra.Command, pfix string) (o *MonitorOpt) {
o = &MonitorOpt{}
bindInt(cmd, &o.Interval,
pFlag(pfix, "monitor-interval"), 300,
"Monitor interval (seconds, 0 = disable)")
BindDuration(cmd, &o.Interval,
pFlag(pfix, "monitor-interval"), 300*time.Second,
"Monitor interval (0 = disable)")
return
}
pkg/cli/flags/provision.go
+10 -4
View File
@@ -6,16 +6,22 @@ import (
type (
ProvisionOpt struct {
Database bool
MigrateDatabase bool
AutoSetup bool
}
)
func Provision(cmd *cobra.Command, pfix string) (o *ProvisionOpt) {
o = &ProvisionOpt{}
bindBool(cmd, &o.Database,
pFlag(pfix, "provision-database"), true,
"Run database migration scripts")
BindBool(cmd, &o.MigrateDatabase,
pFlag(pfix, "provision-migrate-database"), true,
"Run database migration")
BindBool(cmd, &o.AutoSetup,
pFlag(pfix, "provision-auto-setup"), true,
"Run auto-setup procedures on service")
return
}
pkg/cli/flags/pubsub.go
+6 -6
View File
@@ -30,27 +30,27 @@ func PubSub(cmd *cobra.Command, pfix string) (o *PubSubOpt) {
pingPeriod = (pingTimeout * 9) / 10
)
bindString(cmd, &o.Mode,
BindString(cmd, &o.Mode,
pFlag(pfix, "pubsub-mode"), "poll",
"Pub/Sub mode (poll, redis")
bindDuration(cmd, &o.RedisPingTimeout,
BindDuration(cmd, &o.RedisPingTimeout,
pFlag(pfix, "pubsub-polling-interval"), timeout,
"Sub/Sub polling interval")
bindString(cmd, &o.RedisAddr,
BindString(cmd, &o.RedisAddr,
pFlag(pfix, "pubsub-redis-addr"), "redis:6379",
"Pub/Sub mode (poll, redis")
bindDuration(cmd, &o.RedisTimeout,
BindDuration(cmd, &o.RedisTimeout,
pFlag(pfix, "pubsub-redis-timeout"), timeout,
"Websocket connection timeout")
bindDuration(cmd, &o.RedisPingTimeout,
BindDuration(cmd, &o.RedisPingTimeout,
pFlag(pfix, "pubsub-redis-ping-timeout"), pingTimeout,
"Pub/Sub connection ping timeout")
bindDuration(cmd, &o.RedisPingPeriod,
BindDuration(cmd, &o.RedisPingPeriod,
pFlag(pfix, "pubsub-redis-ping-period"), pingPeriod,
"Pub/Sub connection ping period (should be lower than timeout)")
pkg/cli/flags/smtp.go
+5 -5
View File
@@ -17,23 +17,23 @@ type (
func SMTP(cmd *cobra.Command) (o *SMTPOpt) {
o = &SMTPOpt{}
bindString(cmd, &o.Host,
BindString(cmd, &o.Host,
"smtp-host", "localhost:25",
"SMTP hostname")
bindString(cmd, &o.User,
BindString(cmd, &o.User,
"smtp-username", "",
"SMTP server username")
bindString(cmd, &o.Pass,
BindString(cmd, &o.Pass,
"smtp-pass", "",
"SMTP server password")
bindString(cmd, &o.From,
BindString(cmd, &o.From,
"smtp-from", "",
"Sender's email address")
bindInt(cmd, &o.Port,
BindInt(cmd, &o.Port,
"smtp-port", 25,
"SMTP port number")
pkg/cli/flags/websocket.go
+3 -3
View File
@@ -23,15 +23,15 @@ func Websocket(cmd *cobra.Command, pfix string) (o *WebsocketOpt) {
pingPeriod = (pingTimeout * 9) / 10
)
bindDuration(cmd, &o.Timeout,
BindDuration(cmd, &o.Timeout,
pFlag(pfix, "websocket-timeout"), timeout,
"Websocket connection timeout")
bindDuration(cmd, &o.PingTimeout,
BindDuration(cmd, &o.PingTimeout,
pFlag(pfix, "websocket-ping-timeout"), pingTimeout,
"Websocket connection ping timeout")
bindDuration(cmd, &o.PingPeriod,
BindDuration(cmd, &o.PingPeriod,
pFlag(pfix, "websocket-ping-period"), pingPeriod,
"Websocket connection ping period (should be lower than timeout)")
pkg/cli/helpers.go
+13 -83
View File
@@ -1,9 +1,9 @@
package cli
import (
"context"
"fmt"
"os"
"github.com/spf13/cobra"
"go.uber.org/zap"
"github.com/cortezaproject/corteza-server/internal/auth"
@@ -13,90 +13,11 @@ import (
"github.com/cortezaproject/corteza-server/pkg/logger"
)
// SetupProvisionCommands sets-up standard provision commands
// Deprecated: use SetupProvisionSubCommands
func SetupProvisionCommands(ac func() error, md func() error) *cobra.Command {
var (
cmd = &cobra.Command{
Use: "provision",
Short: "Provision tasks",
}
)
// Add only commands with defined callbacks
if ac != nil {
cmd.AddCommand(&cobra.Command{
Use: "access-control-rules",
Short: "Reset access control rules & roles",
RunE: func(cmd *cobra.Command, args []string) error {
return ac()
},
})
}
// Add only commands with defined callbacks
if md != nil {
cmd.AddCommand(&cobra.Command{
Use: "migrate-database",
Short: "Run database migration scripts",
RunE: func(cmd *cobra.Command, args []string) error {
return md()
},
})
}
return cmd
}
type (
provisioner interface {
ProvisionMigrateDatabase(ctx context.Context) error
ProvisionAccessControl(ctx context.Context) error
}
)
func SetupProvisionSubcommands(ctx context.Context, p provisioner) *cobra.Command {
var (
cmd = &cobra.Command{
Use: "provision",
Short: "Provision tasks",
}
)
// Add only commands with defined callbacks
cmd.AddCommand(&cobra.Command{
Use: "access-control-rules",
Short: "Reset access control rules & roles",
RunE: func(cmd *cobra.Command, args []string) error {
return p.ProvisionAccessControl(ctx)
},
})
// Add only commands with defined callbacks
cmd.AddCommand(&cobra.Command{
Use: "migrate-database",
Short: "Run database migration scripts",
RunE: func(cmd *cobra.Command, args []string) error {
return p.ProvisionMigrateDatabase(ctx)
},
})
return cmd
}
func InitGeneralServices(logOpt *flags.LogOpt, smtpOpt *flags.SMTPOpt, jwtOpt *flags.JWTOpt, httpClientOpt *flags.HttpClientOpt) {
// Reset logger's level to whatever we want
var logLevel = zap.InfoLevel
_ = logLevel.Set(logOpt.Level)
if logger.Default() == nil {
logger.Init(logLevel)
} else {
logger.DefaultLevel.SetLevel(logLevel)
}
logger.DefaultLevel.SetLevel(logLevel)
auth.SetupDefault(jwtOpt.Secret, jwtOpt.Expiry)
mail.SetupDialer(smtpOpt.Host, smtpOpt.Port, smtpOpt.User, smtpOpt.Pass, smtpOpt.From)
@@ -105,3 +26,12 @@ func InitGeneralServices(logOpt *flags.LogOpt, smtpOpt *flags.SMTPOpt, jwtOpt *f
httpClientOpt.ClientTSLInsecure,
)
}
func HandleError(err error) {
if err == nil {
return
}
_, _ = fmt.Fprintln(os.Stderr, err.Error())
os.Exit(1)
}
pkg/cli/runner.go
+315
View File
@@ -0,0 +1,315 @@
package cli
import (
"context"
"github.com/go-chi/chi"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"go.uber.org/zap"
"github.com/cortezaproject/corteza-server/internal/db"
"github.com/cortezaproject/corteza-server/pkg/api"
"github.com/cortezaproject/corteza-server/pkg/cli/flags"
"github.com/cortezaproject/corteza-server/pkg/logger"
)
type (
Runner func(ctx context.Context, cmd *cobra.Command, c *Config) error
Runners []Runner
CommandMaker func(ctx context.Context, c *Config) *cobra.Command
CommandMakers []CommandMaker
FlagBinder func(cmd *cobra.Command, c *Config)
FlagBinders []FlagBinder
Mounter func(r chi.Router)
Mounters []Mounter
Config struct {
init bool
// Service name (messaging, system...)
// See comments on other fields for how it is used.
ServiceName string
// Logger name for internal services, defaults to ServiceName
LoggerName string
Log *zap.Logger
// General options/flags
LogOpt *flags.LogOpt
SmtpOpt *flags.SMTPOpt
JwtOpt *flags.JWTOpt
HttpClientOpt *flags.HttpClientOpt
// Per-service options/flags
DbOpt *flags.DBOpt
ProvisionOpt *flags.ProvisionOpt
// DB Connection name, defaults to ServiceName
DatabaseName string
// Root command name, , defaults to "corteza-server-<ServiceName>"
RootCommandName string
// Flags that are bond to root command, no (per-service) prefixed
RootCommandBaseFlags FlagBinders
// Prefix for flags for root command, defaults to ServiceName
RootCommandFlagsPrefix string
// Flags that are bond to root command, (per-service) prefixed
RootCommandPrefixedFlags FlagBinders
// Database setup/connection procedure
// Runner autobinds default runner that tries to connect using DbOpt.DSN
RootCommandDBSetup Runners
// All that needs to be initialized before any sub-comman is executed
RootCommandPreRun Runners
// ******************************************************************
// API Server instance
ApiServer *api.Server
// API Server command name
ApiServerCommandName string
// Prefix for "serve-api command flags", defaults to ServiceName
ApiServerFlagsPrefix string
// Additional command flags for API server
ApiServerAdtFlags FlagBinders
// Code that needs to be executed before HTTP server is started
ApiServerPreRun Runners
// Routers that we mount on HTTP server
ApiServerRoutes Mounters
// Sets-up all available subcommands.
AdtSubCommands CommandMakers
// Database migration code
// This is used for "provision migrate-database" command and after db connection is
// established (if --provision-migrate-database is enabled
ProvisionMigrateDatabase Runners
// Access control initial setup
// Reapplies default access control rules for roles "everyone" [1] and "admin" [2]
ProvisionAccessControl Runners
}
)
func init() {
// Have logger ready in case we need to log anything
// before it gets properly initialized through InitGeneralServices
logger.Init()
}
func (rr Runners) Run(ctx context.Context, cmd *cobra.Command, c *Config) (err error) {
for i := range rr {
err = rr[i](ctx, cmd, c)
if err != nil {
return
}
}
return
}
func (rr Mounters) MountRoutes(r chi.Router) {
for i := range rr {
rr[i](r)
}
}
func (bb FlagBinders) Bind(cmd *cobra.Command, c *Config) {
for i := range bb {
bb[i](cmd, c)
}
}
func (mm CommandMakers) Make(ctx context.Context, c *Config) []*cobra.Command {
var (
valid = make([]*cobra.Command, 0)
cmd *cobra.Command
)
for i := range mm {
if cmd = mm[i](ctx, c); cmd != nil {
valid = append(valid, cmd)
}
}
return valid
}
func CombineFlagBinders(rr ...FlagBinders) (out FlagBinders) {
for i := range rr {
out = append(out, rr[i]...)
}
return out
}
func (c *Config) Init() {
if c.init {
return
}
if c.Log == nil {
c.Log = logger.Default()
}
if c.LoggerName == "" {
c.LoggerName = c.ServiceName
}
c.Log = c.Log.Named(c.LoggerName)
if c.RootCommandName == "" {
c.RootCommandName = "corteza-server-" + c.ServiceName
}
if c.ApiServerCommandName == "" {
c.ApiServerCommandName = "serve-api"
}
if c.ApiServerFlagsPrefix == "" {
c.ApiServerFlagsPrefix = c.ServiceName
}
if c.DatabaseName == "" {
c.DatabaseName = c.ServiceName
}
if c.RootCommandDBSetup == nil {
c.RootCommandDBSetup = Runners{func(ctx context.Context, cmd *cobra.Command, c *Config) (err error) {
_, err = db.TryToConnect(ctx, c.Log, c.DatabaseName, c.DbOpt.DSN, c.DbOpt.Profiler)
if err != nil {
return errors.Wrap(err, "could not connect to database")
}
return
}}
}
// Flags, not prefixed with service name
if c.RootCommandBaseFlags == nil {
c.RootCommandBaseFlags = FlagBinders{
func(cmd *cobra.Command, c *Config) {
c.LogOpt = flags.Log(cmd)
c.SmtpOpt = flags.SMTP(cmd)
c.JwtOpt = flags.JWT(cmd)
c.HttpClientOpt = flags.HttpClient(cmd)
},
}
}
if c.RootCommandFlagsPrefix == "" {
c.RootCommandFlagsPrefix = c.ServiceName
}
// Flags, prefixed with service name
if c.RootCommandPrefixedFlags == nil {
c.RootCommandPrefixedFlags = FlagBinders{
func(cmd *cobra.Command, c *Config) {
c.DbOpt = flags.DB(cmd, c.ServiceName)
c.ProvisionOpt = flags.Provision(cmd, c.ServiceName)
},
}
}
if c.ApiServer == nil {
c.ApiServer = api.NewServer(c.Log)
}
for i := range c.ApiServerRoutes {
c.ApiServer.MountRoutes(c.ApiServerRoutes[i])
}
}
// MakeCLI creates command line interface
//
// It tries to construct "serve-api" and "provision" sub-commands
// if configured properly (see Config struct)
func (c *Config) MakeCLI(ctx context.Context) (cmd *cobra.Command) {
c.Init()
cmd = &cobra.Command{
Use: c.RootCommandName,
TraverseChildren: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
InitGeneralServices(c.LogOpt, c.SmtpOpt, c.JwtOpt, c.HttpClientOpt)
err = c.RootCommandDBSetup.Run(ctx, cmd, c)
if err != nil {
c.Log.Error("Failed to connect to the database", zap.Error(err))
return nil
}
err = c.RootCommandPreRun.Run(ctx, cmd, c)
if err != nil {
c.Log.Error("Failed to run command pre-run scripts", zap.Error(err))
return nil
}
return nil
},
}
c.RootCommandBaseFlags.Bind(cmd, c)
c.RootCommandPrefixedFlags.Bind(cmd, c)
serveApiCmd := c.ApiServer.Command(ctx, c.ApiServerCommandName, c.ApiServerFlagsPrefix, func(ctx context.Context) (err error) {
return c.ApiServerPreRun.Run(ctx, cmd, c)
})
// Bind all flags we need for serving the API
c.ApiServerAdtFlags.Bind(serveApiCmd, c)
cmd.AddCommand(serveApiCmd)
if len(c.ProvisionMigrateDatabase) > 0 || len(c.ProvisionAccessControl) > 0 {
var (
provisionCmd = &cobra.Command{
Use: "provision",
Short: "Provision tasks",
}
)
// Add only commands with defined callbacks
if len(c.ProvisionMigrateDatabase) > 0 {
provisionCmd.AddCommand(&cobra.Command{
Use: "access-control-rules",
Short: "Reset access control rules & roles",
RunE: func(cmd *cobra.Command, args []string) error {
return c.ProvisionAccessControl.Run(ctx, nil, c)
},
})
}
// Add only commands with defined callbacks
if len(c.ProvisionAccessControl) > 0 {
provisionCmd.AddCommand(&cobra.Command{
Use: "migrate-database",
Short: "Run database migration scripts",
RunE: func(cmd *cobra.Command, args []string) error {
return c.ProvisionMigrateDatabase.Run(ctx, nil, c)
},
})
}
cmd.AddCommand(provisionCmd)
}
cmd.AddCommand(c.AdtSubCommands.Make(ctx, c)...)
return
}