DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Improve cli env setup & running

Browse Source
This commit is contained in:
Denis Arh
2019-05-24 12:38:45 +02:00
parent 36b32fa2fe
commit 22476933f4
89 changed files with 945 additions and 907 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/compose/spec/module.json
+1 -1
View File
@@ -156,4 +156,4 @@
}
}
]
}
}
api/compose/spec/permissions.json
+1 -1
View File
@@ -96,4 +96,4 @@
}
}
]
}
}
api/compose/spec/record.json
+1 -1
View File
@@ -194,4 +194,4 @@
}
}
]
}
}
api/messaging/spec/permissions.json
+1 -1
View File
@@ -96,4 +96,4 @@
}
}
]
}
}
api/messaging/spec/webhooks.json
+1 -1
View File
@@ -192,4 +192,4 @@
}
}
]
}
}
api/system/spec/permissions.json
+1 -1
View File
@@ -96,4 +96,4 @@
}
}
]
}
}
api/system/spec/user.json
+1 -1
View File
@@ -185,4 +185,4 @@
}
}
]
}
}
cmd/compose/main.go
+3 -8
View File
@@ -1,17 +1,12 @@
package main
import (
"fmt"
"os"
"github.com/cortezaproject/corteza-server/compose"
"github.com/cortezaproject/corteza-server/pkg/cli"
)
func main() {
c := compose.InitCompose()
if err := c.Command(cli.Context()).Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
cfg := compose.Configure()
cmd := cfg.MakeCLI(cli.Context())
cli.HandleError(cmd.Execute())
}
cmd/messaging/main.go
+3 -8
View File
@@ -1,17 +1,12 @@
package main
import (
"fmt"
"os"
"github.com/cortezaproject/corteza-server/messaging"
"github.com/cortezaproject/corteza-server/pkg/cli"
)
func main() {
m := messaging.InitMessaging()
if err := m.Command(cli.Context()).Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
cfg := messaging.Configure()
cmd := cfg.MakeCLI(cli.Context())
cli.HandleError(cmd.Execute())
}
cmd/monolith/main.go
+3 -8
View File
@@ -1,17 +1,12 @@
package main
import (
"fmt"
"os"
"github.com/cortezaproject/corteza-server/monolith"
"github.com/cortezaproject/corteza-server/pkg/cli"
)
func main() {
c := monolith.InitMonolith()
if err := c.Command(cli.Context()).Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
cfg := monolith.Configure()
cmd := cfg.MakeCLI(cli.Context())
cli.HandleError(cmd.Execute())
}
cmd/system/main.go
+3 -8
View File
@@ -1,17 +1,12 @@
package main
import (
"fmt"
"os"
"github.com/cortezaproject/corteza-server/pkg/cli"
"github.com/cortezaproject/corteza-server/system"
)
func main() {
s := system.InitSystem()
if err := s.Command(cli.Context()).Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
cfg := system.Configure()
cmd := cfg.MakeCLI(cli.Context())
cli.HandleError(cmd.Execute())
}
codegen/templates/http_handlers_inline.tpl
+1 -1
View File
@@ -11,7 +11,7 @@ import (
"github.com/titpetric/factory/resputil"
"github.com/cortezaproject/corteza-server/{project}/rest/request"
"github.com/cortezaproject/corteza-server/internal/logger"
"github.com/cortezaproject/corteza-server/pkg/logger"
)
// Internal API interface
compose/compose.go
+51 -132
View File
@@ -3,158 +3,77 @@ package compose
import (
"context"
"github.com/go-chi/chi"
_ "github.com/joho/godotenv/autoload"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/titpetric/factory"
"go.uber.org/zap"
migrate "github.com/cortezaproject/corteza-server/compose/db"
"github.com/cortezaproject/corteza-server/compose/internal/service"
"github.com/cortezaproject/corteza-server/compose/rest"
"github.com/cortezaproject/corteza-server/internal/db"
"github.com/cortezaproject/corteza-server/pkg/api"
"github.com/cortezaproject/corteza-server/pkg/cli"
"github.com/cortezaproject/corteza-server/pkg/cli/flags"
"github.com/cortezaproject/corteza-server/pkg/logger"
)
const (
compose = "compose"
)
type (
Compose struct {
log *zap.Logger
// General
logOpt *flags.LogOpt
smtpOpt *flags.SMTPOpt
jwtOpt *flags.JWTOpt
httpClientOpt *flags.HttpClientOpt
// Compose specific
dbOpt *flags.DBOpt
provisionOpt *flags.ProvisionOpt
}
)
func init() {
logger.Init(zap.DebugLevel)
}
func InitCompose() *Compose {
return &Compose{
log: logger.Default().Named(compose),
}
}
// Command produces cobra.Command
func (m *Compose) Command(ctx context.Context) (cmd *cobra.Command) {
cmd = &cobra.Command{
Use: "corteza-server-compose",
TraverseChildren: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
cli.InitGeneralServices(m.logOpt, m.smtpOpt, m.jwtOpt, m.httpClientOpt)
return m.StartServices(ctx)
},
}
m.BindGlobalFlags(cmd)
srv := api.NewServer(m.log)
serveApiCmd := srv.Command(ctx, compose, m.ApiServerPreRun)
// Bind all flags we need for serving compose
m.BindApiServerFlags(serveApiCmd)
srv.MountRoutes(m.ApiServerRoutes)
cmd.AddCommand(
serveApiCmd,
cli.SetupProvisionSubcommands(ctx, m),
func Configure() *cli.Config {
var (
accessControlSetup = func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
// Calling grant directly on internal permissions service to avoid AC check for "grant"
var p = service.DefaultPermissions
var ac = service.DefaultAccessControl
return p.Grant(ctx, ac.Whitelist(), ac.DefaultRules()...)
}
)
m.AddCommands(cmd, ctx)
return &cli.Config{
ServiceName: compose,
return
}
RootCommandPreRun: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) (err error) {
if c.ProvisionOpt.MigrateDatabase {
cli.HandleError(c.ProvisionMigrateDatabase.Run(ctx, cmd, c))
}
// AddCommands - other commands that this subservice needs
func (m *Compose) AddCommands(cmd *cobra.Command, ctx context.Context) {}
cli.HandleError(service.Init(ctx, c.Log))
// Binds all global flags
func (m *Compose) BindGlobalFlags(cmd *cobra.Command) {
m.logOpt = flags.Log(cmd)
m.smtpOpt = flags.SMTP(cmd)
m.jwtOpt = flags.JWT(cmd)
m.httpClientOpt = flags.HttpClient(cmd)
}
if c.ProvisionOpt.AutoSetup {
cli.HandleError(accessControlSetup(ctx, cmd, c))
}
// BindApiServerFlags sets & binds all API server flags
func (m *Compose) BindApiServerFlags(cmd *cobra.Command) {
m.dbOpt = flags.DB(cmd, compose)
m.provisionOpt = flags.Provision(cmd, compose)
}
return
},
},
// StartServices
func (m *Compose) StartServices(ctx context.Context) (err error) {
_, err = db.TryToConnect(ctx, m.log, compose, m.dbOpt.DSN, m.dbOpt.Profiler)
if err != nil {
return errors.Wrap(err, "could not connect to database")
ApiServerPreRun: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
go service.Watchers(ctx)
return nil
},
},
ApiServerRoutes: cli.Mounters{
rest.MountRoutes,
},
ProvisionMigrateDatabase: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
var db, err = factory.Database.Get(compose)
if err != nil {
return err
}
db = db.With(ctx)
// Disable profiler for migrations
db.Profiler = nil
return migrate.Migrate(db, c.Log)
},
},
ProvisionAccessControl: cli.Runners{
accessControlSetup,
},
}
if m.provisionOpt.Database {
err = m.ProvisionMigrateDatabase(ctx)
if err != nil {
return
}
}
err = service.Init(ctx)
if err != nil {
return
}
return
}
// ApiServerPreRun is executed before serve-api command runs REST API server
//
// Should initialize all that needs to run in the background
func (m Compose) ApiServerPreRun(ctx context.Context) error {
service.DefaultPermissions.Watch(ctx)
return nil
}
// ApiServerRoutes mounts api server routes
func (m *Compose) ApiServerRoutes(r chi.Router) {
rest.MountRoutes(r)
}
// ProvisionMigrateDatabase migrates database to new version
//
// This is ran by default on serve-api (when not explicitly disabled with --compose-provision-database=false)
// or on demand with "provision migrate-database"
func (m Compose) ProvisionMigrateDatabase(ctx context.Context) error {
var db, err = factory.Database.Get(compose)
if err != nil {
return err
}
db = db.With(ctx)
// Disable profiler for migrations
db.Profiler = nil
return migrate.Migrate(db)
}
// ProvisionAccessControl resets access-control rules for roles admin (2) and everyone (1)
//
// Run with emand with "provision access-control-rules"
func (m Compose) ProvisionAccessControl(ctx context.Context) error {
var ac = service.DefaultAccessControl
return ac.Grant(ctx, ac.DefaultRules()...)
}
compose/compose_test.go
+15
View File
@@ -0,0 +1,15 @@
package compose
import (
"context"
"testing"
"github.com/cortezaproject/corteza-server/internal/test"
)
func TestConfigure(t *testing.T) {
var config = Configure()
test.Assert(t, config != nil, "Configure valid")
test.Assert(t, func() bool { config.Init(); return true }(), "Initialization ok")
test.Assert(t, config.MakeCLI(context.Background()) != nil, "CLI created")
}
compose/db/migrate.go
+14 -10
View File
@@ -2,7 +2,6 @@ package db
import (
"fmt"
"log"
"os"
"path/filepath"
"regexp"
@@ -12,6 +11,7 @@ import (
"github.com/goware/statik/fs"
"github.com/pkg/errors"
"github.com/titpetric/factory"
"go.uber.org/zap"
"github.com/cortezaproject/corteza-server/compose/db/mysql"
)
@@ -23,10 +23,12 @@ func statements(contents []byte, err error) ([]string, error) {
return regexp.MustCompilePOSIX(";$").Split(string(contents), -1), nil
}
func Migrate(db *factory.DB) error {
func Migrate(db *factory.DB, log *zap.Logger) error {
log = log.Named("database.migrations")
statikFS, err := fs.New(mysql.Asset)
if err != nil {
return errors.Wrap(err, "Error creating statik filesystem")
return errors.Wrap(err, "error creating statik filesystem")
}
var files []string
@@ -37,17 +39,18 @@ func Migrate(db *factory.DB) error {
if matched {
files = append(files, filename)
}
return err
}
if err = fs.Walk(statikFS, "/", fn); err != nil {
return errors.Wrap(err, "Error when listing files for migrations")
if err := fs.Walk(statikFS, "/", fn); err != nil {
return errors.Wrap(err, "error when listing files for migrations")
}
sort.Strings(files)
if len(files) == 0 {
return errors.New("No files encoded for migration, need at least one SQL file")
return errors.New("no files encoded for migration, need at least one SQL file")
}
migrate := func(filename string, useLog bool) error {
@@ -67,19 +70,20 @@ func Migrate(db *factory.DB) error {
up := func() error {
stmts, err := statements(fs.ReadFile(statikFS, filename))
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Error reading migration %s", filename))
return errors.Wrap(err, fmt.Sprintf("error reading migration %s", filename))
}
log.Println("Running migration for", filename)
log.Debug("Running migration", zap.String("filename", filename))
for idx, query := range stmts {
if strings.TrimSpace(query) != "" && idx >= status.StatementIndex {
status.StatementIndex = idx
if _, err := db.Exec(query); err != nil {
log.Debug("migration error ", zap.String("filename", filename), zap.Error(err))
return err
}
}
}
log.Println("Migration done OK")
status.Status = "ok"
return nil
}
@@ -90,7 +94,7 @@ func Migrate(db *factory.DB) error {
}
if useLog {
if err := db.Replace("migrations", status); err != nil {
log.Println("replace failed", err)
return errors.Wrap(err, "migration update failed")
}
}
return err
compose/db/migrate_test.go
+2 -1
View File
@@ -6,13 +6,14 @@ import (
"os"
"testing"
"github.com/cortezaproject/corteza-server/pkg/logger"
"github.com/titpetric/factory"
)
func TestMigrations(t *testing.T) {
factory.Database.Add("compose", os.Getenv("COMPOSE_DB_DSN"))
db := factory.Database.MustGet("compose")
if err := Migrate(db); err != nil {
if err := Migrate(db, logger.Default()); err != nil {
t.Fatalf("Unexpected error: %+v", err)
}
}
compose/db/mysql/static.go
+1 -1
View File
File diff suppressed because one or more lines are too long
compose/db/schema/mysql/20190526090000.permissions.up.sql
+8
View File
@@ -0,0 +1,8 @@
CREATE TABLE IF NOT EXISTS compose_permission_rules (
rel_role BIGINT UNSIGNED NOT NULL,
resource VARCHAR(128) NOT NULL,
operation VARCHAR(128) NOT NULL,
access TINYINT(1) NOT NULL,
PRIMARY KEY (rel_role, resource, operation)
) ENGINE=InnoDB;
compose/internal/repository/main_test.go
+1 -2
View File
@@ -8,13 +8,12 @@ import (
"time"
"github.com/titpetric/factory"
"go.uber.org/zap/zapcore"
"github.com/cortezaproject/corteza-server/pkg/logger"
)
func TestMain(m *testing.M) {
logger.Init(zapcore.DebugLevel)
logger.SetDefault(logger.MakeDebugLogger())
factory.Database.Add("compose", os.Getenv("COMPOSE_DB_DSN"))
db := factory.Database.MustGet("compose")
compose/internal/service/main_test.go
+2 -3
View File
@@ -10,7 +10,6 @@ import (
"time"
"github.com/titpetric/factory"
"go.uber.org/zap/zapcore"
composeMigrate "github.com/cortezaproject/corteza-server/compose/db"
"github.com/cortezaproject/corteza-server/compose/types"
@@ -25,14 +24,14 @@ type (
func (mockDB) Transaction(callback func() error) error { return callback() }
func TestMain(m *testing.M) {
logger.Init(zapcore.DebugLevel)
logger.SetDefault(logger.MakeDebugLogger())
factory.Database.Add("compose", os.Getenv("COMPOSE_DB_DSN"))
db := factory.Database.MustGet("compose")
db.Profiler = &factory.DatabaseProfilerStdout{}
// migrate database schema
if err := composeMigrate.Migrate(db); err != nil {
if err := composeMigrate.Migrate(db, logger.Default()); err != nil {
fmt.Printf("Error running migrations: %+v\n", err)
return
}
compose/internal/service/service.go
+6 -3
View File
@@ -9,7 +9,6 @@ import (
"github.com/cortezaproject/corteza-server/compose/internal/repository"
"github.com/cortezaproject/corteza-server/internal/permissions"
"github.com/cortezaproject/corteza-server/internal/store"
"github.com/cortezaproject/corteza-server/pkg/logger"
)
type (
@@ -36,8 +35,8 @@ var (
DefaultNamespace NamespaceService
)
func Init(ctx context.Context) error {
DefaultLogger = logger.Default().Named("compose.service")
func Init(ctx context.Context, log *zap.Logger) (err error) {
DefaultLogger = log.Named("service")
fs, err := store.New("var/store")
if err != nil {
@@ -63,6 +62,10 @@ func Init(ctx context.Context) error {
return nil
}
func Watchers(ctx context.Context) {
DefaultPermissions.Watch(ctx)
}
// Data is stale when new date does not match updatedAt or createdAt (before first update)
func isStale(new *time.Time, updatedAt *time.Time, createdAt time.Time) bool {
if new == nil {
compose/rest/handlers/attachment.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
compose/rest/handlers/chart.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
compose/rest/handlers/module.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
compose/rest/handlers/namespace.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
compose/rest/handlers/notification.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
compose/rest/handlers/page.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
compose/rest/handlers/permissions.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
compose/rest/handlers/record.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
compose/rest/handlers/trigger.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
compose/rest/request/module.go
+1 -2
View File
@@ -26,10 +26,9 @@ import (
"github.com/go-chi/chi"
"github.com/pkg/errors"
"time"
"github.com/cortezaproject/corteza-server/compose/types"
sqlxTypes "github.com/jmoiron/sqlx/types"
"time"
)
var _ = chi.URLParam
internal/permissions/service.go
+2 -2
View File
@@ -145,7 +145,7 @@ func (svc *service) Reload(ctx context.Context) {
defer svc.l.Unlock()
rr, err := svc.repository.With(ctx).Load()
svc.logger.Info(
svc.logger.Debug(
"reloading rules",
zap.Error(err),
zap.Int("before", len(svc.rules)),
@@ -167,7 +167,7 @@ func (svc service) flush(ctx context.Context) (err error) {
u.clear()
svc.rules = u
svc.logger.Info("flushed rules",
svc.logger.Debug("flushed rules",
zap.Int("updated", len(u)),
zap.Int("deleted", len(d)))
messaging/db/migrate.go
+13 -9
View File
@@ -2,7 +2,6 @@ package db
import (
"fmt"
"log"
"os"
"path/filepath"
"regexp"
@@ -12,6 +11,7 @@ import (
"github.com/goware/statik/fs"
"github.com/pkg/errors"
"github.com/titpetric/factory"
"go.uber.org/zap"
"github.com/cortezaproject/corteza-server/messaging/db/mysql"
)
@@ -23,10 +23,12 @@ func statements(contents []byte, err error) ([]string, error) {
return regexp.MustCompilePOSIX(";$").Split(string(contents), -1), nil
}
func Migrate(db *factory.DB) error {
func Migrate(db *factory.DB, log *zap.Logger) error {
log = log.Named("database.migrations")
statikFS, err := fs.New(mysql.Asset)
if err != nil {
return errors.Wrap(err, "Error creating statik filesystem")
return errors.Wrap(err, "error creating statik filesystem")
}
var files []string
@@ -37,17 +39,18 @@ func Migrate(db *factory.DB) error {
if matched {
files = append(files, filename)
}
return err
}
if err := fs.Walk(statikFS, "/", fn); err != nil {
return errors.Wrap(err, "Error when listing files for migrations")
return errors.Wrap(err, "error when listing files for migrations")
}
sort.Strings(files)
if len(files) == 0 {
return errors.New("No files encoded for migration, need at least one SQL file")
return errors.New("no files encoded for migration, need at least one SQL file")
}
migrate := func(filename string, useLog bool) error {
@@ -67,19 +70,20 @@ func Migrate(db *factory.DB) error {
up := func() error {
stmts, err := statements(fs.ReadFile(statikFS, filename))
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Error reading migration %s", filename))
return errors.Wrap(err, fmt.Sprintf("error reading migration %s", filename))
}
log.Println("Running migration for", filename)
log.Debug("Running migration", zap.String("filename", filename))
for idx, query := range stmts {
if strings.TrimSpace(query) != "" && idx >= status.StatementIndex {
status.StatementIndex = idx
if _, err := db.Exec(query); err != nil {
log.Debug("migration error ", zap.String("filename", filename), zap.Error(err))
return err
}
}
}
log.Println("Migration done OK")
status.Status = "ok"
return nil
}
@@ -90,7 +94,7 @@ func Migrate(db *factory.DB) error {
}
if useLog {
if err := db.Replace("migrations", status); err != nil {
log.Println("replace failed", err)
return errors.Wrap(err, "migration update failed")
}
}
return err
messaging/db/migrate_test.go
+2 -1
View File
@@ -6,6 +6,7 @@ import (
"os"
"testing"
"github.com/cortezaproject/corteza-server/pkg/logger"
"github.com/titpetric/factory"
)
@@ -13,7 +14,7 @@ func TestMigrations(t *testing.T) {
factory.Database.Add("messaging", os.Getenv("MESSAGING_DB_DSN"))
db := factory.Database.MustGet("messaging")
db.Profiler = &factory.Database.ProfilerStdout
if err := Migrate(db); err != nil {
if err := Migrate(db, logger.Default()); err != nil {
t.Fatalf("Unexpected error: %#v", err)
}
}
messaging/db/mysql/static.go
+1 -1
View File
File diff suppressed because one or more lines are too long
messaging/db/schema/mysql/20190526090000.permissions.up.sql
+8
View File
@@ -0,0 +1,8 @@
CREATE TABLE IF NOT EXISTS messaging_permission_rules (
rel_role BIGINT UNSIGNED NOT NULL,
resource VARCHAR(128) NOT NULL,
operation VARCHAR(128) NOT NULL,
access TINYINT(1) NOT NULL,
PRIMARY KEY (rel_role, resource, operation)
) ENGINE=InnoDB;
messaging/internal/repository/main_test.go
+2 -1
View File
@@ -10,6 +10,7 @@ import (
"github.com/titpetric/factory"
migrate "github.com/cortezaproject/corteza-server/messaging/db"
"github.com/cortezaproject/corteza-server/pkg/logger"
)
func TestMain(m *testing.M) {
@@ -18,7 +19,7 @@ func TestMain(m *testing.M) {
db.Profiler = &factory.Database.ProfilerStdout
// migrate database schema
if err := migrate.Migrate(db); err != nil {
if err := migrate.Migrate(db, logger.Default()); err != nil {
fmt.Printf("Error running migrations: %+v\n", err)
return
}
messaging/internal/service/main_test.go
+2 -3
View File
@@ -9,7 +9,6 @@ import (
"testing"
"github.com/titpetric/factory"
"go.uber.org/zap/zapcore"
messagingMigrate "github.com/cortezaproject/corteza-server/messaging/db"
"github.com/cortezaproject/corteza-server/pkg/logger"
@@ -20,14 +19,14 @@ type mockDB struct{}
func (mockDB) Transaction(callback func() error) error { return callback() }
func TestMain(m *testing.M) {
logger.Init(zapcore.DebugLevel)
logger.SetDefault(logger.MakeDebugLogger())
factory.Database.Add("messaging", os.Getenv("MESSAGING_DB_DSN"))
db := factory.Database.MustGet("messaging")
db.Profiler = &factory.Database.ProfilerStdout
// migrate database schema
if err := messagingMigrate.Migrate(db); err != nil {
if err := messagingMigrate.Migrate(db, logger.Default()); err != nil {
fmt.Printf("Error running migrations: %+v\n", err)
return
}
messaging/internal/service/service.go
+7 -4
View File
@@ -10,7 +10,6 @@ import (
"github.com/cortezaproject/corteza-server/internal/permissions"
"github.com/cortezaproject/corteza-server/internal/store"
"github.com/cortezaproject/corteza-server/messaging/internal/repository"
"github.com/cortezaproject/corteza-server/pkg/logger"
)
type (
@@ -39,7 +38,9 @@ var (
DefaultWebhook WebhookService
)
func Init(ctx context.Context) error {
func Init(ctx context.Context, log *zap.Logger) (err error) {
DefaultLogger = log.Named("service")
fs, err := store.New("var/store")
if err != nil {
return err
@@ -52,8 +53,6 @@ func Init(ctx context.Context) error {
return err
}
DefaultLogger = logger.Default().Named("messaging.service")
DefaultPermissions = permissions.Service(
ctx,
DefaultLogger,
@@ -70,6 +69,10 @@ func Init(ctx context.Context) error {
return nil
}
func Watchers(ctx context.Context) {
DefaultPermissions.Watch(ctx)
}
func timeNowPtr() *time.Time {
now := time.Now()
return &now
messaging/messaging.go
+66 -141
View File
@@ -5,173 +5,98 @@ import (
"github.com/go-chi/chi"
_ "github.com/joho/godotenv/autoload"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/titpetric/factory"
"go.uber.org/zap"
"github.com/cortezaproject/corteza-server/internal/db"
migrate "github.com/cortezaproject/corteza-server/messaging/db"
"github.com/cortezaproject/corteza-server/messaging/internal/service"
"github.com/cortezaproject/corteza-server/messaging/rest"
"github.com/cortezaproject/corteza-server/messaging/websocket"
"github.com/cortezaproject/corteza-server/pkg/api"
"github.com/cortezaproject/corteza-server/pkg/cli"
"github.com/cortezaproject/corteza-server/pkg/cli/flags"
"github.com/cortezaproject/corteza-server/pkg/logger"
)
const (
messaging = "messaging"
)
type (
Messaging struct {
log *zap.Logger
// General
logOpt *flags.LogOpt
smtpOpt *flags.SMTPOpt
jwtOpt *flags.JWTOpt
httpClientOpt *flags.HttpClientOpt
// Messaging specific
dbOpt *flags.DBOpt
provisionOpt *flags.ProvisionOpt
func Configure() *cli.Config {
var (
// Messaging API Server specific
websocketOpt *flags.WebsocketOpt
// Websocket handler
ws mounter
}
ws *websocket.Websocket
mounter interface {
ApiServerRoutes(r chi.Router)
}
)
func init() {
logger.Init(zap.DebugLevel)
}
func InitMessaging() *Messaging {
return &Messaging{
log: logger.Default().Named(messaging),
}
}
// Command produces cobra.Command
func (m *Messaging) Command(ctx context.Context) (cmd *cobra.Command) {
cmd = &cobra.Command{
Use: "corteza-server-messaging",
TraverseChildren: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
cli.InitGeneralServices(m.logOpt, m.smtpOpt, m.jwtOpt, m.httpClientOpt)
return m.StartServices(ctx)
},
}
m.BindGlobalFlags(cmd)
srv := api.NewServer(m.log)
serveApiCmd := srv.Command(ctx, messaging, m.ApiServerPreRun)
// Bind all flags we need for serving messaging
m.BindApiServerFlags(serveApiCmd)
srv.MountRoutes(m.ApiServerRoutes)
cmd.AddCommand(
serveApiCmd,
cli.SetupProvisionSubcommands(ctx, m),
accessControlSetup = func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
// Calling grant directly on internal permissions service to avoid AC check for "grant"
var p = service.DefaultPermissions
var ac = service.DefaultAccessControl
return p.Grant(ctx, ac.Whitelist(), ac.DefaultRules()...)
}
)
m.AddCommands(cmd, ctx)
return &cli.Config{
ServiceName: messaging,
return
}
RootCommandPreRun: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) (err error) {
if c.ProvisionOpt.MigrateDatabase {
cli.HandleError(c.ProvisionMigrateDatabase.Run(ctx, cmd, c))
}
// AddCommands - other commands that this subservice needs
func (m *Messaging) AddCommands(cmd *cobra.Command, ctx context.Context) {}
cli.HandleError(service.Init(ctx, c.Log))
// Binds all global flags
func (m *Messaging) BindGlobalFlags(cmd *cobra.Command) {
m.logOpt = flags.Log(cmd)
m.smtpOpt = flags.SMTP(cmd)
m.jwtOpt = flags.JWT(cmd)
m.httpClientOpt = flags.HttpClient(cmd)
}
ws = websocket.Init(ctx, &websocket.Config{
Timeout: websocketOpt.Timeout,
PingTimeout: websocketOpt.PingTimeout,
PingPeriod: websocketOpt.PingPeriod,
})
// BindApiServerFlags sets & binds all API server flags
func (m *Messaging) BindApiServerFlags(cmd *cobra.Command) {
m.dbOpt = flags.DB(cmd, messaging)
m.provisionOpt = flags.Provision(cmd, messaging)
m.websocketOpt = flags.Websocket(cmd, messaging)
if c.ProvisionOpt.AutoSetup {
cli.HandleError(accessControlSetup(ctx, cmd, c))
}
}
return
},
},
func (m *Messaging) StartServices(ctx context.Context) (err error) {
_, err = db.TryToConnect(ctx, m.log, messaging, m.dbOpt.DSN, m.dbOpt.Profiler)
if err != nil {
return errors.Wrap(err, "could not connect to database")
ApiServerAdtFlags: cli.FlagBinders{
func(cmd *cobra.Command, c *cli.Config) {
websocketOpt = flags.Websocket(cmd, messaging)
},
},
ApiServerPreRun: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
go service.Watchers(ctx)
return nil
},
},
ApiServerRoutes: cli.Mounters{
rest.MountRoutes,
// Wrap in func() to assure ws is set when mounted
func(r chi.Router) { ws.ApiServerRoutes(r) },
},
ProvisionMigrateDatabase: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
var db, err = factory.Database.Get(messaging)
if err != nil {
return err
}
db = db.With(ctx)
// Disable profiler for migrations
db.Profiler = nil
return migrate.Migrate(db, c.Log)
},
},
ProvisionAccessControl: cli.Runners{
accessControlSetup,
},
}
if m.provisionOpt.Database {
err = m.ProvisionMigrateDatabase(ctx)
if err != nil {
return
}
}
err = service.Init(ctx)
if err != nil {
return
}
m.ws = websocket.Init(ctx, &websocket.Config{
Timeout: m.websocketOpt.Timeout,
PingTimeout: m.websocketOpt.PingTimeout,
PingPeriod: m.websocketOpt.PingPeriod,
})
return
}
// ApiServerPreRun is executed before serve-api command runs REST API server
//
// Should initialize all that needs to run in the background
func (m Messaging) ApiServerPreRun(ctx context.Context) error {
service.DefaultPermissions.Watch(ctx)
return nil
}
// ApiServerRoutes mounts api server routes
func (m *Messaging) ApiServerRoutes(r chi.Router) {
rest.MountRoutes(r)
m.ws.ApiServerRoutes(r)
}
// ProvisionMigrateDatabase migrates database to new version
//
// This is ran by default on serve-api (when not explicitly disabled with --compose-provision-database=false)
// or on demand with "provision migrate-database"
func (m Messaging) ProvisionMigrateDatabase(ctx context.Context) error {
var db, err = factory.Database.Get(messaging)
if err != nil {
return err
}
db = db.With(ctx)
// Disable profiler for migrations
db.Profiler = nil
return migrate.Migrate(db)
}
// ProvisionAccessControl resets access-control rules for roles admin (2) and everyone (1)
//
// Run with emand with "provision access-control-rules"
func (m Messaging) ProvisionAccessControl(ctx context.Context) error {
var ac = service.DefaultAccessControl
return ac.Grant(ctx, ac.DefaultRules()...)
}
messaging/messaging_test.go
+15
View File
@@ -0,0 +1,15 @@
package messaging
import (
"context"
"testing"
"github.com/cortezaproject/corteza-server/internal/test"
)
func TestConfigure(t *testing.T) {
var config = Configure()
test.Assert(t, config != nil, "Configure valid")
test.Assert(t, func() bool { config.Init(); return true }(), "Initialization ok")
test.Assert(t, config.MakeCLI(context.Background()) != nil, "CLI created")
}
messaging/rest/handlers/activity.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
messaging/rest/handlers/attachment.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
messaging/rest/handlers/channel.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
messaging/rest/handlers/commands.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
messaging/rest/handlers/message.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
messaging/rest/handlers/permissions.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
messaging/rest/handlers/search.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
messaging/rest/handlers/status.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
messaging/rest/handlers/webhooks.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
messaging/rest/handlers/webhooks_public.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
messaging/websocket/router.go
+4
View File
@@ -29,6 +29,10 @@ func Init(ctx context.Context, config *Config) *Websocket {
go func() {
for {
if err := eq.feedSessions(ctx, events, store); err != nil {
if err == context.Canceled {
return
}
logger.Default().Error("session event feed error", zap.Error(err))
}
}
monolith/monolith.go
+94 -167
View File
@@ -6,190 +6,117 @@ import (
"github.com/go-chi/chi"
_ "github.com/joho/godotenv/autoload"
"github.com/spf13/cobra"
"go.uber.org/zap"
"github.com/cortezaproject/corteza-server/compose"
"github.com/cortezaproject/corteza-server/messaging"
"github.com/cortezaproject/corteza-server/pkg/api"
"github.com/cortezaproject/corteza-server/pkg/cli"
"github.com/cortezaproject/corteza-server/pkg/cli/flags"
"github.com/cortezaproject/corteza-server/pkg/logger"
"github.com/cortezaproject/corteza-server/system"
)
type (
// Sets up compose messaging & system subservices and runs them as one.
Monolith struct {
log *zap.Logger
func Configure() *cli.Config {
cmp := compose.Configure()
msg := messaging.Configure()
sys := system.Configure()
// General
logOpt *flags.LogOpt
smtpOpt *flags.SMTPOpt
jwtOpt *flags.JWTOpt
httpClientOpt *flags.HttpClientOpt
cmp.Init()
msg.Init()
sys.Init()
compose subservice
messaging subservice
system subservice
}
cmp.RootCommandBaseFlags = nil
msg.RootCommandBaseFlags = nil
sys.RootCommandBaseFlags = nil
subservice interface {
AddCommands(cmd *cobra.Command, ctx context.Context)
BindApiServerFlags(cmd *cobra.Command)
StartServices(ctx context.Context) (err error)
ApiServerPreRun(ctx context.Context) error
ApiServerRoutes(r chi.Router)
ProvisionMigrateDatabase(ctx context.Context) error
ProvisionAccessControl(ctx context.Context) error
}
// Combines all three services/apps and makes them run as one monolith app
return &cli.Config{
ServiceName: "",
runner func(context.Context) error
runners []runner
)
RootCommandDBSetup: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) (err error) {
cli.HandleError(cmp.RootCommandDBSetup.Run(ctx, cmd, cmp))
cli.HandleError(msg.RootCommandDBSetup.Run(ctx, cmd, msg))
cli.HandleError(sys.RootCommandDBSetup.Run(ctx, cmd, sys))
return
},
},
func (rr runners) run(ctx context.Context) (err error) {
for i := range rr {
err = rr[i](ctx)
if err != nil {
return
}
}
RootCommandName: "corteza-server",
RootCommandPreRun: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) (err error) {
cli.HandleError(cmp.RootCommandPreRun.Run(ctx, cmd, cmp))
cli.HandleError(msg.RootCommandPreRun.Run(ctx, cmd, msg))
cli.HandleError(sys.RootCommandPreRun.Run(ctx, cmd, sys))
return
},
},
return
}
RootCommandPrefixedFlags: cli.FlagBinders{
func(cmd *cobra.Command, c *cli.Config) {
cmp.RootCommandPrefixedFlags.Bind(cmd, cmp)
msg.RootCommandPrefixedFlags.Bind(cmd, msg)
sys.RootCommandPrefixedFlags.Bind(cmd, sys)
},
},
func init() {
logger.Init(zap.DebugLevel)
}
ApiServerAdtFlags: cli.CombineFlagBinders(
cmp.ApiServerAdtFlags,
msg.ApiServerAdtFlags,
sys.ApiServerAdtFlags,
),
func InitMonolith() *Monolith {
return &Monolith{
log: logger.Default(),
compose: compose.InitCompose(),
messaging: messaging.InitMessaging(),
system: system.InitSystem(),
}
}
ApiServerRoutes: cli.Mounters{
func(r chi.Router) {
r.Route("/compose", cmp.ApiServerRoutes.MountRoutes)
r.Route("/messaging", msg.ApiServerRoutes.MountRoutes)
r.Route("/system", sys.ApiServerRoutes.MountRoutes)
},
},
// Command produces cobra.Command
func (m *Monolith) Command(ctx context.Context) (cmd *cobra.Command) {
cmd = &cobra.Command{
Use: "corteza-server-monolith",
TraverseChildren: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
cli.InitGeneralServices(m.logOpt, m.smtpOpt, m.jwtOpt, m.httpClientOpt)
AdtSubCommands: cli.CommandMakers{
func(ctx context.Context, c *cli.Config) *cobra.Command {
if cc := cmp.AdtSubCommands; len(cc) > 0 {
sub := &cobra.Command{Use: "compose", Short: "Commands from compose service"}
sub.AddCommand(cc.Make(ctx, c)...)
return sub
}
return m.StartServices(ctx)
return nil
},
func(ctx context.Context, c *cli.Config) *cobra.Command {
if cc := msg.AdtSubCommands; len(cc) > 0 {
sub := &cobra.Command{Use: "messaging", Short: "Commands from messaging service"}
sub.AddCommand(cc.Make(ctx, c)...)
return sub
}
return nil
},
func(ctx context.Context, c *cli.Config) *cobra.Command {
if cc := sys.AdtSubCommands; len(cc) > 0 {
sub := &cobra.Command{Use: "system", Short: "Commands from system service"}
sub.AddCommand(cc.Make(ctx, c)...)
return sub
}
return nil
},
},
ProvisionMigrateDatabase: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) (err error) {
cli.HandleError(cmp.ProvisionMigrateDatabase.Run(ctx, cmd, cmp))
cli.HandleError(msg.ProvisionMigrateDatabase.Run(ctx, cmd, msg))
cli.HandleError(sys.ProvisionMigrateDatabase.Run(ctx, cmd, sys))
return
},
},
ProvisionAccessControl: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) (err error) {
cli.HandleError(cmp.ProvisionAccessControl.Run(ctx, cmd, cmp))
cli.HandleError(msg.ProvisionAccessControl.Run(ctx, cmd, msg))
cli.HandleError(sys.ProvisionAccessControl.Run(ctx, cmd, sys))
return
},
},
}
m.BindGlobalFlags(cmd)
srv := api.NewServer(m.log)
serveApiCmd := srv.Command(ctx, "", m.ApiServerPreRun)
// Bind all flags we need for serving monolith
m.BindApiServerFlags(serveApiCmd)
srv.MountRoutes(m.ApiServerRoutes)
cmd.AddCommand(
serveApiCmd,
cli.SetupProvisionSubcommands(ctx, m),
)
m.AddCommands(cmd, ctx)
return
}
// AddCommands - other commands that this subservices need
//
// We wrap each seubservice's set into a subcommand so that we do not get naming collisions
func (m *Monolith) AddCommands(cmd *cobra.Command, ctx context.Context) {
var (
composeCmd = &cobra.Command{Use: "compose"}
messagingCmd = &cobra.Command{Use: "messaging"}
systemCmd = &cobra.Command{Use: "system"}
)
m.compose.AddCommands(composeCmd, ctx)
if len(composeCmd.Commands()) > 0 {
cmd.AddCommand(composeCmd)
}
m.messaging.AddCommands(messagingCmd, ctx)
if len(messagingCmd.Commands()) > 0 {
cmd.AddCommand(messagingCmd)
}
m.system.AddCommands(systemCmd, ctx)
if len(systemCmd.Commands()) > 0 {
cmd.AddCommand(systemCmd)
}
}
// Binds all global flags
func (m *Monolith) BindGlobalFlags(cmd *cobra.Command) {
m.logOpt = flags.Log(cmd)
m.smtpOpt = flags.SMTP(cmd)
m.jwtOpt = flags.JWT(cmd)
m.httpClientOpt = flags.HttpClient(cmd)
}
// BindApiServerFlags sets & binds all API server flags
func (m *Monolith) BindApiServerFlags(cmd *cobra.Command) {
m.compose.BindApiServerFlags(cmd)
m.messaging.BindApiServerFlags(cmd)
m.system.BindApiServerFlags(cmd)
}
func (m *Monolith) StartServices(ctx context.Context) (err error) {
return (runners{
m.compose.StartServices,
m.messaging.StartServices,
m.system.StartServices,
}).run(ctx)
}
// ApiServerPreRun is executed before serve-api command runs REST API server
//
// Should initialize all that needs to run in the background
func (m Monolith) ApiServerPreRun(ctx context.Context) error {
return (runners{
m.compose.ApiServerPreRun,
m.messaging.ApiServerPreRun,
m.system.ApiServerPreRun,
}).run(ctx)
}
// ApiServerRoutes mounts api server routes
func (m *Monolith) ApiServerRoutes(r chi.Router) {
r.Route("/compose", m.compose.ApiServerRoutes)
r.Route("/messaging", m.messaging.ApiServerRoutes)
r.Route("/system", m.system.ApiServerRoutes)
}
// ProvisionMigrateDatabase migrates database to new version
//
// This is ran by default on serve-api (when not explicitly disabled with --compose-provision-database=false)
// or on demand with "provision migrate-database"
func (m Monolith) ProvisionMigrateDatabase(ctx context.Context) error {
return (runners{
m.compose.ProvisionMigrateDatabase,
m.messaging.ProvisionMigrateDatabase,
m.system.ProvisionMigrateDatabase,
}).run(ctx)
}
// ProvisionAccessControl resets access-control rules for roles admin (2) and everyone (1)
//
// Run with emand with "provision access-control-rules"
func (m Monolith) ProvisionAccessControl(ctx context.Context) error {
return (runners{
m.compose.ProvisionAccessControl,
m.messaging.ProvisionAccessControl,
m.system.ProvisionAccessControl,
}).run(ctx)
}
monolith/monolith_test.go
+15
View File
@@ -0,0 +1,15 @@
package monolith
import (
"context"
"testing"
"github.com/cortezaproject/corteza-server/internal/test"
)
func TestConfigure(t *testing.T) {
var config = Configure()
test.Assert(t, config != nil, "Configure valid")
test.Assert(t, func() bool { config.Init(); return true }(), "Initialization ok")
test.Assert(t, config.MakeCLI(context.Background()) != nil, "CLI created")
}
pkg/api/server.go
+24 -13
View File
@@ -2,12 +2,11 @@ package api
import (
"context"
"fmt"
"net"
"net/http"
"time"
"github.com/go-chi/chi"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/titpetric/factory/resputil"
"go.uber.org/zap"
@@ -37,23 +36,23 @@ func NewServer(log *zap.Logger) *Server {
}
}
func (s *Server) Command(ctx context.Context, prefix string, preRun func(context.Context) error) (cmd *cobra.Command) {
func (s *Server) Command(ctx context.Context, cmdName, prefix string, preRun func(context.Context) error) (cmd *cobra.Command) {
cmd = &cobra.Command{
Use: "serve-api",
Use: cmdName,
Short: "Start HTTP Server with REST API",
// Connect all the wires, prepare services, run watchers, bind endpoints
PreRun: func(cmd *cobra.Command, args []string) {
PreRunE: func(cmd *cobra.Command, args []string) error {
if s.monitorOpt.Interval > 0 {
go NewMonitor(s.monitorOpt.Interval)
go NewMonitor(int(s.monitorOpt.Interval / time.Second))
}
preRun(ctx)
return preRun(ctx)
},
// Run the server
RunE: func(cmd *cobra.Command, args []string) error {
return s.Serve(ctx)
Run: func(cmd *cobra.Command, args []string) {
s.Serve(ctx)
},
}
@@ -70,7 +69,7 @@ func (s *Server) MountRoutes(mm ...func(chi.Router)) {
s.endpoints = append(s.endpoints, mm...)
}
func (s Server) Serve(ctx context.Context) error {
func (s Server) Serve(ctx context.Context) {
s.log.Info("Starting HTTP server with REST API", zap.String("address", s.httpOpt.Addr))
// configure resputil options
@@ -84,7 +83,8 @@ func (s Server) Serve(ctx context.Context) error {
listener, err := net.Listen("tcp", s.httpOpt.Addr)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Can't listen on addr %s", s.httpOpt.Addr))
s.log.Error("Can not start server", zap.Error(err))
return
}
router := chi.NewRouter()
@@ -122,8 +122,19 @@ func (s Server) Serve(ctx context.Context) error {
router.Get("/version", version.HttpHandler)
}
go http.Serve(listener, router)
go func() {
err = http.Serve(listener, router)
}()
<-ctx.Done()
return nil
if err == nil {
err = ctx.Err()
if err == context.Canceled {
err = nil
}
}
s.log.Info("HTTP server stopped", zap.Error(err))
return
}
pkg/cli/flags/db.go
+2 -2
View File
@@ -14,11 +14,11 @@ type (
func DB(cmd *cobra.Command, pfix string) (o *DBOpt) {
o = &DBOpt{}
bindString(cmd, &o.DSN,
BindString(cmd, &o.DSN,
pFlag(pfix, "db-dsn"), "corteza:corteza@tcp(db:3306)/corteza?collation=utf8mb4_general_ci",
"DSN for database connection")
bindString(cmd, &o.Profiler,
BindString(cmd, &o.Profiler,
pFlag(pfix, "db-profiler"), "none",
"Profiler for DB queries (none, stdout, logger)")
pkg/cli/flags/flags.go
+4 -4
View File
@@ -23,7 +23,7 @@ func envKey(s string) string {
return strings.ToUpper(strings.ReplaceAll(s, "-", "_"))
}
func bindString(cmd *cobra.Command, v *string, flag, def string, desc string) {
func BindString(cmd *cobra.Command, v *string, flag, def string, desc string) {
if env, has := os.LookupEnv(envKey(flag)); has {
def = cast.ToString(env)
}
@@ -31,7 +31,7 @@ func bindString(cmd *cobra.Command, v *string, flag, def string, desc string) {
cmd.Flags().StringVar(v, flag, def, desc)
}
func bindBool(cmd *cobra.Command, v *bool, flag string, def bool, desc string) {
func BindBool(cmd *cobra.Command, v *bool, flag string, def bool, desc string) {
if env, has := os.LookupEnv(envKey(flag)); has {
def = cast.ToBool(env)
}
@@ -39,7 +39,7 @@ func bindBool(cmd *cobra.Command, v *bool, flag string, def bool, desc string) {
cmd.Flags().BoolVar(v, flag, def, desc)
}
func bindInt(cmd *cobra.Command, v *int, flag string, def int, desc string) {
func BindInt(cmd *cobra.Command, v *int, flag string, def int, desc string) {
if env, has := os.LookupEnv(envKey(flag)); has {
def = cast.ToInt(env)
}
@@ -47,7 +47,7 @@ func bindInt(cmd *cobra.Command, v *int, flag string, def int, desc string) {
cmd.Flags().IntVar(v, flag, def, desc)
}
func bindDuration(cmd *cobra.Command, v *time.Duration, flag string, def time.Duration, desc string) {
func BindDuration(cmd *cobra.Command, v *time.Duration, flag string, def time.Duration, desc string) {
if env, has := os.LookupEnv(envKey(flag)); has {
def = cast.ToDuration(env)
}
pkg/cli/flags/http.go
+10 -10
View File
@@ -26,44 +26,44 @@ type (
func HTTP(cmd *cobra.Command, pfix string) (o *HTTPOpt) {
o = &HTTPOpt{}
bindString(cmd, &o.Addr,
BindString(cmd, &o.Addr,
pFlag(pfix, "http-addr"), ":80",
"Listen address for HTTP server")
bindBool(cmd, &o.Logging,
BindBool(cmd, &o.Logging,
pFlag(pfix, "http-log"), true,
"Enable/disable HTTP request log")
bindBool(cmd, &o.Pretty,
BindBool(cmd, &o.Pretty,
pFlag(pfix, "http-pretty-json"), false,
"Prettify returned JSON output")
bindBool(cmd, &o.Tracing,
BindBool(cmd, &o.Tracing,
pFlag(pfix, "http-error-tracing"), false,
"Return error stack frame")
bindBool(cmd, &o.EnableVersionRoute,
BindBool(cmd, &o.EnableVersionRoute,
pFlag(pfix, "http-enable-version-route"), true,
"Enable /version route")
bindBool(cmd, &o.EnableDebugRoute,
BindBool(cmd, &o.EnableDebugRoute,
pFlag(pfix, "http-enable-debug-route"), false,
"Enable /debug route with pprof data")
bindBool(cmd, &o.EnableMetrics,
BindBool(cmd, &o.EnableMetrics,
pFlag(pfix, "http-metrics"), false,
"Enable metrics")
bindString(cmd, &o.MetricsServiceLabel,
BindString(cmd, &o.MetricsServiceLabel,
pFlag(pfix, "http-metrics-name"), "corteza",
"Provide metrics service label for Prometheus")
bindString(cmd, &o.MetricsUsername,
BindString(cmd, &o.MetricsUsername,
pFlag(pfix, "http-metrics-username"), "metrics",
"Provide metrics username for Prometheus")
// Setting metrics password to random string to prevent security accidents...
bindString(cmd, &o.MetricsPassword,
BindString(cmd, &o.MetricsPassword,
pFlag(pfix, "http-metrics-password"), string(rand.Bytes(5)),
"Provide metrics password for Prometheus")
pkg/cli/flags/http_client.go
+2 -2
View File
@@ -16,11 +16,11 @@ type (
func HttpClient(cmd *cobra.Command) (o *HttpClientOpt) {
o = &HttpClientOpt{}
bindBool(cmd, &o.ClientTSLInsecure,
BindBool(cmd, &o.ClientTSLInsecure,
"http-client-tsl-insecure", false,
"Skip insecure TSL verification on outbound HTTP requests (allow invalid/self-signed certificates")
bindDuration(cmd, &o.HttpClientTimeout,
BindDuration(cmd, &o.HttpClientTimeout,
"http-client-timeout", 30*time.Second,
"Default HTTP client timeout")
pkg/cli/flags/jwt.go
+2 -2
View File
@@ -17,11 +17,11 @@ func JWT(cmd *cobra.Command) (o *JWTOpt) {
o = &JWTOpt{}
// Setting JWT secret to random string to prevent security accidents...
bindString(cmd, &o.Secret,
BindString(cmd, &o.Secret,
"auth-jwt-secret", string(rand.Bytes(32)),
"JWT Secret")
bindInt(cmd, &o.Expiry,
BindInt(cmd, &o.Expiry,
"auth-jwt-expiry", 60*24*30,
"JWT Expiration in minutes")
pkg/cli/flags/log.go
+3 -6
View File
@@ -5,22 +5,19 @@ import (
)
type (
// Logger's output leve is configured here, but
// dev/prod configuration happens earlier
LogOpt struct {
Level string
JSON bool
}
)
func Log(cmd *cobra.Command) (o *LogOpt) {
o = &LogOpt{}
bindString(cmd, &o.Level,
BindString(cmd, &o.Level,
"log-level", "info",
"Log level (debug, info, warn, error, panic, fatal)")
bindBool(cmd, &o.JSON,
"log-json", true,
"Log in JSON format")
return
}
pkg/cli/flags/monitor.go
+6 -4
View File
@@ -1,21 +1,23 @@
package flags
import (
"time"
"github.com/spf13/cobra"
)
type (
MonitorOpt struct {
Interval int
Interval time.Duration
}
)
func Monitor(cmd *cobra.Command, pfix string) (o *MonitorOpt) {
o = &MonitorOpt{}
bindInt(cmd, &o.Interval,
pFlag(pfix, "monitor-interval"), 300,
"Monitor interval (seconds, 0 = disable)")
BindDuration(cmd, &o.Interval,
pFlag(pfix, "monitor-interval"), 300*time.Second,
"Monitor interval (0 = disable)")
return
}
pkg/cli/flags/provision.go
+10 -4
View File
@@ -6,16 +6,22 @@ import (
type (
ProvisionOpt struct {
Database bool
MigrateDatabase bool
AutoSetup bool
}
)
func Provision(cmd *cobra.Command, pfix string) (o *ProvisionOpt) {
o = &ProvisionOpt{}
bindBool(cmd, &o.Database,
pFlag(pfix, "provision-database"), true,
"Run database migration scripts")
BindBool(cmd, &o.MigrateDatabase,
pFlag(pfix, "provision-migrate-database"), true,
"Run database migration")
BindBool(cmd, &o.AutoSetup,
pFlag(pfix, "provision-auto-setup"), true,
"Run auto-setup procedures on service")
return
}
pkg/cli/flags/pubsub.go
+6 -6
View File
@@ -30,27 +30,27 @@ func PubSub(cmd *cobra.Command, pfix string) (o *PubSubOpt) {
pingPeriod = (pingTimeout * 9) / 10
)
bindString(cmd, &o.Mode,
BindString(cmd, &o.Mode,
pFlag(pfix, "pubsub-mode"), "poll",
"Pub/Sub mode (poll, redis")
bindDuration(cmd, &o.RedisPingTimeout,
BindDuration(cmd, &o.RedisPingTimeout,
pFlag(pfix, "pubsub-polling-interval"), timeout,
"Sub/Sub polling interval")
bindString(cmd, &o.RedisAddr,
BindString(cmd, &o.RedisAddr,
pFlag(pfix, "pubsub-redis-addr"), "redis:6379",
"Pub/Sub mode (poll, redis")
bindDuration(cmd, &o.RedisTimeout,
BindDuration(cmd, &o.RedisTimeout,
pFlag(pfix, "pubsub-redis-timeout"), timeout,
"Websocket connection timeout")
bindDuration(cmd, &o.RedisPingTimeout,
BindDuration(cmd, &o.RedisPingTimeout,
pFlag(pfix, "pubsub-redis-ping-timeout"), pingTimeout,
"Pub/Sub connection ping timeout")
bindDuration(cmd, &o.RedisPingPeriod,
BindDuration(cmd, &o.RedisPingPeriod,
pFlag(pfix, "pubsub-redis-ping-period"), pingPeriod,
"Pub/Sub connection ping period (should be lower than timeout)")
pkg/cli/flags/smtp.go
+5 -5
View File
@@ -17,23 +17,23 @@ type (
func SMTP(cmd *cobra.Command) (o *SMTPOpt) {
o = &SMTPOpt{}
bindString(cmd, &o.Host,
BindString(cmd, &o.Host,
"smtp-host", "localhost:25",
"SMTP hostname")
bindString(cmd, &o.User,
BindString(cmd, &o.User,
"smtp-username", "",
"SMTP server username")
bindString(cmd, &o.Pass,
BindString(cmd, &o.Pass,
"smtp-pass", "",
"SMTP server password")
bindString(cmd, &o.From,
BindString(cmd, &o.From,
"smtp-from", "",
"Sender's email address")
bindInt(cmd, &o.Port,
BindInt(cmd, &o.Port,
"smtp-port", 25,
"SMTP port number")
pkg/cli/flags/websocket.go
+3 -3
View File
@@ -23,15 +23,15 @@ func Websocket(cmd *cobra.Command, pfix string) (o *WebsocketOpt) {
pingPeriod = (pingTimeout * 9) / 10
)
bindDuration(cmd, &o.Timeout,
BindDuration(cmd, &o.Timeout,
pFlag(pfix, "websocket-timeout"), timeout,
"Websocket connection timeout")
bindDuration(cmd, &o.PingTimeout,
BindDuration(cmd, &o.PingTimeout,
pFlag(pfix, "websocket-ping-timeout"), pingTimeout,
"Websocket connection ping timeout")
bindDuration(cmd, &o.PingPeriod,
BindDuration(cmd, &o.PingPeriod,
pFlag(pfix, "websocket-ping-period"), pingPeriod,
"Websocket connection ping period (should be lower than timeout)")
pkg/cli/helpers.go
+13 -83
View File
@@ -1,9 +1,9 @@
package cli
import (
"context"
"fmt"
"os"
"github.com/spf13/cobra"
"go.uber.org/zap"
"github.com/cortezaproject/corteza-server/internal/auth"
@@ -13,90 +13,11 @@ import (
"github.com/cortezaproject/corteza-server/pkg/logger"
)
// SetupProvisionCommands sets-up standard provision commands
// Deprecated: use SetupProvisionSubCommands
func SetupProvisionCommands(ac func() error, md func() error) *cobra.Command {
var (
cmd = &cobra.Command{
Use: "provision",
Short: "Provision tasks",
}
)
// Add only commands with defined callbacks
if ac != nil {
cmd.AddCommand(&cobra.Command{
Use: "access-control-rules",
Short: "Reset access control rules & roles",
RunE: func(cmd *cobra.Command, args []string) error {
return ac()
},
})
}
// Add only commands with defined callbacks
if md != nil {
cmd.AddCommand(&cobra.Command{
Use: "migrate-database",
Short: "Run database migration scripts",
RunE: func(cmd *cobra.Command, args []string) error {
return md()
},
})
}
return cmd
}
type (
provisioner interface {
ProvisionMigrateDatabase(ctx context.Context) error
ProvisionAccessControl(ctx context.Context) error
}
)
func SetupProvisionSubcommands(ctx context.Context, p provisioner) *cobra.Command {
var (
cmd = &cobra.Command{
Use: "provision",
Short: "Provision tasks",
}
)
// Add only commands with defined callbacks
cmd.AddCommand(&cobra.Command{
Use: "access-control-rules",
Short: "Reset access control rules & roles",
RunE: func(cmd *cobra.Command, args []string) error {
return p.ProvisionAccessControl(ctx)
},
})
// Add only commands with defined callbacks
cmd.AddCommand(&cobra.Command{
Use: "migrate-database",
Short: "Run database migration scripts",
RunE: func(cmd *cobra.Command, args []string) error {
return p.ProvisionMigrateDatabase(ctx)
},
})
return cmd
}
func InitGeneralServices(logOpt *flags.LogOpt, smtpOpt *flags.SMTPOpt, jwtOpt *flags.JWTOpt, httpClientOpt *flags.HttpClientOpt) {
// Reset logger's level to whatever we want
var logLevel = zap.InfoLevel
_ = logLevel.Set(logOpt.Level)
if logger.Default() == nil {
logger.Init(logLevel)
} else {
logger.DefaultLevel.SetLevel(logLevel)
}
logger.DefaultLevel.SetLevel(logLevel)
auth.SetupDefault(jwtOpt.Secret, jwtOpt.Expiry)
mail.SetupDialer(smtpOpt.Host, smtpOpt.Port, smtpOpt.User, smtpOpt.Pass, smtpOpt.From)
@@ -105,3 +26,12 @@ func InitGeneralServices(logOpt *flags.LogOpt, smtpOpt *flags.SMTPOpt, jwtOpt *f
httpClientOpt.ClientTSLInsecure,
)
}
func HandleError(err error) {
if err == nil {
return
}
_, _ = fmt.Fprintln(os.Stderr, err.Error())
os.Exit(1)
}
pkg/cli/runner.go
+315
View File
@@ -0,0 +1,315 @@
package cli
import (
"context"
"github.com/go-chi/chi"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"go.uber.org/zap"
"github.com/cortezaproject/corteza-server/internal/db"
"github.com/cortezaproject/corteza-server/pkg/api"
"github.com/cortezaproject/corteza-server/pkg/cli/flags"
"github.com/cortezaproject/corteza-server/pkg/logger"
)
type (
Runner func(ctx context.Context, cmd *cobra.Command, c *Config) error
Runners []Runner
CommandMaker func(ctx context.Context, c *Config) *cobra.Command
CommandMakers []CommandMaker
FlagBinder func(cmd *cobra.Command, c *Config)
FlagBinders []FlagBinder
Mounter func(r chi.Router)
Mounters []Mounter
Config struct {
init bool
// Service name (messaging, system...)
// See comments on other fields for how it is used.
ServiceName string
// Logger name for internal services, defaults to ServiceName
LoggerName string
Log *zap.Logger
// General options/flags
LogOpt *flags.LogOpt
SmtpOpt *flags.SMTPOpt
JwtOpt *flags.JWTOpt
HttpClientOpt *flags.HttpClientOpt
// Per-service options/flags
DbOpt *flags.DBOpt
ProvisionOpt *flags.ProvisionOpt
// DB Connection name, defaults to ServiceName
DatabaseName string
// Root command name, , defaults to "corteza-server-<ServiceName>"
RootCommandName string
// Flags that are bond to root command, no (per-service) prefixed
RootCommandBaseFlags FlagBinders
// Prefix for flags for root command, defaults to ServiceName
RootCommandFlagsPrefix string
// Flags that are bond to root command, (per-service) prefixed
RootCommandPrefixedFlags FlagBinders
// Database setup/connection procedure
// Runner autobinds default runner that tries to connect using DbOpt.DSN
RootCommandDBSetup Runners
// All that needs to be initialized before any sub-comman is executed
RootCommandPreRun Runners
// ******************************************************************
// API Server instance
ApiServer *api.Server
// API Server command name
ApiServerCommandName string
// Prefix for "serve-api command flags", defaults to ServiceName
ApiServerFlagsPrefix string
// Additional command flags for API server
ApiServerAdtFlags FlagBinders
// Code that needs to be executed before HTTP server is started
ApiServerPreRun Runners
// Routers that we mount on HTTP server
ApiServerRoutes Mounters
// Sets-up all available subcommands.
AdtSubCommands CommandMakers
// Database migration code
// This is used for "provision migrate-database" command and after db connection is
// established (if --provision-migrate-database is enabled
ProvisionMigrateDatabase Runners
// Access control initial setup
// Reapplies default access control rules for roles "everyone" [1] and "admin" [2]
ProvisionAccessControl Runners
}
)
func init() {
// Have logger ready in case we need to log anything
// before it gets properly initialized through InitGeneralServices
logger.Init()
}
func (rr Runners) Run(ctx context.Context, cmd *cobra.Command, c *Config) (err error) {
for i := range rr {
err = rr[i](ctx, cmd, c)
if err != nil {
return
}
}
return
}
func (rr Mounters) MountRoutes(r chi.Router) {
for i := range rr {
rr[i](r)
}
}
func (bb FlagBinders) Bind(cmd *cobra.Command, c *Config) {
for i := range bb {
bb[i](cmd, c)
}
}
func (mm CommandMakers) Make(ctx context.Context, c *Config) []*cobra.Command {
var (
valid = make([]*cobra.Command, 0)
cmd *cobra.Command
)
for i := range mm {
if cmd = mm[i](ctx, c); cmd != nil {
valid = append(valid, cmd)
}
}
return valid
}
func CombineFlagBinders(rr ...FlagBinders) (out FlagBinders) {
for i := range rr {
out = append(out, rr[i]...)
}
return out
}
func (c *Config) Init() {
if c.init {
return
}
if c.Log == nil {
c.Log = logger.Default()
}
if c.LoggerName == "" {
c.LoggerName = c.ServiceName
}
c.Log = c.Log.Named(c.LoggerName)
if c.RootCommandName == "" {
c.RootCommandName = "corteza-server-" + c.ServiceName
}
if c.ApiServerCommandName == "" {
c.ApiServerCommandName = "serve-api"
}
if c.ApiServerFlagsPrefix == "" {
c.ApiServerFlagsPrefix = c.ServiceName
}
if c.DatabaseName == "" {
c.DatabaseName = c.ServiceName
}
if c.RootCommandDBSetup == nil {
c.RootCommandDBSetup = Runners{func(ctx context.Context, cmd *cobra.Command, c *Config) (err error) {
_, err = db.TryToConnect(ctx, c.Log, c.DatabaseName, c.DbOpt.DSN, c.DbOpt.Profiler)
if err != nil {
return errors.Wrap(err, "could not connect to database")
}
return
}}
}
// Flags, not prefixed with service name
if c.RootCommandBaseFlags == nil {
c.RootCommandBaseFlags = FlagBinders{
func(cmd *cobra.Command, c *Config) {
c.LogOpt = flags.Log(cmd)
c.SmtpOpt = flags.SMTP(cmd)
c.JwtOpt = flags.JWT(cmd)
c.HttpClientOpt = flags.HttpClient(cmd)
},
}
}
if c.RootCommandFlagsPrefix == "" {
c.RootCommandFlagsPrefix = c.ServiceName
}
// Flags, prefixed with service name
if c.RootCommandPrefixedFlags == nil {
c.RootCommandPrefixedFlags = FlagBinders{
func(cmd *cobra.Command, c *Config) {
c.DbOpt = flags.DB(cmd, c.ServiceName)
c.ProvisionOpt = flags.Provision(cmd, c.ServiceName)
},
}
}
if c.ApiServer == nil {
c.ApiServer = api.NewServer(c.Log)
}
for i := range c.ApiServerRoutes {
c.ApiServer.MountRoutes(c.ApiServerRoutes[i])
}
}
// MakeCLI creates command line interface
//
// It tries to construct "serve-api" and "provision" sub-commands
// if configured properly (see Config struct)
func (c *Config) MakeCLI(ctx context.Context) (cmd *cobra.Command) {
c.Init()
cmd = &cobra.Command{
Use: c.RootCommandName,
TraverseChildren: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
InitGeneralServices(c.LogOpt, c.SmtpOpt, c.JwtOpt, c.HttpClientOpt)
err = c.RootCommandDBSetup.Run(ctx, cmd, c)
if err != nil {
c.Log.Error("Failed to connect to the database", zap.Error(err))
return nil
}
err = c.RootCommandPreRun.Run(ctx, cmd, c)
if err != nil {
c.Log.Error("Failed to run command pre-run scripts", zap.Error(err))
return nil
}
return nil
},
}
c.RootCommandBaseFlags.Bind(cmd, c)
c.RootCommandPrefixedFlags.Bind(cmd, c)
serveApiCmd := c.ApiServer.Command(ctx, c.ApiServerCommandName, c.ApiServerFlagsPrefix, func(ctx context.Context) (err error) {
return c.ApiServerPreRun.Run(ctx, cmd, c)
})
// Bind all flags we need for serving the API
c.ApiServerAdtFlags.Bind(serveApiCmd, c)
cmd.AddCommand(serveApiCmd)
if len(c.ProvisionMigrateDatabase) > 0 || len(c.ProvisionAccessControl) > 0 {
var (
provisionCmd = &cobra.Command{
Use: "provision",
Short: "Provision tasks",
}
)
// Add only commands with defined callbacks
if len(c.ProvisionMigrateDatabase) > 0 {
provisionCmd.AddCommand(&cobra.Command{
Use: "access-control-rules",
Short: "Reset access control rules & roles",
RunE: func(cmd *cobra.Command, args []string) error {
return c.ProvisionAccessControl.Run(ctx, nil, c)
},
})
}
// Add only commands with defined callbacks
if len(c.ProvisionAccessControl) > 0 {
provisionCmd.AddCommand(&cobra.Command{
Use: "migrate-database",
Short: "Run database migration scripts",
RunE: func(cmd *cobra.Command, args []string) error {
return c.ProvisionMigrateDatabase.Run(ctx, nil, c)
},
})
}
cmd.AddCommand(provisionCmd)
}
cmd.AddCommand(c.AdtSubCommands.Make(ctx, c)...)
return
}
pkg/logger/logger.go
+45 -22
View File
@@ -1,35 +1,54 @@
package logger
import (
"os"
"strings"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
var (
DefaultLevel = zap.NewAtomicLevel()
defaultLogger *zap.Logger
defaultLogger = zap.NewNop()
)
func Init(level zapcore.Level) {
DefaultLevel.SetLevel(level)
func MakeDebugLogger() *zap.Logger {
conf := zap.NewDevelopmentConfig()
conf.Level = DefaultLevel
logger, err := conf.Build()
if err != nil {
panic(err)
}
return logger
}
func Init() {
var (
err error
conf = zap.Config{
Level: DefaultLevel,
Development: false,
Encoding: "json",
EncoderConfig: zap.NewProductionEncoderConfig(),
OutputPaths: []string{"stderr"},
ErrorOutputPaths: []string{"stderr"},
}
err error
isProduction bool
)
// Make logstash's life a bit easier
conf.EncoderConfig.LevelKey = "appLogLevel"
conf.EncoderConfig.MessageKey = "message"
conf.EncoderConfig.TimeKey = "@timestamp"
conf.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
if env := os.Getenv("ENVIRONMENT"); strings.Index(env, "prod") == 0 {
// Try to guess if production logging from environment
isProduction = true
} else if vh := os.Getenv("VIRTUAL_HOST"); len(vh) > 0 {
// Try to guess if in production logging from the fact that VIRTUAL_HOST env is set
isProduction = true
}
if !isProduction {
defaultLogger = MakeDebugLogger()
return
}
conf := zap.NewProductionConfig()
conf.Level = DefaultLevel
// We do not want sampling
conf.Sampling = nil
conf.Level = zap.NewAtomicLevelAt(zap.InfoLevel)
defaultLogger, err = conf.Build()
if err != nil {
@@ -38,9 +57,13 @@ func Init(level zapcore.Level) {
}
func Default() *zap.Logger {
if defaultLogger == nil {
panic("default logger not initialised")
}
return defaultLogger
}
func SetDefault(logger *zap.Logger) {
if logger == nil {
logger = zap.NewNop()
}
defaultLogger = logger
}
system/commands/commands.go
+1 -1
View File
@@ -6,6 +6,6 @@ import (
)
func exit(err error) {
fmt.Fprintln(os.Stderr, err.Error())
_, _ = fmt.Fprintln(os.Stderr, err.Error())
os.Exit(1)
}
system/commands/settings.go
+2 -2
View File
@@ -29,7 +29,7 @@ func Settings(ctx context.Context) *cobra.Command {
Run: func(cmd *cobra.Command, args []string) {
_, _ = service.DefaultSettings.LoadAuthSettings()
settingsAutoConfigure(
SettingsAutoConfigure(
cmd,
systemApiUrl,
authFrontendUrl,
@@ -137,7 +137,7 @@ func Settings(ctx context.Context) *cobra.Command {
return cmd
}
func settingsAutoConfigure(cmd *cobra.Command, systemApiUrl, frontendUrl, fromAddress, fromName string) {
func SettingsAutoConfigure(cmd *cobra.Command, systemApiUrl, frontendUrl, fromAddress, fromName string) {
set := func(name string, value interface{}) {
var (
v *settings.Value
system/db/migrate.go
+12 -9
View File
@@ -2,7 +2,6 @@ package db
import (
"fmt"
"log"
"os"
"path/filepath"
"regexp"
@@ -12,6 +11,7 @@ import (
"github.com/goware/statik/fs"
"github.com/pkg/errors"
"github.com/titpetric/factory"
"go.uber.org/zap"
"github.com/cortezaproject/corteza-server/system/db/mysql"
)
@@ -23,10 +23,12 @@ func statements(contents []byte, err error) ([]string, error) {
return regexp.MustCompilePOSIX(";$").Split(string(contents), -1), nil
}
func Migrate(db *factory.DB) error {
func Migrate(db *factory.DB, log *zap.Logger) error {
log = log.Named("database.migrations")
statikFS, err := fs.New(mysql.Asset)
if err != nil {
return errors.Wrap(err, "Error creating statik filesystem")
return errors.Wrap(err, "error creating statik filesystem")
}
var files []string
@@ -42,13 +44,13 @@ func Migrate(db *factory.DB) error {
}
if err := fs.Walk(statikFS, "/", fn); err != nil {
return errors.Wrap(err, "Error when listing files for migrations")
return errors.Wrap(err, "error when listing files for migrations")
}
sort.Strings(files)
if len(files) == 0 {
return errors.New("No files encoded for migration, need at least one SQL file")
return errors.New("no files encoded for migration, need at least one SQL file")
}
migrate := func(filename string, useLog bool) error {
@@ -68,19 +70,20 @@ func Migrate(db *factory.DB) error {
up := func() error {
stmts, err := statements(fs.ReadFile(statikFS, filename))
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Error reading migration %s", filename))
return errors.Wrap(err, fmt.Sprintf("error reading migration %s", filename))
}
log.Println("Running migration for", filename)
log.Debug("Running migration", zap.String("filename", filename))
for idx, query := range stmts {
if strings.TrimSpace(query) != "" && idx >= status.StatementIndex {
status.StatementIndex = idx
if _, err := db.Exec(query); err != nil {
log.Debug("migration error ", zap.String("filename", filename), zap.Error(err))
return err
}
}
}
log.Println("Migration done OK")
status.Status = "ok"
return nil
}
@@ -91,7 +94,7 @@ func Migrate(db *factory.DB) error {
}
if useLog {
if err := db.Replace("migrations", status); err != nil {
log.Println("replace failed", err)
return errors.Wrap(err, "migration update failed")
}
}
return err
system/db/migrate_test.go
+2 -1
View File
@@ -6,6 +6,7 @@ import (
"os"
"testing"
"github.com/cortezaproject/corteza-server/pkg/logger"
"github.com/titpetric/factory"
)
@@ -14,7 +15,7 @@ func TestMigrations(t *testing.T) {
db := factory.Database.MustGet("system")
db.Profiler = &factory.Database.ProfilerStdout
if err := Migrate(db); err != nil {
if err := Migrate(db, logger.Default()); err != nil {
t.Fatalf("Unexpected error: %#v", err)
}
}
system/internal/auth/external/main_test.go
Vendored
+1 -3
View File
@@ -4,12 +4,10 @@ import (
"os"
"testing"
"go.uber.org/zap/zapcore"
"github.com/cortezaproject/corteza-server/pkg/logger"
)
func TestMain(m *testing.M) {
logger.Init(zapcore.DebugLevel)
logger.SetDefault(logger.MakeDebugLogger())
os.Exit(m.Run())
}
system/internal/repository/main_test.go
+2 -1
View File
@@ -9,6 +9,7 @@ import (
"github.com/titpetric/factory"
"github.com/cortezaproject/corteza-server/pkg/logger"
systemMigrate "github.com/cortezaproject/corteza-server/system/db"
)
@@ -18,7 +19,7 @@ func TestMain(m *testing.M) {
db.Profiler = &factory.Database.ProfilerStdout
// migrate database schema
if err := systemMigrate.Migrate(db); err != nil {
if err := systemMigrate.Migrate(db, logger.Default()); err != nil {
fmt.Printf("Error running migrations: %+v\n", err)
os.Exit(1)
}
system/internal/service/main_test.go
+2 -3
View File
@@ -8,21 +8,20 @@ import (
"testing"
"github.com/titpetric/factory"
"go.uber.org/zap/zapcore"
"github.com/cortezaproject/corteza-server/pkg/logger"
systemMigrate "github.com/cortezaproject/corteza-server/system/db"
)
func TestMain(m *testing.M) {
logger.Init(zapcore.DebugLevel)
logger.SetDefault(logger.MakeDebugLogger())
factory.Database.Add("system", os.Getenv("SYSTEM_DB_DSN"))
db := factory.Database.MustGet("system")
db.Profiler = &factory.Database.ProfilerStdout
// migrate database schema
if err := systemMigrate.Migrate(db); err != nil {
if err := systemMigrate.Migrate(db, logger.Default()); err != nil {
fmt.Printf("Error running migrations: %+v\n", err)
return
}
system/internal/service/service.go
+3 -4
View File
@@ -7,7 +7,6 @@ import (
"github.com/cortezaproject/corteza-server/internal/permissions"
internalSettings "github.com/cortezaproject/corteza-server/internal/settings"
"github.com/cortezaproject/corteza-server/pkg/logger"
"github.com/cortezaproject/corteza-server/system/internal/repository"
)
@@ -40,10 +39,10 @@ var (
DefaultApplication ApplicationService
)
func Init(ctx context.Context) (err error) {
DefaultIntSettings = internalSettings.NewService(internalSettings.NewRepository(repository.DB(ctx), "sys_settings"))
func Init(ctx context.Context, log *zap.Logger) (err error) {
DefaultLogger = log.Named("service")
DefaultLogger = logger.Default().Named("system.service")
DefaultIntSettings = internalSettings.NewService(internalSettings.NewRepository(repository.DB(ctx), "sys_settings"))
DefaultPermissions = permissions.Service(
ctx,
system/rest/handlers/application.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
system/rest/handlers/auth.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
system/rest/handlers/auth_internal.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
system/rest/handlers/organisation.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
system/rest/handlers/permissions.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
system/rest/handlers/role.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
system/rest/handlers/settings.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
system/rest/handlers/user.go
+1
View File
@@ -17,6 +17,7 @@ package handlers
import (
"context"
"net/http"
"github.com/go-chi/chi"
system/service/init.go
-17
View File
@@ -1,17 +0,0 @@
package service
import (
"context"
"github.com/cortezaproject/corteza-server/system/internal/service"
)
func Init(ctx context.Context) error {
err := service.Init(ctx)
DefaultUser = service.DefaultUser
return err
}
func Watchers(ctx context.Context) {
service.Watchers(ctx)
}
system/service/user.go
-39
View File
@@ -1,39 +0,0 @@
package service
import (
"context"
"io"
"testing"
"github.com/cortezaproject/corteza-server/system/internal/service"
"github.com/cortezaproject/corteza-server/system/types"
)
type (
UserService interface {
Create(input *types.User) (*types.User, error)
Update(mod *types.User) (*types.User, error)
CreateWithAvatar(input *types.User, avatar io.Reader) (*types.User, error)
UpdateWithAvatar(mod *types.User, avatar io.Reader) (*types.User, error)
Delete(id uint64) error
FindByUsername(username string) (*types.User, error)
FindByEmail(email string) (*types.User, error)
FindByID(id uint64) (*types.User, error)
FindByIDs(id ...uint64) (types.UserSet, error)
Find(filter *types.UserFilter) (types.UserSet, error)
}
)
var DefaultUser = service.DefaultUser
func User(ctx context.Context) UserService {
return DefaultUser.With(ctx)
}
// Expose the full User API for testing
func TestUser(_ *testing.T, ctx context.Context) service.UserService {
return DefaultUser.With(ctx)
}
system/system.go
+76 -138
View File
@@ -3,18 +3,11 @@ package system
import (
"context"
"github.com/go-chi/chi"
_ "github.com/joho/godotenv/autoload"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/titpetric/factory"
"go.uber.org/zap"
"github.com/cortezaproject/corteza-server/internal/db"
"github.com/cortezaproject/corteza-server/pkg/api"
"github.com/cortezaproject/corteza-server/pkg/cli"
"github.com/cortezaproject/corteza-server/pkg/cli/flags"
"github.com/cortezaproject/corteza-server/pkg/logger"
"github.com/cortezaproject/corteza-server/system/commands"
migrate "github.com/cortezaproject/corteza-server/system/db"
"github.com/cortezaproject/corteza-server/system/internal/service"
@@ -25,143 +18,88 @@ const (
system = "system"
)
type (
System struct {
log *zap.Logger
func Configure() *cli.Config {
var (
accessControlSetup = func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
// Calling grant directly on internal permissions service to avoid AC check for "grant"
var p = service.DefaultPermissions
var ac = service.DefaultAccessControl
return p.Grant(ctx, ac.Whitelist(), ac.DefaultRules()...)
}
)
// General
logOpt *flags.LogOpt
smtpOpt *flags.SMTPOpt
jwtOpt *flags.JWTOpt
httpClientOpt *flags.HttpClientOpt
return &cli.Config{
ServiceName: system,
// System specific
dbOpt *flags.DBOpt
provisionOpt *flags.ProvisionOpt
}
)
RootCommandPreRun: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) (err error) {
if c.ProvisionOpt.MigrateDatabase {
cli.HandleError(c.ProvisionMigrateDatabase.Run(ctx, cmd, c))
}
func init() {
logger.Init(zap.DebugLevel)
}
cli.HandleError(service.Init(ctx, c.Log))
func InitSystem() *System {
return &System{
log: logger.Default().Named(system),
}
}
if c.ProvisionOpt.AutoSetup {
cli.HandleError(accessControlSetup(ctx, cmd, c))
// Command produces cobra.Command
func (m *System) Command(ctx context.Context) (cmd *cobra.Command) {
cmd = &cobra.Command{
Use: "corteza-server-system",
TraverseChildren: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
cli.InitGeneralServices(m.logOpt, m.smtpOpt, m.jwtOpt, m.httpClientOpt)
// Run auto configuration
commands.SettingsAutoConfigure(cmd, "", "", "", "")
return m.StartServices(ctx)
// Reload auto-configured settings
service.DefaultAuthSettings, _ = service.DefaultSettings.LoadAuthSettings()
}
return
},
},
ApiServerPreRun: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
go service.Watchers(ctx)
return nil
},
},
ApiServerRoutes: cli.Mounters{
rest.MountRoutes,
},
AdtSubCommands: cli.CommandMakers{
func(ctx context.Context, c *cli.Config) *cobra.Command {
return commands.Settings(ctx)
},
func(ctx context.Context, c *cli.Config) *cobra.Command {
return commands.Auth(ctx)
},
func(ctx context.Context, c *cli.Config) *cobra.Command {
return commands.Users(ctx)
},
func(ctx context.Context, c *cli.Config) *cobra.Command {
return commands.Roles(ctx)
},
},
ProvisionMigrateDatabase: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
if !c.ProvisionOpt.MigrateDatabase {
return nil
}
var db, err = factory.Database.Get(system)
if err != nil {
return err
}
db = db.With(ctx)
// Disable profiler for migrations
db.Profiler = nil
return migrate.Migrate(db, c.Log)
},
},
ProvisionAccessControl: cli.Runners{
accessControlSetup,
},
}
m.BindGlobalFlags(cmd)
srv := api.NewServer(m.log)
serveApiCmd := srv.Command(ctx, system, m.ApiServerPreRun)
// Bind all flags we need for serving system
m.BindApiServerFlags(serveApiCmd)
srv.MountRoutes(m.ApiServerRoutes)
cmd.AddCommand(
serveApiCmd,
cli.SetupProvisionSubcommands(ctx, m),
)
m.AddCommands(cmd, ctx)
return
}
// AddCommands - other commands that this subservice needs
func (m *System) AddCommands(cmd *cobra.Command, ctx context.Context) {
cmd.AddCommand(
commands.Settings(ctx),
commands.Auth(ctx),
commands.Users(ctx),
commands.Roles(ctx),
)
}
// Binds all global flags
func (m *System) BindGlobalFlags(cmd *cobra.Command) {
m.logOpt = flags.Log(cmd)
m.smtpOpt = flags.SMTP(cmd)
m.jwtOpt = flags.JWT(cmd)
m.httpClientOpt = flags.HttpClient(cmd)
}
// BindApiServerFlags sets & binds all API server flags
func (m *System) BindApiServerFlags(cmd *cobra.Command) {
m.dbOpt = flags.DB(cmd, system)
m.provisionOpt = flags.Provision(cmd, system)
}
func (m *System) StartServices(ctx context.Context) (err error) {
_, err = db.TryToConnect(ctx, m.log, system, m.dbOpt.DSN, m.dbOpt.Profiler)
if err != nil {
return errors.Wrap(err, "could not connect to database")
}
if m.provisionOpt.Database {
err = m.ProvisionMigrateDatabase(ctx)
if err != nil {
return
}
}
err = service.Init(ctx)
if err != nil {
return
}
return
}
// ApiServerPreRun is executed before serve-api command runs REST API server
//
// Should initialize all that needs to run in the background
func (m System) ApiServerPreRun(ctx context.Context) error {
service.DefaultPermissions.Watch(ctx)
return nil
}
// ApiServerRoutes mounts api server routes
func (m *System) ApiServerRoutes(r chi.Router) {
rest.MountRoutes(r)
}
// ProvisionMigrateDatabase migrates database to new version
//
// This is ran by default on serve-api (when not explicitly disabled with --compose-provision-database=false)
// or on demand with "provision migrate-database"
func (m System) ProvisionMigrateDatabase(ctx context.Context) error {
var db, err = factory.Database.Get(system)
if err != nil {
return err
}
db = db.With(ctx)
// Disable profiler for migrations
db.Profiler = nil
return migrate.Migrate(db)
}
// ProvisionAccessControl resets access-control rules for roles admin (2) and everyone (1)
//
// Run with emand with "provision access-control-rules"
func (m System) ProvisionAccessControl(ctx context.Context) error {
var ac = service.DefaultAccessControl
return ac.Grant(ctx, ac.DefaultRules()...)
}
system/system_test.go
+15
View File
@@ -0,0 +1,15 @@
package system
import (
"context"
"testing"
"github.com/cortezaproject/corteza-server/internal/test"
)
func TestConfigure(t *testing.T) {
var config = Configure()
test.Assert(t, config != nil, "Configure valid")
test.Assert(t, func() bool { config.Init(); return true }(), "Initialization ok")
test.Assert(t, config.MakeCLI(context.Background()) != nil, "CLI created")
}