Refactor app bootstraping, improve ext provider autodiscovery

compose/autosetup.go

@@ -10,6 +10,8 @@ import (
 )
 
 func accessControlSetup(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
+	c.InitServices(ctx, c)
+
 	// Calling grant directly on internal permissions service to avoid AC check for "grant"
 	var p = service.DefaultPermissions
 	var ac = service.DefaultAccessControl

compose/compose.go

@@ -19,6 +19,8 @@ const (
 )
 
 func Configure() *cli.Config {
+	var servicesInitialized bool
+
 	return &cli.Config{
 		ServiceName: compose,
 
@@ -28,15 +30,23 @@ func Configure() *cli.Config {
 			},
 		},
 
+		InitServices: func(ctx context.Context, c *cli.Config) {
+			if servicesInitialized {
+				return
+			}
+			servicesInitialized = true
+
+			storagePath := options.EnvString("", "COMPOSE_STORAGE_PATH", "var/store")
+			cli.HandleError(service.Init(ctx, c.Log, storagePath))
+		},
+
 		ApiServerPreRun: cli.Runners{
 			func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
 				if c.ProvisionOpt.MigrateDatabase {
 					cli.HandleError(c.ProvisionMigrateDatabase.Run(ctx, cmd, c))
 				}
 
-				storagePath := options.EnvString("", "COMPOSE_STORAGE_PATH", "var/store")
-
-				cli.HandleError(service.Init(ctx, c.Log, storagePath))
+				c.InitServices(ctx, c)
 
 				if c.ProvisionOpt.AutoSetup {
 					cli.HandleError(accessControlSetup(ctx, cmd, c))

messaging/autosetup.go

@@ -13,6 +13,8 @@ import (
 )
 
 func accessControlSetup(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
+	c.InitServices(ctx, c)
+
 	// Calling grant directly on internal permissions service to avoid AC check for "grant"
 	var p = service.DefaultPermissions
 	var ac = service.DefaultAccessControl

messaging/messaging.go

@@ -22,6 +22,8 @@ const (
 
 func Configure() *cli.Config {
 	var (
+		servicesInitialized bool
+
 		// Websocket handler
 		ws *websocket.Websocket
 	)
@@ -35,15 +37,23 @@ func Configure() *cli.Config {
 			},
 		},
 
+		InitServices: func(ctx context.Context, c *cli.Config) {
+			if servicesInitialized {
+				return
+			}
+			servicesInitialized = true
+
+			storagePath := options.EnvString("", "MESSAGING_STORAGE_PATH", "var/store")
+			cli.HandleError(service.Init(ctx, c.Log, storagePath))
+		},
+
 		ApiServerPreRun: cli.Runners{
 			func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
 				if c.ProvisionOpt.MigrateDatabase {
 					cli.HandleError(c.ProvisionMigrateDatabase.Run(ctx, cmd, c))
 				}
 
-				storagePath := options.EnvString("", "MESSAGING_STORAGE_PATH", "var/store")
-
-				cli.HandleError(service.Init(ctx, c.Log, storagePath))
+				c.InitServices(ctx, c)
 
 				var websocketOpt = options.Websocket(messaging)
 

monolith/monolith.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/cortezaproject/corteza-server/compose"
 	"github.com/cortezaproject/corteza-server/messaging"
+	"github.com/cortezaproject/corteza-server/pkg/api"
 	"github.com/cortezaproject/corteza-server/pkg/cli"
 	"github.com/cortezaproject/corteza-server/system"
 )
@@ -22,6 +23,9 @@ func Configure() *cli.Config {
 	msg.Init()
 	sys.Init()
 
+	// Set API as a monolith build
+	api.Monolith = true
+
 	// Combines all three services/apps and makes them run as one monolith app
 	return &cli.Config{
 		ServiceName: "",

pkg/cli/runner.go

@@ -88,6 +88,12 @@ type (
 		// Access control initial setup
 		// Reapplies default access control rules for roles "everyone" [1] and "admin" [2]
 		ProvisionAccessControl Runners
+
+		// ******************************************************************
+
+		// This callback behaves a bit differently and should be called manually
+		// from wherever we need service initialized
+		InitServices func(ctx context.Context, c *Config)
 	}
 )
 

system/autosetup.go

@@ -14,6 +14,8 @@ import (
 )
 
 func accessControlSetup(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
+	c.InitServices(ctx, c)
+
 	// Calling grant directly on internal permissions service to avoid AC check for "grant"
 	var p = service.DefaultPermissions
 	var ac = service.DefaultAccessControl

system/commands/auth.go

@@ -2,6 +2,7 @@ package commands
 
 import (
 	"context"
+	"net/url"
 	"regexp"
 	"strconv"
 	"strings"
@@ -19,7 +20,7 @@ import (
 )
 
 // Will perform OpenID connect auto-configuration
-func Auth(ctx context.Context) *cobra.Command {
+func Auth(ctx context.Context, c *cli.Config) *cobra.Command {
 	var (
 		enableDiscoveredProvider bool
 	)
@@ -34,12 +35,26 @@ func Auth(ctx context.Context) *cobra.Command {
 		Short: "Auto discovers new OIDC client",
 		Args:  cobra.ExactArgs(2),
 		Run: func(cmd *cobra.Command, args []string) {
-			var name, url = args[0], args[1]
+			c.InitServices(ctx, c)
+
+			var (
+				name, providerUrl = args[0], args[1]
+			)
 
 			eas, err := external.ExternalAuthSettings(service.DefaultIntSettings)
 			cli.HandleError(err)
 
-			eap, err := external.RegisterNewOpenIdClient(ctx, eas, name, url)
+			// Do basic validation of external auth settings
+			// will fail if secret or url are not set
+			cli.HandleError(eas.ValidateStatic())
+
+			// Do full rediredct-URL check
+			cli.HandleError(eas.ValidateRedirectURL())
+
+			p, err := parseExternalProviderUrl(providerUrl)
+			cli.HandleError(err)
+
+			eap, err := external.RegisterNewOpenIdClient(ctx, eas, name, p.String())
 			cli.HandleError(err)
 
 			vv, err := eap.MakeValueSet("openid-connect." + name)
@@ -62,9 +77,9 @@ func Auth(ctx context.Context) *cobra.Command {
 			cli.HandleError(service.DefaultIntSettings.BulkSet(vv))
 
 			if enableDiscoveredProvider {
-				cmd.Printf("OIDC provider successfully added and enabled.")
+				cmd.Println("OIDC provider successfully added and enabled.")
 			} else {
-				cmd.Printf("OIDC provider successfully added (still disabled).")
+				cmd.Println("OIDC provider successfully added (still disabled).")
 			}
 		},
 	}
@@ -80,6 +95,7 @@ func Auth(ctx context.Context) *cobra.Command {
 		Short: "Generates new JWT for a user",
 		Args:  cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
+
 			var (
 				db = factory.Database.MustGet("system")
 
@@ -95,6 +111,8 @@ func Auth(ctx context.Context) *cobra.Command {
 				userStr = args[0]
 			)
 
+			c.InitServices(ctx, c)
+
 			if user, err = userRepo.FindByEmail(userStr); repository.ErrUserNotFound.Eq(err) {
 				if regexp.MustCompile(`/^\d+$/`).MatchString(userStr) {
 					if ID, err = strconv.ParseUint(userStr, 10, 64); err == nil {
@@ -122,6 +140,8 @@ func Auth(ctx context.Context) *cobra.Command {
 		Short: "Sends samples of all authentication notification to receipient",
 		Args:  cobra.ExactArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
+			c.InitServices(ctx, c)
+
 			var (
 				err error
 				ntf = service.DefaultAuthNotification.With(ctx)
@@ -144,3 +164,20 @@ func Auth(ctx context.Context) *cobra.Command {
 
 	return cmd
 }
+
+func parseExternalProviderUrl(in string) (p *url.URL, err error) {
+	if i := strings.Index(in, "://"); i == -1 {
+		// Add schema if missing
+		in = "https://" + in
+	}
+
+	if p, err = url.Parse(in); err != nil {
+		// Try to parse it
+		return
+	} else if i := strings.Index(p.Path, external.WellKnown); i > -1 {
+		// Cut off well-known-path
+		p.Path = p.Path[:i]
+	}
+
+	return
+}

system/commands/auth_test.go

@@ -0,0 +1,68 @@
+package commands
+
+import (
+	"net/url"
+	"reflect"
+	"testing"
+
+	"github.com/cortezaproject/corteza-server/system/internal/auth/external"
+)
+
+func Test_parseExternalProviderUrl(t *testing.T) {
+	mustParseURL := func(in string) *url.URL {
+		r, err := url.Parse(in)
+		if err != nil {
+			panic(err)
+		}
+
+		return r
+	}
+
+	type args struct {
+		in string
+	}
+
+	tests := []struct {
+		name    string
+		args    args
+		wantP   *url.URL
+		wantErr bool
+	}{
+		{
+			"happy-path",
+			args{"https://foo.bar"},
+			mustParseURL("https://foo.bar"),
+			false,
+		},
+		{
+			"bad input",
+			args{":\\"},
+			nil,
+			true,
+		},
+		{
+			"add schema",
+			args{"cortezaproject.org"},
+			mustParseURL("https://cortezaproject.org"),
+			false,
+		},
+		{
+			"add schema and remove well-known",
+			args{"cortezaproject.org/some-subdir/" + external.WellKnown},
+			mustParseURL("https://cortezaproject.org/some-subdir/"),
+			false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotP, err := parseExternalProviderUrl(tt.args.in)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseExternalProviderUrl() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotP, tt.wantP) {
+				t.Errorf("parseExternalProviderUrl() gotP = %v, want %v", gotP, tt.wantP)
+			}
+		})
+	}
+}

system/commands/roles.go

@@ -13,7 +13,7 @@ import (
 	"github.com/cortezaproject/corteza-server/system/types"
 )
 
-func Roles(ctx context.Context) *cobra.Command {
+func Roles(ctx context.Context, c *cli.Config) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "roles",
 		Short: "Role management",
@@ -41,6 +41,8 @@ func Roles(ctx context.Context) *cobra.Command {
 				err error
 			)
 
+			c.InitServices(ctx, c)
+
 			// Try to find role by name and by ID
 			if rr, err = roleRepo.Find(&types.RoleFilter{Query: roleStr}); err != nil {
 				cli.HandleError(err)

system/commands/settings.go

@@ -13,7 +13,7 @@ import (
 	"github.com/cortezaproject/corteza-server/system/internal/service"
 )
 
-func Settings(ctx context.Context) *cobra.Command {
+func Settings(ctx context.Context, c *cli.Config) *cobra.Command {
 	var (
 		cmd = &cobra.Command{
 			Use:   "settings",
@@ -25,6 +25,8 @@ func Settings(ctx context.Context) *cobra.Command {
 		Use:   "list",
 		Short: "List all",
 		Run: func(cmd *cobra.Command, args []string) {
+			c.InitServices(ctx, c)
+
 			prefix := cmd.Flags().Lookup("prefix").Value.String()
 			if kv, err := service.DefaultIntSettings.FindByPrefix(prefix); err != nil {
 				cli.HandleError(err)
@@ -51,6 +53,8 @@ func Settings(ctx context.Context) *cobra.Command {
 		Short: "Get value (raw JSON) for a specific key",
 		Args:  cobra.ExactArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
+			c.InitServices(ctx, c)
+
 			if v, err := service.DefaultIntSettings.Get(args[0], 0); err != nil {
 				cli.HandleError(err)
 			} else if v != nil {
@@ -64,6 +68,8 @@ func Settings(ctx context.Context) *cobra.Command {
 		Short: "Set value (raw JSON) for a specific key",
 		Args:  cobra.ExactArgs(2),
 		Run: func(cmd *cobra.Command, args []string) {
+			c.InitServices(ctx, c)
+
 			value := args[1]
 			v := &settings.Value{
 				Name: args[0],
@@ -82,6 +88,8 @@ func Settings(ctx context.Context) *cobra.Command {
 		Short: "Import settings as JSON from stdin or file",
 		Args:  cobra.MaximumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
+			c.InitServices(ctx, c)
+
 			var (
 				fh  *os.File
 				err error
@@ -120,6 +128,8 @@ func Settings(ctx context.Context) *cobra.Command {
 		Short: "Import settings as JSON to stdout or file",
 		Args:  cobra.MaximumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
+			c.InitServices(ctx, c)
+
 			var (
 				fh  *os.File
 				err error
@@ -151,6 +161,8 @@ func Settings(ctx context.Context) *cobra.Command {
 		Short: "Set value (raw JSON) for a specific key",
 		Args:  cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
+			c.InitServices(ctx, c)
+
 			for a := 0; a < len(args); a++ {
 				cli.HandleError(service.DefaultIntSettings.Delete(args[a], 0))
 			}

system/commands/users.go

@@ -15,7 +15,7 @@ import (
 	"github.com/cortezaproject/corteza-server/system/types"
 )
 
-func Users(ctx context.Context) *cobra.Command {
+func Users(ctx context.Context, c *cli.Config) *cobra.Command {
 	// User management commands.
 	cmd := &cobra.Command{
 		Use:   "users",
@@ -27,6 +27,8 @@ func Users(ctx context.Context) *cobra.Command {
 		Use:   "list",
 		Short: "List users",
 		Run: func(cmd *cobra.Command, args []string) {
+			c.InitServices(ctx, c)
+
 			var (
 				db = factory.Database.MustGet("system")
 			)
@@ -65,6 +67,8 @@ func Users(ctx context.Context) *cobra.Command {
 		Short: "Add new user",
 		Args:  cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
+			c.InitServices(ctx, c)
+
 			var (
 				db = factory.Database.MustGet("system")
 
@@ -105,6 +109,8 @@ func Users(ctx context.Context) *cobra.Command {
 		Short: "Change password for user",
 		Args:  cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
+			c.InitServices(ctx, c)
+
 			var (
 				db = factory.Database.MustGet("system")
 

system/internal/auth/external/goth.go

@@ -18,6 +18,8 @@ import (
 // external auth loop in 15 minutes.
 const (
 	gothMaxSessionStoreAge = 60 * 15 // In seconds.
+
+	WellKnown = "/.well-known/openid-configuration"
 )
 
 func setupGoth(eas *externalAuthSettings) {
@@ -79,7 +81,7 @@ func setupGothProviders(eas *externalAuthSettings) {
 				continue
 			}
 
-			wellKnown := strings.TrimSuffix(pc.issuerUrl, "/") + "/.well-known/openid-configuration"
+			wellKnown := strings.TrimSuffix(pc.issuerUrl, "/") + WellKnown
 
 			if provider, err = openidConnect.New(pc.key, pc.secret, pc.redirectUrl, wellKnown, scopes...); err != nil {
 				log.Error("failed to discover OIDC provider", zap.Error(err), zap.String("well-known", wellKnown))

system/internal/auth/external/settings.go

@@ -3,6 +3,9 @@ package external
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -60,6 +63,61 @@ func ExternalAuthProvider(kv intset.KV) (eap externalAuthProvider, err error) {
 	return
 }
 
+func (eas externalAuthSettings) Enabled() bool {
+	return eas.enabled
+}
+
+func (eas externalAuthSettings) ValidateStatic() error {
+	if eas.redirectUrl == "" {
+		return errors.New("redirect URL is empty")
+	}
+
+	const (
+		tpt = "test-provider-test"
+	)
+	p, err := url.Parse(fmt.Sprintf(eas.redirectUrl, tpt))
+	if err != nil {
+		return errors.Wrap(err, "invalid redirect URL")
+	}
+
+	if !strings.Contains(p.Path, tpt+"/callback") {
+		return errors.Wrap(err, "could find injected provider in the URL, make sure you use '%s' as a placeholder")
+	}
+
+	if eas.sessionStoreSecret == "" {
+		return errors.New("session store secret is empty")
+	}
+
+	if eas.sessionStoreSecure && p.Scheme != "https" {
+		return errors.New("session store is secure, redirect URL should have HTTPS")
+	}
+
+	return nil
+}
+
+func (eas externalAuthSettings) ValidateRedirectURL() error {
+	const tpt = "test-provider-test"
+	const cb = "/callback"
+
+	// Replace placeholders & remove /callback
+	var url = fmt.Sprintf(eas.redirectUrl, tpt)
+	url = url[0 : len(url)-len(cb)]
+
+	rsp, err := http.DefaultClient.Get(url)
+	if err != nil {
+		return errors.Wrap(err, "could not get response from redirect URL")
+	}
+
+	defer rsp.Body.Close()
+	body, err := ioutil.ReadAll(rsp.Body)
+
+	if strings.Contains(string(body), tpt) {
+		return nil
+	}
+
+	return errors.New("could not validate external auth redirection URL")
+}
+
 func (eap externalAuthProvider) MakeValueSet(name string) (vv intset.ValueSet, err error) {
 	set := func(name string, value interface{}) error {
 		v := &intset.Value{Name: name}

system/system.go

@@ -20,6 +20,10 @@ const (
 )
 
 func Configure() *cli.Config {
+	var (
+		servicesInitialized bool
+	)
+
 	return &cli.Config{
 		ServiceName: system,
 
@@ -29,13 +33,24 @@ func Configure() *cli.Config {
 			},
 		},
 
+		InitServices: func(ctx context.Context, c *cli.Config) {
+			if servicesInitialized {
+				return
+			}
+			servicesInitialized = true
+
+			// storagePath := options.EnvString("", "SYSTEM_STORAGE_PATH", "var/store")
+			cli.HandleError(service.Init(ctx, c.Log))
+
+		},
+
 		ApiServerPreRun: cli.Runners{
 			func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
 				if c.ProvisionOpt.MigrateDatabase {
 					cli.HandleError(c.ProvisionMigrateDatabase.Run(ctx, cmd, c))
 				}
 
-				cli.HandleError(service.Init(ctx, c.Log))
+				c.InitServices(ctx, c)
 
 				if c.ProvisionOpt.AutoSetup {
 					cli.HandleError(accessControlSetup(ctx, cmd, c))
@@ -61,16 +76,16 @@ func Configure() *cli.Config {
 
 		AdtSubCommands: cli.CommandMakers{
 			func(ctx context.Context, c *cli.Config) *cobra.Command {
-				return commands.Settings(ctx)
+				return commands.Settings(ctx, c)
 			},
 			func(ctx context.Context, c *cli.Config) *cobra.Command {
-				return commands.Auth(ctx)
+				return commands.Auth(ctx, c)
 			},
 			func(ctx context.Context, c *cli.Config) *cobra.Command {
-				return commands.Users(ctx)
+				return commands.Users(ctx, c)
 			},
 			func(ctx context.Context, c *cli.Config) *cobra.Command {
-				return commands.Roles(ctx)
+				return commands.Roles(ctx, c)
 			},
 		},