3
0

Improve settings auto-discovery, make it provisionable

This commit is contained in:
Denis Arh
2019-07-14 11:24:43 +02:00
parent dd5820253f
commit ab4b74620f
10 changed files with 481 additions and 234 deletions

View File

@@ -24,6 +24,12 @@ func Configure() *cli.Config {
RootCommandPreRun: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) (err error) {
return
},
},
ApiServerPreRun: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
if c.ProvisionOpt.MigrateDatabase {
cli.HandleError(c.ProvisionMigrateDatabase.Run(ctx, cmd, c))
}
@@ -36,12 +42,6 @@ func Configure() *cli.Config {
cli.HandleError(accessControlSetup(ctx, cmd, c))
}
return
},
},
ApiServerPreRun: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
go service.Watchers(ctx)
return nil
},

View File

@@ -42,7 +42,7 @@ func (f *Filter) Normalize() {
}
}
func (v *Value) SetValueAsString(str string) error {
func (v *Value) SetRawValue(str string) error {
var dummy interface{}
// Test input to be sure we can save it...
if err := json.Unmarshal([]byte(str), &dummy); err != nil {
@@ -58,10 +58,28 @@ func (v *Value) SetValue(value interface{}) (err error) {
return
}
func (ss ValueSet) KV() KV {
func (v *Value) String() (out string) {
if v == nil {
return ""
}
_ = v.Value.Unmarshal(&out)
return
}
func (v *Value) Bool() (out bool) {
if v == nil {
return false
}
_ = v.Value.Unmarshal(&out)
return
}
func (set ValueSet) KV() KV {
m := KV{}
_ = ss.Walk(func(v *Value) error {
_ = set.Walk(func(v *Value) error {
m[v.Name] = v.Value
return nil
})
@@ -77,7 +95,7 @@ func (kv KV) Has(k string) (ok bool) {
func (kv KV) Bool(k string) (out bool) {
out = false
if v, ok := kv[k]; ok {
v.Unmarshal(&out)
_ = v.Unmarshal(&out)
}
return
@@ -86,7 +104,7 @@ func (kv KV) Bool(k string) (out bool) {
func (kv KV) String(k string) (out string) {
out = ""
if v, ok := kv[k]; ok {
v.Unmarshal(&out)
_ = v.Unmarshal(&out)
}
return
@@ -102,3 +120,57 @@ func (kv KV) Filter(prefix string) KV {
return out
}
// Replace finds and updates existing or appends new value
func (set *ValueSet) Replace(n *Value) {
for _, v := range *set {
if v.Name == n.Name {
v.Value = n.Value
return
}
}
*set = append(*set, n)
}
// Replace finds and updates existing or appends new value
func (set *ValueSet) Has(name string) bool {
return set.First(name) != nil
}
// First finds and returns first value
func (set ValueSet) First(name string) *Value {
for _, v := range set {
if v.Name == name {
return v
}
}
return nil
}
// Returns all valus that changed or do not exist in the original set
func (set ValueSet) Changed(in ValueSet) (out ValueSet) {
input:
for _, i := range in {
for _, s := range set {
if s.Name != i.Name {
// Different name, not interested
continue
}
if s.String() == i.String() {
// Value did not change, continue with next input set
continue input
}
// Value changed, break out the loop
break
}
// Hande changed or missing value
out = append(out, i)
}
return
}

View File

@@ -53,10 +53,45 @@ func TestKV_Bool(t *testing.T) {
}
func TestSettingValueAsString(t *testing.T) {
test.NoError(t, (&Value{}).SetValueAsString(`"string"`), "unable to set value as string")
test.NoError(t, (&Value{}).SetValueAsString(`false`), "unable to set value as string")
test.NoError(t, (&Value{}).SetValueAsString(`null`), "unable to set value as string")
test.NoError(t, (&Value{}).SetValueAsString(`42`), "unable to set value as string")
test.NoError(t, (&Value{}).SetValueAsString(`3.14`), "unable to set value as string")
test.Error(t, (&Value{}).SetValueAsString(`error`), "expecting error when not setting JSON")
test.NoError(t, (&Value{}).SetRawValue(`"string"`), "unable to set value as string")
test.NoError(t, (&Value{}).SetRawValue(`false`), "unable to set value as string")
test.NoError(t, (&Value{}).SetRawValue(`null`), "unable to set value as string")
test.NoError(t, (&Value{}).SetRawValue(`42`), "unable to set value as string")
test.NoError(t, (&Value{}).SetRawValue(`3.14`), "unable to set value as string")
test.Error(t, (&Value{}).SetRawValue(`error`), "expecting error when not setting JSON")
}
func TestValueSet_Upsert(t *testing.T) {
var vv = ValueSet{}
test.Assert(t, len(vv) == 0, "expecting length to be 0")
vv.Replace(&Value{Name: "name"})
test.Assert(t, len(vv) == 1, "expecting length to be 1")
vv.Replace(&Value{Name: "name", Value: []byte("42")})
test.Assert(t, len(vv) == 1, "expecting length to be 1")
test.Assert(t, string(vv[0].Value) == "42", "expecting value to be 42")
}
func TestValueSet_Changed(t *testing.T) {
var (
// make string value
msv = func(n, v string) *Value {
o := &Value{Name: n}
_ = o.SetValue(v)
return o
}
org = ValueSet{msv("a", "a1"), msv("b", "b1"), msv("d", "d1")}
inp = ValueSet{msv("a", "a2"), msv("c", "c1"), msv("d", "d1")}
out ValueSet
)
out = org.Changed(inp)
test.Assert(t, len(out) == 2, "expecting length to be 2, got %d", len(out))
test.Assert(t, out.First("a").String() == "a2", "expecting 'a' to have 'a2' value")
test.Assert(t, out.First("b").String() == "", "expecting 'b' to be missing")
test.Assert(t, out.First("c").String() == "c1", "expecting 'c' to have 'c1' value")
test.Assert(t, out.First("d").String() == "", "expecting 'd' to be missing")
}

View File

@@ -31,6 +31,12 @@ func Configure() *cli.Config {
RootCommandPreRun: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) (err error) {
return
},
},
ApiServerPreRun: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
if c.ProvisionOpt.MigrateDatabase {
cli.HandleError(c.ProvisionMigrateDatabase.Run(ctx, cmd, c))
}
@@ -52,12 +58,6 @@ func Configure() *cli.Config {
cli.HandleError(makeDefaultChannels(ctx, cmd, c))
}
return
},
},
ApiServerPreRun: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
go service.Watchers(ctx)
return nil
},

View File

@@ -29,6 +29,11 @@ type (
}
)
var (
Monolith = false
BaseURL = "/"
)
func NewServer(log *zap.Logger) *Server {
return &Server{
endpoints: make([]func(r chi.Router), 0),

View File

@@ -71,3 +71,7 @@ func makeDefaultApplications(ctx context.Context, cmd *cobra.Command, c *cli.Con
return nil
})
}
func discoverSettings(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
return service.DefaultSettings.With(ctx).Discover()
}

View File

@@ -8,7 +8,6 @@ import (
"github.com/spf13/cobra"
"github.com/cortezaproject/corteza-server/internal/rand"
"github.com/cortezaproject/corteza-server/internal/settings"
"github.com/cortezaproject/corteza-server/pkg/cli"
"github.com/cortezaproject/corteza-server/system/internal/service"
@@ -16,58 +15,12 @@ import (
func Settings(ctx context.Context) *cobra.Command {
var (
systemApiUrl, authFrontendUrl, authFromAddress, authFromName string
cmd = &cobra.Command{
Use: "settings",
Short: "Settings management",
}
)
auto := &cobra.Command{
Use: "auto-configure",
Short: "Run autoconfiguration",
Run: func(cmd *cobra.Command, args []string) {
_, _ = service.DefaultSettings.LoadAuthSettings()
SettingsAutoConfigure(
cmd,
systemApiUrl,
authFrontendUrl,
authFromAddress,
authFromName,
)
},
}
auto.Flags().StringVar(
&systemApiUrl,
"system-api-url",
"",
"System API URL (http://sytem.api.example.tld)",
)
auto.Flags().StringVar(
&authFrontendUrl,
"auth-frontend-url",
"",
"http://example.tld",
)
auto.Flags().StringVar(
&authFromAddress,
"auth-from-address",
"",
"name@example.tld",
)
auto.Flags().StringVar(
&authFromName,
"auth-from-name",
"",
"Name Surname",
)
list := &cobra.Command{
Use: "list",
Short: "List all",
@@ -116,7 +69,7 @@ func Settings(ctx context.Context) *cobra.Command {
Name: args[0],
}
if err := v.SetValueAsString(value); err != nil {
if err := v.SetRawValue(value); err != nil {
cli.HandleError(err)
}
@@ -205,7 +158,6 @@ func Settings(ctx context.Context) *cobra.Command {
}
cmd.AddCommand(
auto,
list,
get,
set,
@@ -217,123 +169,8 @@ func Settings(ctx context.Context) *cobra.Command {
return cmd
}
func SettingsAutoConfigure(cmd *cobra.Command, systemApiUrl, frontendUrl, fromAddress, fromName string) {
set := func(name string, value interface{}) {
var (
v *settings.Value
ok bool
err error
)
if setFn, ok := value.(func() interface{}); ok {
// No existing value found
if value = setFn(); value == nil {
// setFn returned nil, skip everything..
return
}
}
// Did we get something else than *settings.Value?
// if so, wrap/marshal it into *settings.Value
if v, ok = value.(*settings.Value); !ok {
v = &settings.Value{Name: name}
if v.Value, err = json.Marshal(value); err != nil {
cmd.Printf("could not marshal setting value: %v", err)
return
}
}
err = service.DefaultIntSettings.Set(v)
if err != nil {
cmd.Printf("could not store setting: %v", err)
}
}
setIfMissing := func(name string, value interface{}) {
if existing, err := service.DefaultIntSettings.Get(name, 0); err == nil && existing == nil {
set(name, value)
}
}
// Where should external authentication providers redirect to?
setIfMissing("auth.external.redirect-url", func() interface{} {
path := "/auth/external/%s/callback"
if len(systemApiUrl) > 0 {
if strings.Index(systemApiUrl, "http") != 0 {
return "https://" + systemApiUrl + path
} else {
return systemApiUrl + path
}
}
if leHost, has := os.LookupEnv("LETSENCRYPT_HOST"); has {
return "https://" + leHost + path
} else if vHost, has := os.LookupEnv("VIRTUAL_HOST"); has {
return "http://" + vHost + path
} else {
// Fallback to local
return "http://system.api.local.crust.tech" + path
}
})
setIfMissing("auth.external.session-store-secret", func() interface{} {
// Generate session store secret if missing
return string(rand.Bytes(64))
})
setIfMissing("auth.external.session-store-secure", func() interface{} {
// Try to determines if we need secure session store from redirect URL scheme
extRedirUrl, _ := service.DefaultIntSettings.GetGlobalString("auth.external.redirect-url")
return strings.Contains(extRedirUrl, "https://")
})
if len(frontendUrl) > 0 {
setIfMissing("auth.frontend.url.password-reset", func() interface{} {
return frontendUrl + "/auth/reset-password?token="
})
setIfMissing("auth.frontend.url.email-confirmation", func() interface{} {
return frontendUrl + "/auth/confirm-email?token="
})
setIfMissing("auth.frontend.url.redirect", func() interface{} {
return frontendUrl + "/auth/"
})
setIfMissing("auth.frontend.url.base", func() interface{} {
return frontendUrl + "/"
})
}
// Auth email (password reset, email confirmation)
setIfMissing("auth.mail.from-address", func() interface{} {
if len(fromAddress) > 0 {
return fromAddress
}
return "change-me@example.tld"
})
setIfMissing("auth.mail.from-name", func() interface{} {
if len(fromName) > 0 {
return fromName
}
return "Corteza Team"
})
// No external providers preconfigured, so disable
setIfMissing("auth.external.enabled", false)
// Enable internal by default
setIfMissing("auth.internal.enabled", true)
// Enable user creation
setIfMissing("auth.internal.signup.enabled", true)
// No need to confirm email
setIfMissing("auth.internal.signup-email-confirmation-required", false)
// We need password reset
setIfMissing("auth.internal.password-reset.enabled", true)
func SettingsAutoConfigure(cmd *cobra.Command) {
// @todo load
// autoDiscoverAuthSettings()
// @todo store
}

View File

@@ -3,12 +3,10 @@ package external
import (
"encoding/json"
"fmt"
"os"
"strings"
"github.com/pkg/errors"
"github.com/cortezaproject/corteza-server/internal/rand"
intset "github.com/cortezaproject/corteza-server/internal/settings"
)
@@ -112,40 +110,9 @@ func ExternalAuthSettings(s intset.Service) (eas *externalAuthSettings, err erro
sessionStoreSecret: kv.String(settingsKeySessionStoreSecret),
}
storeGenerated := func(name string, value interface{}) (err error) {
v := &intset.Value{Name: name}
if v.Value, err = json.Marshal(value); err != nil {
return
}
return s.Set(v)
}
if eas.redirectUrl == "" {
path := "/auth/external/%s/callback"
if leHost, has := os.LookupEnv("LETSENCRYPT_HOST"); has {
eas.redirectUrl = "https://" + leHost + path
} else if vHost, has := os.LookupEnv("VIRTUAL_HOST"); has {
eas.redirectUrl = "http://" + vHost + path
} else {
// Fallback to local
eas.redirectUrl = "http://system.api.local.crust.tech" + path
}
if err = storeGenerated(settingsKeyRedirectUrl, eas.redirectUrl); err != nil {
return
}
}
if eas.sessionStoreSecret == "" {
eas.sessionStoreSecret = string(rand.Bytes(64))
if err = storeGenerated(settingsKeySessionStoreSecret, eas.sessionStoreSecret); err != nil {
return
}
}
if !kv.Has(settingsKeySessionStoreSecure) {
// Assume we want to use secure store if we are accessed via HTTPS
// If auth.external.session-store-secure is not explicitly set;
// check if redirectUrl uses HTTPS schema and assume we want secure session store
eas.sessionStoreSecure = strings.Index(eas.redirectUrl, "https://") == 0
} else {
eas.sessionStoreSecure = kv.Bool(settingsKeySessionStoreSecure)

View File

@@ -2,12 +2,19 @@ package service
import (
"context"
"errors"
"net/url"
"os"
"strings"
_ "github.com/joho/godotenv/autoload"
"github.com/pkg/errors"
"github.com/spf13/cast"
"github.com/titpetric/factory"
"go.uber.org/zap"
"github.com/cortezaproject/corteza-server/internal/rand"
internalSettings "github.com/cortezaproject/corteza-server/internal/settings"
"github.com/cortezaproject/corteza-server/pkg/api"
"github.com/cortezaproject/corteza-server/system/internal/repository"
)
@@ -35,6 +42,7 @@ type (
Get(name string, ownedBy uint64) (out *internalSettings.Value, err error)
LoadAuthSettings() (authSettings, error)
Discover() error
}
)
@@ -104,3 +112,321 @@ func (svc settings) LoadAuthSettings() (authSettings, error) {
}
return AuthSettings(vv.KV()), nil
}
func (svc settings) Discover() (err error) {
var (
current, discovered internalSettings.ValueSet
)
current, err = svc.internalSettings.FindByPrefix("")
if err != nil {
return
}
discovered, err = svc.discover(current)
if err != nil || len(discovered) == 0 {
return
}
return svc.internalSettings.BulkSet(discovered)
}
// Discovers "auth.%" settings from the environment
//
// This could (should) probably be refactored into something more general.
func (svc settings) discover(current internalSettings.ValueSet) (internalSettings.ValueSet, error) {
type (
stringWrapper func() string
boolWrapper func() bool
)
var (
new = current
log = svc.logger.Named("discovery")
// Setter
//
// Finds existing settings, tries with environmental "PROVISION_SETTINGS_AUTH_..." probing
// and falls back to default value
//
// We are extremely verbose here - we want to show all the info available and
// how settings were discovered and set
set = func(name string, env string, def interface{}) {
var (
log = log.With(
zap.String("name", name),
)
v = current.First(name)
value interface{}
)
if v != nil {
// Nothing to discover, already set
log.Info("already set", zap.Any("value", v.String()))
return
}
v = &internalSettings.Value{Name: name}
value, envExists := os.LookupEnv(env)
switch dfn := def.(type) {
case stringWrapper:
log = log.With(zap.String("type", "string"))
// already a string, no need to do any magic
if envExists {
log = log.With(zap.String("env", env), zap.Any("value", value))
} else {
value = dfn()
log = log.With(zap.Any("default", value))
}
case boolWrapper:
log = log.With(zap.String("type", "bool"))
if envExists {
value = cast.ToBool(value)
log = log.With(zap.String("env", env), zap.Any("value", value))
} else {
value = dfn()
log = log.With(zap.Any("default", value))
}
default:
log.Error("unsupported type")
return
}
if err := v.SetValue(value); err != nil {
log.Error("could not set value", zap.Error(err))
return
}
log.Info("value auto-discovered")
new.Replace(v)
}
// Default value functions
//
// all are wrapped (stringWrapper, boolWrapper) to delay execution
// of the function to the very last point
frontendUrl = func(path string) stringWrapper {
return func() (base string) {
base = new.First("auth.frontend.url.base").String()
if len(base) == 0 {
// Not found, try to get it from the external redirect URL
base = new.First("auth.external.redirect-url").String()
if len(base) == 0 {
return
}
p, err := url.Parse(base)
if err != nil {
return
}
h := p.Host
s := "api."
if i := strings.Index(h, s); i > 0 {
// If there is a "api." prefix in the hostname of the external redirect-uri value
// cut it off and use that as a frontend url base
h = h[i+len(s):]
}
base = p.Scheme + "://" + h
}
if len(base) > 0 {
return strings.TrimRight(base, "/") + path
}
return ""
}
}
// Assuming secure backend when redirect URL starts with https://
isSecure = func() boolWrapper {
return func() bool {
return strings.Index(new.First("auth.external.redirect-url").String(), "https://") == 0
}
}
// Assume we have emailing capabilities if SMTP_HOST variable is set
emailCapabilities = func() boolWrapper {
return func() bool {
val, has := os.LookupEnv("SMTP_HOST")
return has && len(val) > 0
}
}
// Where should external authentication providers redirect to?
// we need to set full, absolute URL to the callback endpoint
externalAuthRedirectUrl = func() stringWrapper {
return func() string {
var (
path = "/auth/external/%s/callback"
// All env keys we'll check, first that has any value set, will be used as hostname
keysWithHostnames = []string{
"DOMAIN",
"LETSENCRYPT_HOST",
"VIRTUAL_HOST",
"HOSTNAME",
"HOST",
}
)
// Prefix path if we're running wrapped as a monolith:
if api.Monolith {
path = "/system" + path
}
// Finally, add any prefix
path = strings.TrimRight(api.BaseURL, "/") + path
log.Info("scanning env variables for hostname", zap.Strings("candidates", keysWithHostnames))
for _, key := range keysWithHostnames {
if host, has := os.LookupEnv(key); has {
log.Info("hostname env variable found", zap.String("env", key))
// Make life easier for development in local environment,
// and set HTTP schema. Might cause problems if someone
// is using valid external hostname
if strings.Contains(host, "local.") {
return "http://" + host + path
} else {
return "https://" + host + path
}
} else {
}
}
// Fallback is empty string
// this will cause error when doing OIDC auto-discovery (and we want that)
// @todo ^^
return ""
}
}
rand stringWrapper = func() string {
return string(rand.Bytes(64))
}
wrapBool = func(val bool) boolWrapper {
return func() bool { return val }
}
wrapString = func(val string) stringWrapper {
return func() string { return val }
}
)
// List of name-value pairs we need to iterate and set
list := []struct {
// Setting name
nme string
// provision environmental variable name
// we're using full variable name here so developers
// can find where things are comming from
env string
// default value
// expects one of the *wrapper() functions
// this also determinate the value type of the setting and casting rules for the env value
def interface{}
}{
// // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // //
// External auth
// Enable external auth
{
"auth.external.enabled",
"PROVISION_SETTINGS_AUTH_EXTERNAL_ENABLED",
wrapBool(true)},
{
"auth.external.redirect-url",
"PROVISION_SETTINGS_AUTH_EXTERNAL_REDIRECT_URL",
externalAuthRedirectUrl()},
{
"auth.external.session-store-secret",
"PROVISION_SETTINGS_AUTH_EXTERNAL_SESSION_STORE_SECRET",
rand},
// Disable external auth
{
"auth.external.session-store-secure",
"PROVISION_SETTINGS_AUTH_EXTERNAL_SESSION_STORE_SECURE",
isSecure()},
// // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // //
// Auth frontend
{
"auth.frontend.url.base",
"PROVISION_SETTINGS_AUTH_FRONTEND_URL_BASE",
frontendUrl("/")},
// @todo w/o token=
{
"auth.frontend.url.password-reset",
"PROVISION_SETTINGS_AUTH_FRONTEND_URL_PASSWORD_RESET",
frontendUrl("/auth/reset-password?token=")},
// @todo w/o token=
{
"auth.frontend.url.email-confirmation",
"PROVISION_SETTINGS_AUTH_FRONTEND_URL_EMAIL_CONFIRMATION",
frontendUrl("/auth/confirm-email?token=")},
// @todo check if this is correct?!
{
"auth.frontend.url.redirect",
"PROVISION_SETTINGS_AUTH_FRONTEND_URL_REDIRECT",
frontendUrl("/auth")},
// Auth email
{
"auth.mail.from-address",
"PROVISION_SETTINGS_AUTH_EMAIL_FROM_ADDRESS",
wrapString("to-be-configured@example.tld")},
{
"auth.mail.from-name",
"PROVISION_SETTINGS_AUTH_EMAIL_FROM_NAME",
wrapString("Corteza Team (to-be-configured)")},
// // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // // //
// Enable internal signup
{
"auth.internal.signup.enabled",
"PROVISION_SETTINGS_AUTH_INTERNAL_SIGNUP_ENABLED",
wrapBool(true)},
// Enable email confirmation if we have email capabilities
{
"auth.internal.signup-email-confirmation-required",
"PROVISION_SETTINGS_AUTH_INTERNAL_SIGNUP_EMAIL_CONFIRMATION_REQUIRED",
emailCapabilities()},
// Enable password reset if we have email capabilities
{
"auth.internal.password-reset.enabled",
"PROVISION_SETTINGS_AUTH_INTERNAL_PASSWORD_RESET_ENABLED",
emailCapabilities()},
}
for _, item := range list {
set(item.nme, item.env, item.def)
}
// return new, nil
return current.Changed(new), nil
}

View File

@@ -25,6 +25,12 @@ func Configure() *cli.Config {
RootCommandPreRun: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) (err error) {
return
},
},
ApiServerPreRun: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
if c.ProvisionOpt.MigrateDatabase {
cli.HandleError(c.ProvisionMigrateDatabase.Run(ctx, cmd, c))
}
@@ -34,20 +40,15 @@ func Configure() *cli.Config {
if c.ProvisionOpt.AutoSetup {
cli.HandleError(accessControlSetup(ctx, cmd, c))
cli.HandleError(makeDefaultApplications(ctx, cmd, c))
cli.HandleError(discoverSettings(ctx, cmd, c))
// Run auto configuration
commands.SettingsAutoConfigure(cmd, "", "", "", "")
commands.SettingsAutoConfigure(cmd)
// Reload auto-configured settings
service.DefaultAuthSettings, _ = service.DefaultSettings.LoadAuthSettings()
}
return
},
},
ApiServerPreRun: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
external.Init(service.DefaultIntSettings)
go service.Watchers(ctx)
return nil