From bba02ec9df45ecaea2b8407d5cbb9e253a514738 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Mon, 1 Aug 2022 14:18:46 +0200 Subject: [PATCH] Implement base valuestore for env variables * Base facility to store generic values, * integrate with templates, * integrate with workflows. --- .env.example | 12 +++ app/boot_levels.go | 77 ++++++++++++++ app/options/auth.cue | 6 +- app/options/http_server.cue | 10 ++ pkg/options/helpers.go | 28 +++-- pkg/options/options.gen.go | 10 +- pkg/valuestore/valuestore.go | 46 +++++++++ pkg/valuestore/valuestore_test.go | 41 ++++++++ store/adapters/rdbms/aux_types.gen.go | 3 +- system/automation/valuestore_handler.gen.go | 109 ++++++++++++++++++++ system/automation/valuestore_handler.go | 33 ++++++ system/automation/valuestore_handler.yaml | 20 ++++ system/renderer/funcs.go | 9 ++ system/renderer/html.go | 1 + system/renderer/text.go | 5 +- system/service/service.go | 8 ++ 16 files changed, 403 insertions(+), 15 deletions(-) create mode 100644 pkg/valuestore/valuestore.go create mode 100644 pkg/valuestore/valuestore_test.go create mode 100644 system/automation/valuestore_handler.gen.go create mode 100644 system/automation/valuestore_handler.go create mode 100644 system/automation/valuestore_handler.yaml create mode 100644 system/renderer/funcs.go diff --git a/.env.example b/.env.example index 3714c23f6..a361303c3 100644 --- a/.env.example +++ b/.env.example @@ -53,6 +53,18 @@ # HTTP Server # +############################################################################### +# Domain for the HTTP server. +# Type: string +# Default: localhost +# DOMAIN=localhost + +############################################################################### +# Domain for the HTTP webapp. +# Type: string +# Default: localhost +# DOMAIN_WEBAPP=localhost + ############################################################################### # IP and port for the HTTP server. # Type: string diff --git a/app/boot_levels.go b/app/boot_levels.go index 94132482f..74609cfc9 100644 --- a/app/boot_levels.go +++ b/app/boot_levels.go @@ -4,7 +4,9 @@ import ( "context" "crypto/tls" "fmt" + "net/url" "os" + "strings" "time" authService "github.com/cortezaproject/corteza-server/auth" @@ -34,6 +36,8 @@ import ( "github.com/cortezaproject/corteza-server/pkg/scheduler" "github.com/cortezaproject/corteza-server/pkg/seeder" "github.com/cortezaproject/corteza-server/pkg/sentry" + "github.com/cortezaproject/corteza-server/pkg/valuestore" + "github.com/cortezaproject/corteza-server/pkg/version" "github.com/cortezaproject/corteza-server/pkg/websocket" "github.com/cortezaproject/corteza-server/store" "github.com/cortezaproject/corteza-server/system/service" @@ -292,6 +296,8 @@ func (app *CortezaApp) InitServices(ctx context.Context) (err error) { return fmt.Errorf("can not initialize auth: %w", err) } + initValuestore(app.Opt) + app.WsServer = websocket.Server( app.Log, app.Opt.Websocket, @@ -699,6 +705,77 @@ func updateLocaleSettings(opt options.LocaleOpt) { }) } +// initValuestore initializes and sets the global valuestore with environment variables +func initValuestore(opt *options.Options) { + s := valuestore.New() + + apiHostname := options.GuessApiHostname() + + // Base variables + vars := map[string]any{ + // General environment variables such as environment name and version info + "name": opt.Environment.Environment, + "is-development": opt.Environment.IsDevelopment(), + "is-test": opt.Environment.IsTest(), + "is-production": opt.Environment.IsProduction(), + + "version": version.Version, + "build-time": version.BuildTime, + } + + // Auth variables + vars["auth.base-url"] = opt.Auth.BaseURL + vars["auth.domain"] = apiHostname + + // In case there is a missmatch in the auth base URL and server domain, + // guess the domain from the auth baseURL. + if !strings.Contains(opt.Auth.BaseURL, apiHostname) { + u, err := url.Parse(opt.Auth.BaseURL) + if err != nil { + panic(err.Error()) + } + vars["auth.domain"] = u.Host + } + + // API variables + + // API related values -- domain, base url, base sink route, ... + vars["api.domain"] = apiHostname + vars["api.base-url"] = options.FullURL(opt.HTTPServer.BaseUrl, opt.HTTPServer.ApiBaseUrl) + + // Web applications + webappDomain := "" + webappBaseURL := "" + webappBaseURLWebapps := map[string]string{} + + if opt.HTTPServer.WebappEnabled { + // When served from the server container, use server variables + webappDomain = apiHostname + webappBaseURL = options.FullURL(opt.HTTPServer.BaseUrl, opt.HTTPServer.WebappBaseUrl) + } else { + // When not served from the server, use client variables + webappDomain = options.GuessWebappHostname() + webappBaseURL = options.FullWebappURL(opt.HTTPServer.BaseUrl, opt.HTTPServer.WebappBaseUrl) + } + // Web applications + for _, w := range strings.Split(opt.HTTPServer.WebappList, ",") { + w = strings.TrimSpace(w) + webappBaseURLWebapps[w] = webappBaseURL + w + } + + // Webapp related values -- domain, base url (for webapps), ... + // Splitting the two since the webapps can be served somewhere else on + // a completely different domain + vars["webapp.domain"] = webappDomain + vars["webapp.base-url"] = webappBaseURL + for k, v := range webappBaseURLWebapps { + vars[fmt.Sprintf("webapp.base-url.%s", k)] = v + } + + s.SetEnv(vars) + valuestore.SetGlobal(s) +} + // takes current options (SMTP_* env variables) and copies their values to settings func applySmtpOptionsToSettings(ctx context.Context, log *zap.Logger, opt options.SMTPOpt, current *types.AppSettings) (err error) { if len(opt.Host) == 0 { diff --git a/app/options/auth.cue b/app/options/auth.cue index 9f2121fe0..653aa9378 100644 --- a/app/options/auth.cue +++ b/app/options/auth.cue @@ -90,7 +90,7 @@ auth: schema.#optionsGroup & { `provider` placeholder is replaced with the actual value when used. """ - defaultGoExpr: "fullURL(\"/auth/external/{provider}/callback\")" + defaultGoExpr: "FullURL(\"/auth/external/{provider}/callback\")" } external_cookie_secret: { description: """ @@ -111,7 +111,7 @@ auth: schema.#optionsGroup & { This is used for some redirects and links in auth emails. """ - defaultGoExpr: "fullURL(\"/auth\")" + defaultGoExpr: "FullURL(\"/auth\")" } session_cookie_name: { description: "Session cookie name" @@ -122,7 +122,7 @@ auth: schema.#optionsGroup & { defaultGoExpr: "pathPrefix(\"/auth\")" } session_cookie_domain: { - defaultGoExpr: "guessHostname()" + defaultGoExpr: "GuessApiHostname()" description: "Session cookie domain" } session_cookie_secure: { diff --git a/app/options/http_server.cue b/app/options/http_server.cue index 8ca1f769a..476cd6f06 100644 --- a/app/options/http_server.cue +++ b/app/options/http_server.cue @@ -13,6 +13,16 @@ HTTPServer: schema.#optionsGroup & { ] options: { + domain: { + defaultValue: "localhost" + description: "Domain for the HTTP server." + env: "DOMAIN" + } + domainWebapp: { + defaultValue: "localhost" + description: "Domain for the HTTP webapp." + env: "DOMAIN_WEBAPP" + } addr: { defaultValue: ":80" description: "IP and port for the HTTP server." diff --git a/pkg/options/helpers.go b/pkg/options/helpers.go index b8d0406d9..af8b9a040 100644 --- a/pkg/options/helpers.go +++ b/pkg/options/helpers.go @@ -61,15 +61,23 @@ func fill(opt interface{}) { } } -func guessHostname() string { +func GuessApiHostname() string { + return guessHostname(os.Getenv("DOMAIN")) +} + +func GuessWebappHostname() string { + return guessHostname(os.Getenv("DOMAIN_WEBAPP")) +} + +func guessHostname(base ...string) string { // All env keys we'll check, first that has any value set, will be used as hostname - candidates := []string{ - os.Getenv("DOMAIN"), + candidates := append(base, []string{ + os.Getenv("LETSENCRYPT_HOST"), os.Getenv("VIRTUAL_HOST"), os.Getenv("HOSTNAME"), os.Getenv("HOST"), - } + }...) for _, host := range candidates { if len(host) > 0 { @@ -86,10 +94,18 @@ func pathPrefix(pp ...string) string { } // will return base URL with domain and prefix path -func fullURL(pp ...string) string { +func FullURL(pp ...string) string { + return fullUrl(GuessApiHostname()) +} + +func FullWebappURL(pp ...string) string { + return fullUrl(GuessWebappHostname()) +} + +// will return base URL with domain and prefix path +func fullUrl(host string, pp ...string) string { var ( full string - host = guessHostname() ) if strings.Contains(host, "local.") || strings.Contains(host, "localhost") || !isSecure() { diff --git a/pkg/options/options.gen.go b/pkg/options/options.gen.go index 81992e2b7..2a8353f88 100644 --- a/pkg/options/options.gen.go +++ b/pkg/options/options.gen.go @@ -23,6 +23,8 @@ type ( } HttpServerOpt struct { + Domain string `env:"DOMAIN"` + DomainWebapp string `env:"DOMAIN_WEBAPP"` Addr string `env:"HTTP_ADDR"` LogRequest bool `env:"HTTP_LOG_REQUEST"` LogResponse bool `env:"HTTP_LOG_RESPONSE"` @@ -327,6 +329,8 @@ func HTTPClient() (o *HTTPClientOpt) { // This function is auto-generated func HttpServer() (o *HttpServerOpt) { o = &HttpServerOpt{ + Domain: "localhost", + DomainWebapp: "localhost", Addr: ":80", EnableHealthcheckRoute: true, EnableVersionRoute: true, @@ -518,12 +522,12 @@ func Auth() (o *AuthOpt) { Secret: getSecretFromEnv("jwt secret"), AccessTokenLifetime: time.Hour * 2, RefreshTokenLifetime: time.Hour * 24 * 3, - ExternalRedirectURL: fullURL("/auth/external/{provider}/callback"), + ExternalRedirectURL: FullURL("/auth/external/{provider}/callback"), ExternalCookieSecret: getSecretFromEnv("external cookie secret"), - BaseURL: fullURL("/auth"), + BaseURL: FullURL("/auth"), SessionCookieName: "session", SessionCookiePath: pathPrefix("/auth"), - SessionCookieDomain: guessHostname(), + SessionCookieDomain: GuessApiHostname(), SessionCookieSecure: isSecure(), SessionLifetime: 24 * time.Hour, SessionPermLifetime: 360 * 24 * time.Hour, diff --git a/pkg/valuestore/valuestore.go b/pkg/valuestore/valuestore.go new file mode 100644 index 000000000..fe8c92669 --- /dev/null +++ b/pkg/valuestore/valuestore.go @@ -0,0 +1,46 @@ +package valuestore + +import ( + "strings" +) + +type ( + store struct { + env map[string]any + } +) + +var ( + gStore *store +) + +func SetGlobal(s *store) { + gStore = s +} + +func New() *store { + return &store{} +} + +func Global() *store { + return gStore +} + +func EnvGetter() func(string) any { + return gStore.Env +} + +func (s *store) SetEnv(env map[string]any) { + if s.env != nil { + panic("cannot redefine environment variables") + } + + s.env = env +} + +func (s *store) Env(k string) (v any) { + if s.env == nil { + panic("valuestore env not initialized") + } + return s.env[strings.ToLower(k)] +} diff --git a/pkg/valuestore/valuestore_test.go b/pkg/valuestore/valuestore_test.go new file mode 100644 index 000000000..253d0b642 --- /dev/null +++ b/pkg/valuestore/valuestore_test.go @@ -0,0 +1,41 @@ +package valuestore + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSettingEnv(t *testing.T) { + s := New() + s.SetEnv(map[string]any{ + "k1": "v1", + "k2": "v2", + }) + + require.Equal(t, "v1", s.Env("k1")) + require.Equal(t, "v2", s.Env("k2")) +} + +func TestPreventResettingEnv(t *testing.T) { + s := New() + s.SetEnv(map[string]any{ + "k1": "v1", + "k2": "v2", + }) + + require.Panics(t, func() { + s.SetEnv(map[string]any{ + "k3": "v3", + "k4": "v4", + }) + }) +} + +func TestPreventUninitializedAccess(t *testing.T) { + s := New() + + require.Panics(t, func() { + s.Env("k1") + }) +} diff --git a/store/adapters/rdbms/aux_types.gen.go b/store/adapters/rdbms/aux_types.gen.go index 8fc7ef5b6..89b593b2a 100644 --- a/store/adapters/rdbms/aux_types.gen.go +++ b/store/adapters/rdbms/aux_types.gen.go @@ -7,8 +7,6 @@ package rdbms // import ( - "time" - automationType "github.com/cortezaproject/corteza-server/automation/types" composeType "github.com/cortezaproject/corteza-server/compose/types" federationType "github.com/cortezaproject/corteza-server/federation/types" @@ -19,6 +17,7 @@ import ( labelsType "github.com/cortezaproject/corteza-server/pkg/label/types" rbacType "github.com/cortezaproject/corteza-server/pkg/rbac" systemType "github.com/cortezaproject/corteza-server/system/types" + "time" ) type ( diff --git a/system/automation/valuestore_handler.gen.go b/system/automation/valuestore_handler.gen.go new file mode 100644 index 000000000..2541b3340 --- /dev/null +++ b/system/automation/valuestore_handler.gen.go @@ -0,0 +1,109 @@ +package automation + +// This file is auto-generated. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +// Definitions file that controls how this file is generated: +// system/automation/valuestore_handler.yaml + +import ( + "context" + atypes "github.com/cortezaproject/corteza-server/automation/types" + "github.com/cortezaproject/corteza-server/pkg/expr" + "github.com/cortezaproject/corteza-server/pkg/wfexec" +) + +var _ wfexec.ExecResponse + +type ( + valuestoreHandlerRegistry interface { + AddFunctions(ff ...*atypes.Function) + Type(ref string) expr.Type + } +) + +func (h valuestoreHandler) register() { + h.reg.AddFunctions( + h.Env(), + ) +} + +type ( + valuestoreEnvArgs struct { + hasKey bool + Key string + } + + valuestoreEnvResults struct { + Value interface{} + } +) + +// Env function Get ENV variable +// +// expects implementation of env function: +// func (h valuestoreHandler) env(ctx context.Context, args *valuestoreEnvArgs) (results *valuestoreEnvResults, err error) { +// return +// } +func (h valuestoreHandler) Env() *atypes.Function { + return &atypes.Function{ + Ref: "valuestoreEnv", + Kind: "function", + Labels: map[string]string{"templates": "step,workflow"}, + Meta: &atypes.FunctionMeta{ + Short: "Get ENV variable", + Description: "Get ENV variable for the specified key. If the key doesn't correspond to any value, nil is returned", + }, + + Parameters: []*atypes.Param{ + { + Name: "key", + Types: []string{"String"}, Required: true, + }, + }, + + Results: []*atypes.Param{ + + { + Name: "value", + Types: []string{"Any"}, + }, + }, + + Handler: func(ctx context.Context, in *expr.Vars) (out *expr.Vars, err error) { + var ( + args = &valuestoreEnvArgs{ + hasKey: in.Has("key"), + } + ) + + if err = in.Decode(args); err != nil { + return + } + + var results *valuestoreEnvResults + if results, err = h.env(ctx, args); err != nil { + return + } + + out = &expr.Vars{} + + { + // converting results.Value (interface{}) to Any + var ( + tval expr.TypedValue + ) + + if tval, err = h.reg.Type("Any").Cast(results.Value); err != nil { + return + } else if err = expr.Assign(out, "value", tval); err != nil { + return + } + } + + return + }, + } +} diff --git a/system/automation/valuestore_handler.go b/system/automation/valuestore_handler.go new file mode 100644 index 000000000..397951dac --- /dev/null +++ b/system/automation/valuestore_handler.go @@ -0,0 +1,33 @@ +package automation + +import ( + "context" +) + +type ( + envGetter interface { + Env(k string) (v any) + } + + valuestoreHandler struct { + reg valuestoreHandlerRegistry + getter envGetter + } +) + +func ValuestoreHandler(reg valuestoreHandlerRegistry, getter envGetter) *valuestoreHandler { + h := &valuestoreHandler{ + reg: reg, + getter: getter, + } + + h.register() + return h +} + +func (h valuestoreHandler) env(ctx context.Context, args *valuestoreEnvArgs) (results *valuestoreEnvResults, err error) { + results = &valuestoreEnvResults{ + Value: h.getter.Env(args.Key), + } + return +} diff --git a/system/automation/valuestore_handler.yaml b/system/automation/valuestore_handler.yaml new file mode 100644 index 000000000..5dde1e5c3 --- /dev/null +++ b/system/automation/valuestore_handler.yaml @@ -0,0 +1,20 @@ +snippets: + +labels: &labels + templates: "step,workflow" + +functions: + env: + meta: + short: Get ENV variable + description: Get ENV variable for the specified key. If the key doesn't correspond to any value, nil is returned + params: + key: + required: true + types: + - { wf: String } + labels: + <<: *labels + results: + value: + wf: Any diff --git a/system/renderer/funcs.go b/system/renderer/funcs.go new file mode 100644 index 000000000..7cb1cbdde --- /dev/null +++ b/system/renderer/funcs.go @@ -0,0 +1,9 @@ +package renderer + +import "github.com/cortezaproject/corteza-server/pkg/valuestore" + +func envGetter() func(k string) any { + return func(k string) any { + return valuestore.Global().Env(k) + } +} diff --git a/system/renderer/html.go b/system/renderer/html.go index d180e305f..b63ba0c8c 100644 --- a/system/renderer/html.go +++ b/system/renderer/html.go @@ -50,6 +50,7 @@ func preprocHTMLTemplate(pl *driverPayload) (*template.Template, error) { raw := base64.RawStdEncoding.EncodeToString(bb) return template.URL("data:" + rsp.Header.Get("Content-Type") + ";base64," + raw), nil }, + "env": envGetter(), }) // Prep the original template diff --git a/system/renderer/text.go b/system/renderer/text.go index 751ab8a4a..fa8a5472b 100644 --- a/system/renderer/text.go +++ b/system/renderer/text.go @@ -15,7 +15,10 @@ func preprocPlainTemplate(tpl io.Reader, pp map[string]io.Reader) (*template.Tem } gtpl := template.New("text/plain_render"). - Funcs(sprig.TxtFuncMap()) + Funcs(sprig.TxtFuncMap()). + Funcs(template.FuncMap{ + "env": envGetter(), + }) // Prep the original template t, err := gtpl.Parse(string(bb)) diff --git a/system/service/service.go b/system/service/service.go index 2e151cec4..aeadc07ae 100644 --- a/system/service/service.go +++ b/system/service/service.go @@ -7,6 +7,7 @@ import ( "github.com/cortezaproject/corteza-server/pkg/dal" "github.com/cortezaproject/corteza-server/pkg/discovery" + "github.com/cortezaproject/corteza-server/pkg/valuestore" automationService "github.com/cortezaproject/corteza-server/automation/service" "github.com/cortezaproject/corteza-server/pkg/actionlog" @@ -255,6 +256,13 @@ func Initialize(ctx context.Context, log *zap.Logger, s store.Storer, ws websock DefaultRole, ) + // ValuestoreHandler isn't (yet) a system thing but this initialization resides + // here just so we can easily register it + automation.ValuestoreHandler( + automationService.Registry(), + valuestore.Global(), + ) + if c.ActionLog.WorkflowFunctionsEnabled { // register action-log functions & types only when enabled automation.ActionlogHandler(