Implement base valuestore for env variables

* Base facility to store generic values,
* integrate with templates,
* integrate with workflows.

.env.example

@@ -53,6 +53,18 @@
 # HTTP Server
 #
 
+###############################################################################
+# Domain for the HTTP server.
+# Type:    string
+# Default: localhost
+# DOMAIN=localhost
+
+###############################################################################
+# Domain for the HTTP webapp.
+# Type:    string
+# Default: localhost
+# DOMAIN_WEBAPP=localhost
+
 ###############################################################################
 # IP and port for the HTTP server.
 # Type:    string

app/boot_levels.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"net/url"
 	"os"
+	"strings"
 	"time"
 
 	authService "github.com/cortezaproject/corteza-server/auth"
@@ -34,6 +36,8 @@ import (
 	"github.com/cortezaproject/corteza-server/pkg/scheduler"
 	"github.com/cortezaproject/corteza-server/pkg/seeder"
 	"github.com/cortezaproject/corteza-server/pkg/sentry"
+	"github.com/cortezaproject/corteza-server/pkg/valuestore"
+	"github.com/cortezaproject/corteza-server/pkg/version"
 	"github.com/cortezaproject/corteza-server/pkg/websocket"
 	"github.com/cortezaproject/corteza-server/store"
 	"github.com/cortezaproject/corteza-server/system/service"
@@ -292,6 +296,8 @@ func (app *CortezaApp) InitServices(ctx context.Context) (err error) {
 		return fmt.Errorf("can not initialize auth: %w", err)
 	}
 
+	initValuestore(app.Opt)
+
 	app.WsServer = websocket.Server(
 		app.Log,
 		app.Opt.Websocket,
@@ -699,6 +705,77 @@ func updateLocaleSettings(opt options.LocaleOpt) {
 	})
 }
 
+// initValuestore initializes and sets the global valuestore with environment variables
+func initValuestore(opt *options.Options) {
+	s := valuestore.New()
+
+	apiHostname := options.GuessApiHostname()
+
+	// Base variables
+	vars := map[string]any{
+		// General environment variables such as environment name and version info
+		"name":           opt.Environment.Environment,
+		"is-development": opt.Environment.IsDevelopment(),
+		"is-test":        opt.Environment.IsTest(),
+		"is-production":  opt.Environment.IsProduction(),
+
+		"version":    version.Version,
+		"build-time": version.BuildTime,
+	}
+
+	// Auth variables
+	vars["auth.base-url"] = opt.Auth.BaseURL
+	vars["auth.domain"] = apiHostname
+
+	// In case there is a missmatch in the auth base URL and server domain,
+	// guess the domain from the auth baseURL.
+	if !strings.Contains(opt.Auth.BaseURL, apiHostname) {
+		u, err := url.Parse(opt.Auth.BaseURL)
+		if err != nil {
+			panic(err.Error())
+		}
+		vars["auth.domain"] = u.Host
+	}
+
+	// API variables
+
+	// API related values -- domain, base url, base sink route, ...
+	vars["api.domain"] = apiHostname
+	vars["api.base-url"] = options.FullURL(opt.HTTPServer.BaseUrl, opt.HTTPServer.ApiBaseUrl)
+
+	// Web applications
+	webappDomain := ""
+	webappBaseURL := ""
+	webappBaseURLWebapps := map[string]string{}
+
+	if opt.HTTPServer.WebappEnabled {
+		// When served from the server container, use server variables
+		webappDomain = apiHostname
+		webappBaseURL = options.FullURL(opt.HTTPServer.BaseUrl, opt.HTTPServer.WebappBaseUrl)
+	} else {
+		// When not served from the server, use client variables
+		webappDomain = options.GuessWebappHostname()
+		webappBaseURL = options.FullWebappURL(opt.HTTPServer.BaseUrl, opt.HTTPServer.WebappBaseUrl)
+	}
+	// Web applications
+	for _, w := range strings.Split(opt.HTTPServer.WebappList, ",") {
+		w = strings.TrimSpace(w)
+		webappBaseURLWebapps[w] = webappBaseURL + w
+	}
+
+	// Webapp related values -- domain, base url (for webapps), ...
+	// Splitting the two since the webapps can be served somewhere else on
+	// a completely different domain
+	vars["webapp.domain"] = webappDomain
+	vars["webapp.base-url"] = webappBaseURL
+	for k, v := range webappBaseURLWebapps {
+		vars[fmt.Sprintf("webapp.base-url.%s", k)] = v
+	}
+
+	s.SetEnv(vars)
+	valuestore.SetGlobal(s)
+}
+
 // takes current options (SMTP_* env variables) and copies their values to settings
 func applySmtpOptionsToSettings(ctx context.Context, log *zap.Logger, opt options.SMTPOpt, current *types.AppSettings) (err error) {
 	if len(opt.Host) == 0 {

app/options/auth.cue

@@ -90,7 +90,7 @@ auth: schema.#optionsGroup & {
 
 				`provider` placeholder is replaced with the actual value when used.
 				"""
-			defaultGoExpr: "fullURL(\"/auth/external/{provider}/callback\")"
+			defaultGoExpr: "FullURL(\"/auth/external/{provider}/callback\")"
 		}
 		external_cookie_secret: {
 			description: """
@@ -111,7 +111,7 @@ auth: schema.#optionsGroup & {
 				This is used for some redirects and links in auth emails.
 				"""
 
-			defaultGoExpr: "fullURL(\"/auth\")"
+			defaultGoExpr: "FullURL(\"/auth\")"
 		}
 		session_cookie_name: {
 			description:  "Session cookie name"
@@ -122,7 +122,7 @@ auth: schema.#optionsGroup & {
 			defaultGoExpr: "pathPrefix(\"/auth\")"
 		}
 		session_cookie_domain: {
-			defaultGoExpr: "guessHostname()"
+			defaultGoExpr: "GuessApiHostname()"
 			description:   "Session cookie domain"
 		}
 		session_cookie_secure: {

app/options/http_server.cue

@@ -13,6 +13,16 @@ HTTPServer: schema.#optionsGroup & {
 	]
 
 	options: {
+		domain: {
+			defaultValue: "localhost"
+			description:  "Domain for the HTTP server."
+			env:          "DOMAIN"
+		}
+		domainWebapp: {
+			defaultValue: "localhost"
+			description:  "Domain for the HTTP webapp."
+			env:          "DOMAIN_WEBAPP"
+		}
 		addr: {
 			defaultValue: ":80"
 			description:  "IP and port for the HTTP server."

pkg/options/helpers.go

@@ -61,15 +61,23 @@ func fill(opt interface{}) {
 	}
 }
 
-func guessHostname() string {
+func GuessApiHostname() string {
+	return guessHostname(os.Getenv("DOMAIN"))
+}
+
+func GuessWebappHostname() string {
+	return guessHostname(os.Getenv("DOMAIN_WEBAPP"))
+}
+
+func guessHostname(base ...string) string {
 	// All env keys we'll check, first that has any value set, will be used as hostname
-	candidates := []string{
-		os.Getenv("DOMAIN"),
+	candidates := append(base, []string{
+
 		os.Getenv("LETSENCRYPT_HOST"),
 		os.Getenv("VIRTUAL_HOST"),
 		os.Getenv("HOSTNAME"),
 		os.Getenv("HOST"),
-	}
+	}...)
 
 	for _, host := range candidates {
 		if len(host) > 0 {
@@ -86,10 +94,18 @@ func pathPrefix(pp ...string) string {
 }
 
 // will return base URL with domain and prefix path
-func fullURL(pp ...string) string {
+func FullURL(pp ...string) string {
+	return fullUrl(GuessApiHostname())
+}
+
+func FullWebappURL(pp ...string) string {
+	return fullUrl(GuessWebappHostname())
+}
+
+// will return base URL with domain and prefix path
+func fullUrl(host string, pp ...string) string {
 	var (
 		full string
-		host = guessHostname()
 	)
 
 	if strings.Contains(host, "local.") || strings.Contains(host, "localhost") || !isSecure() {

pkg/options/options.gen.go

@@ -23,6 +23,8 @@ type (
 	}
 
 	HttpServerOpt struct {
+		Domain                 string `env:"DOMAIN"`
+		DomainWebapp           string `env:"DOMAIN_WEBAPP"`
 		Addr                   string `env:"HTTP_ADDR"`
 		LogRequest             bool   `env:"HTTP_LOG_REQUEST"`
 		LogResponse            bool   `env:"HTTP_LOG_RESPONSE"`
@@ -327,6 +329,8 @@ func HTTPClient() (o *HTTPClientOpt) {
 // This function is auto-generated
 func HttpServer() (o *HttpServerOpt) {
 	o = &HttpServerOpt{
+		Domain:                 "localhost",
+		DomainWebapp:           "localhost",
 		Addr:                   ":80",
 		EnableHealthcheckRoute: true,
 		EnableVersionRoute:     true,
@@ -518,12 +522,12 @@ func Auth() (o *AuthOpt) {
 		Secret:                   getSecretFromEnv("jwt secret"),
 		AccessTokenLifetime:      time.Hour * 2,
 		RefreshTokenLifetime:     time.Hour * 24 * 3,
-		ExternalRedirectURL:      fullURL("/auth/external/{provider}/callback"),
+		ExternalRedirectURL:      FullURL("/auth/external/{provider}/callback"),
 		ExternalCookieSecret:     getSecretFromEnv("external cookie secret"),
-		BaseURL:                  fullURL("/auth"),
+		BaseURL:                  FullURL("/auth"),
 		SessionCookieName:        "session",
 		SessionCookiePath:        pathPrefix("/auth"),
-		SessionCookieDomain:      guessHostname(),
+		SessionCookieDomain:      GuessApiHostname(),
 		SessionCookieSecure:      isSecure(),
 		SessionLifetime:          24 * time.Hour,
 		SessionPermLifetime:      360 * 24 * time.Hour,

pkg/valuestore/valuestore.go

@@ -0,0 +1,46 @@
+package valuestore
+
+import (
+	"strings"
+)
+
+type (
+	store struct {
+		env map[string]any
+	}
+)
+
+var (
+	gStore *store
+)
+
+func SetGlobal(s *store) {
+	gStore = s
+}
+
+func New() *store {
+	return &store{}
+}
+
+func Global() *store {
+	return gStore
+}
+
+func EnvGetter() func(string) any {
+	return gStore.Env
+}
+
+func (s *store) SetEnv(env map[string]any) {
+	if s.env != nil {
+		panic("cannot redefine environment variables")
+	}
+
+	s.env = env
+}
+
+func (s *store) Env(k string) (v any) {
+	if s.env == nil {
+		panic("valuestore env not initialized")
+	}
+	return s.env[strings.ToLower(k)]
+}

pkg/valuestore/valuestore_test.go

@@ -0,0 +1,41 @@
+package valuestore
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSettingEnv(t *testing.T) {
+	s := New()
+	s.SetEnv(map[string]any{
+		"k1": "v1",
+		"k2": "v2",
+	})
+
+	require.Equal(t, "v1", s.Env("k1"))
+	require.Equal(t, "v2", s.Env("k2"))
+}
+
+func TestPreventResettingEnv(t *testing.T) {
+	s := New()
+	s.SetEnv(map[string]any{
+		"k1": "v1",
+		"k2": "v2",
+	})
+
+	require.Panics(t, func() {
+		s.SetEnv(map[string]any{
+			"k3": "v3",
+			"k4": "v4",
+		})
+	})
+}
+
+func TestPreventUninitializedAccess(t *testing.T) {
+	s := New()
+
+	require.Panics(t, func() {
+		s.Env("k1")
+	})
+}

store/adapters/rdbms/aux_types.gen.go

@@ -7,8 +7,6 @@ package rdbms
 //
 
 import (
-	"time"
-
 	automationType "github.com/cortezaproject/corteza-server/automation/types"
 	composeType "github.com/cortezaproject/corteza-server/compose/types"
 	federationType "github.com/cortezaproject/corteza-server/federation/types"
@@ -19,6 +17,7 @@ import (
 	labelsType "github.com/cortezaproject/corteza-server/pkg/label/types"
 	rbacType "github.com/cortezaproject/corteza-server/pkg/rbac"
 	systemType "github.com/cortezaproject/corteza-server/system/types"
+	"time"
 )
 
 type (

system/automation/valuestore_handler.gen.go

@@ -0,0 +1,109 @@
+package automation
+
+// This file is auto-generated.
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+//
+// Definitions file that controls how this file is generated:
+// system/automation/valuestore_handler.yaml
+
+import (
+	"context"
+	atypes "github.com/cortezaproject/corteza-server/automation/types"
+	"github.com/cortezaproject/corteza-server/pkg/expr"
+	"github.com/cortezaproject/corteza-server/pkg/wfexec"
+)
+
+var _ wfexec.ExecResponse
+
+type (
+	valuestoreHandlerRegistry interface {
+		AddFunctions(ff ...*atypes.Function)
+		Type(ref string) expr.Type
+	}
+)
+
+func (h valuestoreHandler) register() {
+	h.reg.AddFunctions(
+		h.Env(),
+	)
+}
+
+type (
+	valuestoreEnvArgs struct {
+		hasKey bool
+		Key    string
+	}
+
+	valuestoreEnvResults struct {
+		Value interface{}
+	}
+)
+
+// Env function Get ENV variable
+//
+// expects implementation of env function:
+// func (h valuestoreHandler) env(ctx context.Context, args *valuestoreEnvArgs) (results *valuestoreEnvResults, err error) {
+//    return
+// }
+func (h valuestoreHandler) Env() *atypes.Function {
+	return &atypes.Function{
+		Ref:    "valuestoreEnv",
+		Kind:   "function",
+		Labels: map[string]string{"templates": "step,workflow"},
+		Meta: &atypes.FunctionMeta{
+			Short:       "Get ENV variable",
+			Description: "Get ENV variable for the specified key. If the key doesn't correspond to any value, nil is returned",
+		},
+
+		Parameters: []*atypes.Param{
+			{
+				Name:  "key",
+				Types: []string{"String"}, Required: true,
+			},
+		},
+
+		Results: []*atypes.Param{
+
+			{
+				Name:  "value",
+				Types: []string{"Any"},
+			},
+		},
+
+		Handler: func(ctx context.Context, in *expr.Vars) (out *expr.Vars, err error) {
+			var (
+				args = &valuestoreEnvArgs{
+					hasKey: in.Has("key"),
+				}
+			)
+
+			if err = in.Decode(args); err != nil {
+				return
+			}
+
+			var results *valuestoreEnvResults
+			if results, err = h.env(ctx, args); err != nil {
+				return
+			}
+
+			out = &expr.Vars{}
+
+			{
+				// converting results.Value (interface{}) to Any
+				var (
+					tval expr.TypedValue
+				)
+
+				if tval, err = h.reg.Type("Any").Cast(results.Value); err != nil {
+					return
+				} else if err = expr.Assign(out, "value", tval); err != nil {
+					return
+				}
+			}
+
+			return
+		},
+	}
+}

system/automation/valuestore_handler.go

@@ -0,0 +1,33 @@
+package automation
+
+import (
+	"context"
+)
+
+type (
+	envGetter interface {
+		Env(k string) (v any)
+	}
+
+	valuestoreHandler struct {
+		reg    valuestoreHandlerRegistry
+		getter envGetter
+	}
+)
+
+func ValuestoreHandler(reg valuestoreHandlerRegistry, getter envGetter) *valuestoreHandler {
+	h := &valuestoreHandler{
+		reg:    reg,
+		getter: getter,
+	}
+
+	h.register()
+	return h
+}
+
+func (h valuestoreHandler) env(ctx context.Context, args *valuestoreEnvArgs) (results *valuestoreEnvResults, err error) {
+	results = &valuestoreEnvResults{
+		Value: h.getter.Env(args.Key),
+	}
+	return
+}

system/automation/valuestore_handler.yaml

@@ -0,0 +1,20 @@
+snippets:
+
+labels: &labels
+  templates:  "step,workflow"
+
+functions:
+  env:
+    meta:
+      short: Get ENV variable
+      description: Get ENV variable for the specified key. If the key doesn't correspond to any value, nil is returned
+    params:
+      key: 
+        required: true
+        types:
+          - { wf: String }
+    labels:
+      <<: *labels
+    results:
+      value:
+        wf: Any

system/renderer/funcs.go

@@ -0,0 +1,9 @@
+package renderer
+
+import "github.com/cortezaproject/corteza-server/pkg/valuestore"
+
+func envGetter() func(k string) any {
+	return func(k string) any {
+		return valuestore.Global().Env(k)
+	}
+}

system/renderer/html.go

@@ -50,6 +50,7 @@ func preprocHTMLTemplate(pl *driverPayload) (*template.Template, error) {
 				raw := base64.RawStdEncoding.EncodeToString(bb)
 				return template.URL("data:" + rsp.Header.Get("Content-Type") + ";base64," + raw), nil
 			},
+			"env": envGetter(),
 		})
 
 	// Prep the original template

system/renderer/text.go

@@ -15,7 +15,10 @@ func preprocPlainTemplate(tpl io.Reader, pp map[string]io.Reader) (*template.Tem
 	}
 
 	gtpl := template.New("text/plain_render").
-		Funcs(sprig.TxtFuncMap())
+		Funcs(sprig.TxtFuncMap()).
+		Funcs(template.FuncMap{
+			"env": envGetter(),
+		})
 
 	// Prep the original template
 	t, err := gtpl.Parse(string(bb))

system/service/service.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/cortezaproject/corteza-server/pkg/dal"
 	"github.com/cortezaproject/corteza-server/pkg/discovery"
+	"github.com/cortezaproject/corteza-server/pkg/valuestore"
 
 	automationService "github.com/cortezaproject/corteza-server/automation/service"
 	"github.com/cortezaproject/corteza-server/pkg/actionlog"
@@ -255,6 +256,13 @@ func Initialize(ctx context.Context, log *zap.Logger, s store.Storer, ws websock
 		DefaultRole,
 	)
 
+	// ValuestoreHandler isn't (yet) a system thing but this initialization resides
+	// here just so we can easily register it
+	automation.ValuestoreHandler(
+		automationService.Registry(),
+		valuestore.Global(),
+	)
+
 	if c.ActionLog.WorkflowFunctionsEnabled {
 		// register action-log functions & types only when enabled
 		automation.ActionlogHandler(