Refactor lang loading, support sorting, enabling

app/boot_levels.go

@@ -94,14 +94,13 @@ func (app *CortezaApp) Setup() (err error) {
 	{
 		var (
 			localeLog = zap.NewNop()
-			languages *locale.Languages
 		)
 
 		if app.Opt.Locale.Log {
 			localeLog = app.Log
 		}
 
-		if languages, err = locale.New(localeLog, app.Opt.Locale); err != nil {
+		if languages, err := locale.Service(localeLog, app.Opt.Locale); err != nil {
 			return err
 		} else {
 			locale.SetGlobal(languages)

auth/handlers/handler.go

@@ -230,14 +230,14 @@ func (h *AuthHandlers) handle(fn handlerFn) http.HandlerFunc {
 			// because we need request's context to detect the language from!
 			ttf = func(t *template.Template) *template.Template {
 				return t.Funcs(map[string]interface{}{
-					"language": func() string { return loc.Current(req.Context()).String() },
+					"language": func() string { return locale.GetLanguageFromContext(req.Context()).String() },
 					"tr": func(key string, pp ...interface{}) template.HTML {
 						ss := make([]string, len(pp))
 						for i := range pp {
 							ss[i] = cast.ToString(pp[i])
 						}
 
-						return template.HTML(loc.Get(req.Context(), "auth", key, ss...))
+						return template.HTML(loc.T(req.Context(), "auth", key, ss...))
 					},
 				})
 			}
@@ -401,5 +401,5 @@ func anonyOnly(fn handlerFn) handlerFn {
 }
 
 func translator(req *request.AuthReq, ns string) func(key string, rr ...string) string {
-	return (&locale.Languages{}).GetNS(req.Context(), ns)
+	return locale.Global().NS(req.Context(), ns)
 }

pkg/errors/http.go

@@ -12,6 +12,12 @@ import (
 	"github.com/cortezaproject/corteza-server/pkg/locale"
 )
 
+type (
+	translatable interface {
+		Translate(tr func(string, string, ...string) string) error
+	}
+)
+
 // ServeHTTP Prepares and encodes given error for HTTP transport
 //
 // mask arg hides extra/debug info
@@ -118,11 +124,13 @@ func writeHttpJSON(ctx context.Context, w io.Writer, err error, mask bool) {
 		err = fmt.Errorf(err.Error())
 	}
 
-	if tr, is := err.(interface {
-		Translate(tr func(string, string, ...string) string) error
-	}); is {
+	// if error is translatable, pass in the lambda that returns
+	// translated error
+	//
+	// we're using global locale service directly for now
+	if tr, is := err.(translatable); is {
 		err = tr.Translate(func(ns string, key string, pairs ...string) string {
-			return locale.Global().Get(ctx, ns, key, pairs...)
+			return locale.Global().T(ctx, ns, key, pairs...)
 		})
 	}
 

pkg/locale/context.go

@@ -15,6 +15,6 @@ func GetLanguageFromContext(ctx context.Context) language.Tag {
 	if tag, ok := ctx.Value(language.Tag{}).(language.Tag); ok {
 		return tag
 	} else {
-		return defaultLanguage
+		return language.Tag{}
 	}
 }

pkg/locale/http.go

@@ -9,70 +9,86 @@ import (
 
 const AcceptLanguageHeader = "Accept-Language"
 
-func DetectLanguage(ll *Languages) func(next http.Handler) http.Handler {
+func DetectLanguage(ll *service) func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			if ll.opt.DevelopmentMode {
-				if err := ll.Reload(); err != nil {
-					// when in development mode, refresh languages for every request
-					ll.log.Error("failed to load locales", zap.Error(err))
-				}
-			}
-
-			var (
-				rawLanguageTag string
-			)
-
-			// try to detect the language from the request's query string:
-			if ll.opt.QueryStringParam != "" {
-				rawLanguageTag = r.URL.Query().Get(ll.opt.QueryStringParam)
-			}
-
-			// try to detect the language from the request's headers:
-			if len(rawLanguageTag) == 0 {
-				rawLanguageTag = r.Header.Get(AcceptLanguageHeader)
-			}
-
-			if len(rawLanguageTag) > 0 {
-				// parse & ignore errors
-				var (
-					preferred = ll.Default()
-					supported = ll.Tags()
-
-					accepted, _, err = language.ParseAcceptLanguage(rawLanguageTag)
-				)
-
-				if err == nil {
-					// ignoring index & confidence
-					preferred, _, _ = language.NewMatcher(supported).Match(accepted...)
-
-					var match bool
-					for _, s := range supported {
-						if s == preferred {
-							match = true
-							break
-						}
-					}
-
-					if !match {
-						base, _ := preferred.Base()
-						preferred = language.MustParse(base.String())
-					}
-				}
-
-				// new request with new context
-				r = r.WithContext(SetLanguageToContext(r.Context(), preferred))
-
-				if ll.opt.DevelopmentMode {
-					ll.log.Debug(
-						"language detected",
-						zap.String("preferred", preferred.String()),
-						zap.String("raw", rawLanguageTag),
-					)
-				}
-			}
-
-			next.ServeHTTP(w, r)
+			next.ServeHTTP(w, upgradeRequest(ll, r))
 		})
 	}
 }
+
+func upgradeRequest(ll *service, r *http.Request) *http.Request {
+	return r.WithContext(SetLanguageToContext(r.Context(), detectLanguage(ll, r)))
+}
+
+func detectLanguage(ll *service, r *http.Request) (tag language.Tag) {
+	if ll.opt.DevelopmentMode {
+		if err := ll.Reload(); err != nil {
+			// when in development mode, refresh languages for every request
+			ll.log.Error("failed to load locales", zap.Error(err))
+			return
+		}
+	}
+
+	if ll.Default() == nil {
+		// locale service does not have anything loaded...
+		return
+	}
+
+	var (
+		raw string
+	)
+
+	// try to detect the language from the request's query string:
+	if ll.opt.QueryStringParam != "" {
+		raw = r.URL.Query().Get(ll.opt.QueryStringParam)
+	}
+
+	// try to detect the language from the request's headers:
+	if len(raw) == 0 {
+		raw = r.Header.Get(AcceptLanguageHeader)
+	}
+
+	if len(raw) == 0 {
+		// no need for lang detection
+		return
+	}
+
+	// parse & ignore errors
+	var (
+		preferred = ll.Default().Tag
+		supported = ll.Tags()
+
+		accepted, _, err = language.ParseAcceptLanguage(raw)
+	)
+
+	if err == nil {
+		// ignoring index & confidence
+		preferred, _, _ = language.NewMatcher(supported).Match(accepted...)
+
+		var match bool
+		for _, s := range supported {
+			if s == preferred {
+				match = true
+				break
+			}
+		}
+
+		if !match {
+			base, _ := preferred.Base()
+			preferred = language.MustParse(base.String())
+		}
+	}
+
+	if ll.opt.DevelopmentMode {
+		ll.log.Debug(
+			"language detected",
+			zap.String("preferred", preferred.String()),
+			zap.String("raw", raw),
+		)
+	}
+
+	// new request with new context
+	return preferred
+
+}

pkg/locale/load.go

@@ -11,7 +11,6 @@ import (
 	"path/filepath"
 	"strings"
 
-	"go.uber.org/zap"
 	"golang.org/x/text/language"
 	"gopkg.in/yaml.v3"
 )
@@ -20,7 +19,8 @@ const serverApplication = "corteza-server"
 
 var defaultLanguage = language.English
 
-func load(log *zap.Logger, pp ...string) (ll []*Language, err error) {
+// Load all configs that are reachable in the given paths
+func loadConfigs(pp ...string) (ll []*Language, err error) {
 	var (
 		pattern string
 		f       *os.File
@@ -39,9 +39,8 @@ func load(log *zap.Logger, pp ...string) (ll []*Language, err error) {
 			tag := language.Make(filepath.Base(filepath.Dir(c)))
 
 			lang := &Language{
-				Tag:      tag,
-				internal: make(internal),
-				external: make(external),
+				Tag: tag,
+				src: filepath.Dir(c),
 			}
 
 			err = func() error {
@@ -51,23 +50,19 @@ func load(log *zap.Logger, pp ...string) (ll []*Language, err error) {
 				}
 
 				defer f.Close()
-
 				if err = yaml.NewDecoder(f).Decode(&lang); err != nil {
 					return fmt.Errorf("could not decode %s: %v", p, err)
 				}
 
-				return loadTranslations(lang, filepath.Dir(c))
+				return nil
 			}()
 
 			if err != nil {
 				return nil, err
 			}
 
-			log.Info("language loaded", zap.String("tag", lang.Tag.String()), zap.String("config", c))
-
 			ll = append(ll, lang)
 		}
-
 	}
 
 	return

pkg/locale/locale.go

@@ -1,15 +1,11 @@
 package locale
 
 import (
-	"context"
 	"fmt"
 	"io"
-	"sort"
 	"strings"
 	"sync"
 
-	"github.com/cortezaproject/corteza-server/pkg/options"
-	"go.uber.org/zap"
 	"golang.org/x/text/language"
 )
 
@@ -26,13 +22,19 @@ type (
 	Language struct {
 		l sync.RWMutex
 
+		// location of the language files
+		src string
+
 		Tag  language.Tag
 		Name string
 
+		// Language (tag) that this language is extending
 		// @todo
-		Extends string
+		Extends language.Tag
+
+		// Points to extended language
 		// @todo
-		base *Language
+		extends *Language
 
 		// Internal translations (application == corteza-server)
 		internal internal
@@ -41,135 +43,11 @@ type (
 		external external
 	}
 
-	Languages struct {
-		l sync.RWMutex
-
-		log *zap.Logger
-
-		opt options.LocaleOpt
-
-		// sources
-		src []string
-		ll  map[language.Tag]*Language
-
-		def language.Tag
-	}
-
 	ErrorMetaNamespace struct{}
 	ErrorMetaKey       struct{}
 )
 
-func New(log *zap.Logger, opt options.LocaleOpt) (*Languages, error) {
-	ll := &Languages{
-		opt: opt,
-		src: strings.Split(opt.Path, ":"),
-		log: log.Named("locale"),
-	}
-	return ll, ll.Reload()
-}
-
-func (set *Languages) Default() language.Tag {
-	return set.def
-}
-
-func (set *Languages) Tags() (tt []language.Tag) {
-	tt = make([]language.Tag, 0, len(set.ll))
-	for t := range set.ll {
-		tt = append(tt, t)
-	}
-
-	return
-}
-
-func (set *Languages) Reload() error {
-	set.l.RLock()
-	defer set.l.RUnlock()
-
-	set.log.Info("reloading", zap.Strings("src", set.src))
-
-	set.ll = make(map[language.Tag]*Language)
-	ll, err := load(set.log, set.src...)
-	if err != nil {
-		return err
-	}
-
-	for i, l := range ll {
-		if i == 0 {
-			// set first one as default
-			set.def = l.Tag
-		}
-
-		set.ll[l.Tag] = l
-	}
-
-	return nil
-}
-
-// List returns list of all languages
-func (set *Languages) List() []*Language {
-	set.l.RLock()
-	defer set.l.RUnlock()
-
-	ll := make([]*Language, 0, len(set.ll))
-	for _, l := range set.ll {
-		ll = append(ll, &Language{
-			Tag:  l.Tag,
-			Name: l.Name,
-		})
-	}
-
-	sort.SliceStable(ll, func(i, j int) bool {
-		return strings.Compare(ll[i].Name, ll[j].Name) < 0
-	})
-
-	return ll
-}
-
-func (set *Languages) EncodeExternal(w io.Writer, lang language.Tag, app string) error {
-	set.l.RLock()
-	defer set.l.RUnlock()
-
-	if set.ll[lang] == nil {
-		return fmt.Errorf("language %q not found", lang)
-	}
-
-	if set.ll[lang].external[app] == nil {
-		return fmt.Errorf("application %q not found", app)
-	}
-
-	_, err := io.Copy(w, set.ll[lang].external[app])
-	return err
-}
-
-func (set *Languages) GetNS(ctx context.Context, ns string) func(key string, rr ...string) string {
-	var (
-		code = GetLanguageFromContext(ctx)
-	)
-
-	return func(key string, rr ...string) string {
-		return set.get(code, ns, key, rr...)
-	}
-}
-
-func (set *Languages) Current(ctx context.Context) language.Tag {
-	return GetLanguageFromContext(ctx)
-}
-
-func (set *Languages) Get(ctx context.Context, ns, key string, rr ...string) string {
-	return set.get(GetLanguageFromContext(ctx), ns, key, rr...)
-}
-
-func (set *Languages) get(code language.Tag, ns, key string, rr ...string) string {
-	if set != nil && set.ll != nil {
-		if l, has := set.ll[code]; has {
-			return l.get(ns, key, rr...)
-		}
-	}
-
-	return key
-}
-
-func (l *Language) get(ns, key string, rr ...string) string {
+func (l *Language) t(ns, key string, rr ...string) string {
 	l.l.RLock()
 	defer l.l.RUnlock()
 
@@ -182,8 +60,8 @@ func (l *Language) get(ns, key string, rr ...string) string {
 		return strings.NewReplacer(rr...).Replace(msg)
 	}
 
-	if l.base != nil {
-		return l.base.get(ns, key, rr...)
+	if l.extends != nil {
+		return l.extends.t(ns, key, rr...)
 	}
 
 	return key

pkg/locale/service.go

@@ -1,16 +1,221 @@
 package locale
 
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+
+	"github.com/cortezaproject/corteza-server/pkg/options"
+	"go.uber.org/zap"
+	"golang.org/x/text/language"
+)
+
+type (
+	service struct {
+		l sync.RWMutex
+
+		// logger facility for the locale service
+		log *zap.Logger
+
+		// options
+		opt options.LocaleOpt
+
+		// configured sources where we can load languages from
+		src []string
+
+		// all configured tags
+		tags []language.Tag
+
+		// set of all known languages
+		set map[language.Tag]*Language
+
+		// default language
+		def *Language
+	}
+)
+
 var (
 	// Global locales service
-	global *Languages
+	global *service
 )
 
 // Global returns global RBAC service
-func Global() *Languages {
+func Global() *service {
 	return global
 }
 
 // SetGlobal re-sets global service
-func SetGlobal(ll *Languages) {
+func SetGlobal(ll *service) {
 	global = ll
 }
+
+func Service(log *zap.Logger, opt options.LocaleOpt) (*service, error) {
+	svc := &service{
+		opt: opt,
+		src: strings.Split(opt.Path, ":"),
+		log: log.Named("locale"),
+	}
+
+	for _, lang := range strings.Split(opt.Languages, ",") {
+		lang = strings.TrimSpace(lang)
+		tag := language.Make(lang)
+		if tag.IsRoot() {
+			return nil, fmt.Errorf("failed to parse language '%s'", lang)
+		}
+
+		svc.tags = append(svc.tags, tag)
+	}
+
+	return svc, svc.Reload()
+}
+
+// Default language
+func (svc *service) Default() *Language {
+	return svc.def
+}
+
+// Tags of all loaded languages
+func (svc *service) Tags() (tt []language.Tag) {
+	tt = make([]language.Tag, 0, len(svc.set))
+	for t := range svc.set {
+		tt = append(tt, t)
+	}
+
+	return
+}
+
+// Reload all language configurations (as configured via path options) and
+// all translation files
+func (svc *service) Reload() error {
+	svc.l.RLock()
+	defer svc.l.RUnlock()
+
+	svc.log.Info("reloading",
+		zap.Strings("path", svc.src),
+		zap.Strings("tags", tagsToStrings(svc.tags)),
+	)
+
+	svc.set = make(map[language.Tag]*Language)
+	configs, err := loadConfigs(svc.src...)
+	if err != nil {
+		return err
+	}
+
+	for i, lang := range configs {
+		if !hasTag(lang.Tag, svc.tags) {
+			svc.log.Info(
+				"language skipped (see LOCALE_LANGUAGES and LOCALE_PATH)",
+				zap.Stringer("tag", lang.Tag),
+			)
+
+			continue
+		}
+
+		if err = loadTranslations(lang, lang.src); err != nil {
+			return err
+		}
+
+		svc.log.Info(
+			"language loaded",
+			zap.Stringer("tag", lang.Tag),
+			zap.String("src", lang.src),
+			zap.Stringer("extends", lang.Extends),
+		)
+
+		if i == 0 && svc.def == nil {
+			// set first one as default
+			svc.def = lang
+		}
+
+		svc.set[lang.Tag] = lang
+	}
+
+	return nil
+}
+
+// List returns list of all languages
+func (svc *service) List() []*Language {
+	svc.l.RLock()
+	defer svc.l.RUnlock()
+
+	ll := make([]*Language, 0, len(svc.set))
+	for _, tag := range svc.tags {
+		ll = append(ll, svc.set[tag])
+	}
+
+	return ll
+}
+
+func (svc *service) HasLanguage(lang language.Tag) bool {
+	return svc.set[lang] != nil
+}
+
+func (svc *service) HasApplication(lang language.Tag, app string) bool {
+	return svc.set[lang] != nil && svc.set[lang].external[app] != nil
+}
+
+// EncodeExternal writes into
+func (svc *service) EncodeExternal(w io.Writer, lang language.Tag, app string) (err error) {
+	svc.l.RLock()
+	defer svc.l.RUnlock()
+
+	if svc.HasApplication(lang, app) {
+		_, err = io.Copy(w, svc.set[lang].external[app])
+	} else {
+		err = fmt.Errorf("application or language missing")
+	}
+
+	return err
+}
+
+// NS returns translator function for the current language (from context) and a specific namespace
+//
+// Language is picked from the context
+func (svc *service) NS(ctx context.Context, ns string) func(key string, rr ...string) string {
+	var (
+		code = GetLanguageFromContext(ctx)
+	)
+
+	return func(key string, rr ...string) string {
+		return svc.t(code, ns, key, rr...)
+	}
+}
+
+// T returns translated key from namespaces using list of replacement pairs
+//
+// Language is picked from the context
+func (svc *service) T(ctx context.Context, ns, key string, rr ...string) string {
+	return svc.t(GetLanguageFromContext(ctx), ns, key, rr...)
+}
+
+// Finds language and uses it to translate the given key
+func (svc *service) t(code language.Tag, ns, key string, rr ...string) string {
+	if svc != nil && svc.set != nil {
+		if l, has := svc.set[code]; has {
+			return l.t(ns, key, rr...)
+		}
+	}
+
+	return key
+}
+
+func hasTag(t language.Tag, tt []language.Tag) bool {
+	for _, tag := range tt {
+		if tag.String() == t.String() {
+			return true
+		}
+	}
+
+	return false
+}
+
+func tagsToStrings(tt []language.Tag) []string {
+	out := make([]string, len(tt))
+	for t := range tt {
+		out[t] = tt[t].String()
+	}
+
+	return out
+}

pkg/options/locale.gen.go

@@ -10,6 +10,7 @@ package options
 
 type (
 	LocaleOpt struct {
+		Languages        string `env:"LOCALE_LANGUAGES"`
 		Path             string `env:"LOCALE_PATH"`
 		QueryStringParam string `env:"LOCALE_QUERY_STRING_PARAM"`
 		Log              bool   `env:"LOCALE_LOG"`
@@ -20,6 +21,7 @@ type (
 // Locale initializes and returns a LocaleOpt with default values
 func Locale() (o *LocaleOpt) {
 	o = &LocaleOpt{
+		Languages:        "en",
 		QueryStringParam: "lng",
 	}
 

pkg/options/locale.yaml

@@ -1,6 +1,14 @@
 props:
+  - name: languages
+    default: "en"
+    description: |
+      List of compa delimited languages (language tags) to enable.
+      In case when an enabled language can not be loaded, error is logged.
+
+      When loading language configurations (config.xml) from the configured path(s).
+
   - name: path
-    description: Path to locale config and translation files, separated by colon
+    description: One or more paths to locale config and translation files, separated by colon
 
   - name: queryStringParam
     description: |

system/rest/locales.go

@@ -44,11 +44,31 @@ func (ctrl Locale) List(ctx context.Context, r *request.LocaleList) (interface{}
 }
 
 func (ctrl Locale) Get(ctx context.Context, r *request.LocaleGet) (interface{}, error) {
+	svc := locale.Global()
+
 	return func(w http.ResponseWriter, req *http.Request) {
-		w.Header().Set("Content-Type", "application/json;charset=UTF-8")
-		err := locale.Global().EncodeExternal(w, language.Make(r.Lang), r.Application)
-		if err != nil {
-			errors.ProperlyServeHTTP(w, req, err, false)
+		if !svc.HasLanguage(language.Make(r.Lang)) {
+			errors.ProperlyServeHTTP(w, req, errors.New(
+				errors.KindNotFound,
+				"no such language",
+				errors.StackTrimAtFn("http.HandlerFunc.ServeHTTP"),
+			), true)
+			return
 		}
+
+		if !svc.HasApplication(language.Make(r.Lang), r.Application) {
+			errors.ProperlyServeHTTP(w, req, errors.New(
+				errors.KindNotFound,
+				"no such application",
+				errors.StackTrimAtFn("http.HandlerFunc.ServeHTTP"),
+			), true)
+			return
+		}
+
+		if err := locale.Global().EncodeExternal(w, language.Make(r.Lang), r.Application); err != nil {
+			errors.ProperlyServeHTTP(w, req, err, true)
+		}
+
+		w.Header().Set("Content-Type", "application/json;charset=UTF-8")
 	}, nil
 }