DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Support setting values within Envoy

Browse Source
This commit is contained in:
Tomaž Jerman
2023-03-31 14:56:02 +02:00
parent 8cf8161e50
commit 2c42ec631e
17 changed files with 319 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
server/auth/external/register.go vendored
View File

@@ -149,7 +149,7 @@ func RegisterOidcProvider(ctx context.Context, log *zap.Logger, s store.SettingV
if enable {
err = vv.Walk(func(value *types.SettingValue) error {
if strings.HasSuffix(value.Name, ".enabled") {
return value.SetValue(true)
return value.SetSetting(true)
}
return nil
@@ -160,7 +160,7 @@ func RegisterOidcProvider(ctx context.Context, log *zap.Logger, s store.SettingV
}
v := &types.SettingValue{Name: "auth.external.enabled"}
err = v.SetValue(true)
err = v.SetSetting(true)
if err != nil {
return
}

7
server/automation/envoy/yaml_decode.gen.go generated
View File

@@ -134,6 +134,13 @@ func (d *auxYamlDoc) UnmarshalYAML(n *yaml.Node) (err error) {
}
return err
// Offload to custom handlers
default:
aux, err = d.unmarshalYAML(kv, v)
d.nodes = append(d.nodes, aux...)
if err != nil {
err = errors.Wrap(err, "failed to unmarshal node")
}
}
return nil
})

2
server/automation/envoy/yaml_decode.go
View File

@@ -8,3 +8,5 @@ import (
func (d *auxYamlDoc) unmarshalTriggersExtendedNode(dctx documentContext, n *yaml.Node, meta ...*yaml.Node) (out envoyx.NodeSet, err error) {
return d.unmarshalTriggerNode(dctx, n, meta...)
}
func (d *auxYamlDoc) unmarshalYAML(k string, n *yaml.Node) (out envoyx.NodeSet, err error) { return }

7
server/compose/envoy/yaml_decode.gen.go generated
View File

@@ -181,6 +181,13 @@ func (d *auxYamlDoc) UnmarshalYAML(n *yaml.Node) (err error) {
}
return err
// Offload to custom handlers
default:
aux, err = d.unmarshalYAML(kv, v)
d.nodes = append(d.nodes, aux...)
if err != nil {
err = errors.Wrap(err, "failed to unmarshal node")
}
}
return nil
})

2
server/compose/envoy/yaml_decode.go
View File

@@ -42,6 +42,8 @@ const (
ComposeRecordDatasourceAuxType = "corteza::compose:record-datasource"
)
func (d *auxYamlDoc) unmarshalYAML(k string, n *yaml.Node) (out envoyx.NodeSet, err error) { return }
func (d *auxYamlDoc) unmarshalChartConfigNode(r *types.Chart, n *yaml.Node) (refs map[string]envoyx.Ref, idents envoyx.Identifiers, err error) {
err = y7s.EachMap(n, func(k, v *yaml.Node) error {
if k.Value != "reports" {

2
server/pkg/envoy/resource/settings.go
View File

@@ -25,7 +25,7 @@ func NewSettings(vv map[string]interface{}) *Settings {
sv := &types.SettingValue{
Name: k,
}
sv.SetValue(v)
sv.SetSetting(v)
r.Res = append(r.Res, sv)
}

8
server/system/commands/settings.go
View File

@@ -87,12 +87,12 @@ func Settings(ctx context.Context, app serviceInitializer) *cobra.Command {
}
if cmd.Flags().Lookup("as-string").Changed {
cli.HandleError(v.SetValue(value))
cli.HandleError(v.SetSetting(value))
} else {
err := v.SetRawValue(value)
err := v.SetRawSetting(value)
if _, is := err.(*json.SyntaxError); is {
// Quote the raw value and re-parse
err = v.SetRawValue(`"` + value + `"`)
err = v.SetRawSetting(`"` + value + `"`)
}
cli.HandleError(err)
@@ -136,7 +136,7 @@ func Settings(ctx context.Context, app serviceInitializer) *cobra.Command {
for k, v := range input {
val := &types.SettingValue{Name: k}
cli.HandleError(val.SetValue(v))
cli.HandleError(val.SetSetting(v))
vv = append(vv, val)
}

37
server/system/envoy/store_decode.go
View File

@@ -11,6 +11,43 @@ import (
)
func (d StoreDecoder) extendDecoder(ctx context.Context, s store.Storer, dl dal.FullService, rt string, nodes map[string]*envoyx.Node, f envoyx.ResourceFilter) (out envoyx.NodeSet, err error) {
switch rt {
case types.SettingValueResourceType:
return d.decodeSettingValue(ctx, s, dl, types.SettingsFilter{})
}
return
}
func (d StoreDecoder) decodeSettingValue(ctx context.Context, s store.Storer, dl dal.FullService, f types.SettingsFilter) (out envoyx.NodeSet, err error) {
// @todo this might need to be improved.
// Currently, no resource is vast enough to pose a problem.
rr, _, err := store.SearchSettingValues(ctx, s, f)
if err != nil {
return
}
for _, r := range rr {
var n *envoyx.Node
n, err = SettingValueToEnvoyNode(r)
if err != nil {
return
}
out = append(out, n)
}
return
}
func SettingValueToEnvoyNode(r *types.SettingValue) (node *envoyx.Node, err error) {
// SettingValues don't have references so it can be omitted
node = &envoyx.Node{
Resource: r,
ResourceType: types.SettingValueResourceType,
Identifiers: envoyx.MakeIdentifiers(
r.Name,
),
}
return
}

4
server/system/envoy/store_encode.go
View File

@@ -16,6 +16,8 @@ func (e StoreEncoder) prepare(ctx context.Context, p envoyx.EncodeParams, s stor
return e.prepareRbacRule(ctx, p, s, nn)
case types.ResourceTranslationResourceType:
return e.prepareResourceTranslation(ctx, p, s, nn)
case types.SettingValueResourceType:
return e.prepareSetting(ctx, p, s, nn)
}
return
@@ -27,6 +29,8 @@ func (e StoreEncoder) encode(ctx context.Context, p envoyx.EncodeParams, s store
return e.encodeRbacRules(ctx, p, s, nn, tree)
case types.ResourceTranslationResourceType:
return e.encodeResourceTranslations(ctx, p, s, nn, tree)
case types.SettingValueResourceType:
return e.encodeSettings(ctx, p, s, nn, tree)
}
return

63
server/system/envoy/store_encode_setting.go Normal file
View File

@@ -0,0 +1,63 @@
package envoy
import (
"context"
"github.com/cortezaproject/corteza/server/pkg/envoyx"
"github.com/cortezaproject/corteza/server/store"
"github.com/cortezaproject/corteza/server/system/types"
systemTypes "github.com/cortezaproject/corteza/server/system/types"
)
func (e StoreEncoder) prepareSetting(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet) (err error) {
// @todo existing settings?
for _, n := range nn {
if n.Resource == nil {
panic("unexpected state: cannot call prepareSetting with nodes without a defined Resource")
}
res, ok := n.Resource.(*types.SettingValue)
if !ok {
panic("unexpected resource type: node expecting type of SettingValue")
}
// Run expressions on the nodes
err = e.runEvals(ctx, false, n)
if err != nil {
return
}
// @todo merge conflicts if we do existing assertion
n.Resource = res
}
return
}
// encodeSettings encodes a set of resource into the database
func (e StoreEncoder) encodeSettings(ctx context.Context, p envoyx.EncodeParams, s store.Storer, nn envoyx.NodeSet, tree envoyx.Traverser) (err error) {
for _, n := range nn {
err = e.encodeSetting(ctx, p, s, n, tree)
if err != nil {
return
}
}
return
}
// encodeSetting encodes the resource into the database
func (e StoreEncoder) encodeSetting(ctx context.Context, p envoyx.EncodeParams, s store.Storer, n *envoyx.Node, tree envoyx.Traverser) (err error) {
// SettingValues don't have any references so there is no need to handle them
// Flush to the DB
if !n.Evaluated.Skip {
err = store.UpsertSettingValue(ctx, s, n.Resource.(*systemTypes.SettingValue))
if err != nil {
return
}
}
return
}

7
server/system/envoy/yaml_decode.gen.go generated
View File

@@ -292,6 +292,13 @@ func (d *auxYamlDoc) UnmarshalYAML(n *yaml.Node) (err error) {
err = errors.Wrap(err, "failed to unmarshal node: locale")
}
// Offload to custom handlers
default:
aux, err = d.unmarshalYAML(kv, v)
d.nodes = append(d.nodes, aux...)
if err != nil {
err = errors.Wrap(err, "failed to unmarshal node")
}
}
return nil
})

129
server/system/envoy/yaml_decode.go
View File

@@ -1,15 +1,144 @@
package envoy
import (
"encoding/json"
"fmt"
"strings"
"github.com/cortezaproject/corteza/server/pkg/envoyx"
"github.com/cortezaproject/corteza/server/pkg/y7s"
"github.com/cortezaproject/corteza/server/system/types"
systemTypes "github.com/cortezaproject/corteza/server/system/types"
sqlt "github.com/jmoiron/sqlx/types"
"golang.org/x/text/language"
"gopkg.in/yaml.v3"
)
func (d *auxYamlDoc) unmarshalYAML(k string, n *yaml.Node) (out envoyx.NodeSet, err error) {
switch k {
case "settings", "setting":
out, err = d.unmarshalSettingsNode(n)
}
return
}
func (d *auxYamlDoc) unmarshalSettingsNode(n *yaml.Node) (out envoyx.NodeSet, err error) {
out = make(envoyx.NodeSet, 0, len(n.Content))
err = y7s.Each(n, func(k, v *yaml.Node) (err error) {
if v == nil {
return y7s.NodeErr(n, "malformed setting definition")
}
s := &types.SettingValue{}
switch v.Kind {
case yaml.MappingNode:
if err = v.Decode(&s); err != nil {
return
}
default:
if err = y7s.DecodeScalar(k, "setting name", &s.Name); err != nil {
return err
}
if y7s.IsKind(v, yaml.SequenceNode) {
aux := make([]interface{}, 0, 10)
y7s.EachSeq(v, func(n *yaml.Node) error {
var vx interface{}
err := y7s.DecodeScalar(n, "setting value", &vx)
if err != nil {
return err
}
aux = append(aux, vx)
return nil
})
m, err := json.Marshal(aux)
if err != nil {
return err
}
s.Value = sqlt.JSONText(m)
} else if y7s.IsKind(v, yaml.MappingNode) {
aux := make(map[string]interface{})
y7s.EachMap(v, func(k, v *yaml.Node) error {
var vx interface{}
err := y7s.DecodeScalar(v, "setting value", &vx)
if err != nil {
return err
}
aux[k.Value] = vx
return nil
})
m, err := json.Marshal(aux)
if err != nil {
return err
}
s.Value = sqlt.JSONText(m)
} else {
var aux interface{}
err = y7s.DecodeScalar(v, "setting value", &aux)
if err != nil {
return err
}
m, err := json.Marshal(aux)
if err != nil {
return err
}
s.Value = sqlt.JSONText(m)
}
}
out = append(out, &envoyx.Node{
Resource: s,
ResourceType: types.SettingValueResourceType,
})
return
})
err = y7s.EachMap(n, func(lang, loc *yaml.Node) error {
langTag := systemTypes.Lang{Tag: language.Make(lang.Value)}
return y7s.EachMap(loc, func(res, kv *yaml.Node) error {
return y7s.EachMap(kv, func(k, msg *yaml.Node) error {
out = append(out, &envoyx.Node{
Resource: &systemTypes.ResourceTranslation{
Resource: res.Value,
Lang: langTag,
K: k.Value,
Message: msg.Value,
},
// Providing resource type as plain text to reduce cross component references
ResourceType: "corteza::system:resource-translation",
References: envoyx.SplitResourceIdentifier(res.Value),
})
return nil
})
})
})
if err != nil {
return
}
for _, o := range out {
for _, r := range o.References {
if r.Scope.IsEmpty() {
continue
}
o.Scope = r.Scope
break
}
}
return
}
func (d *auxYamlDoc) unmarshalFiltersExtendedNode(dctx documentContext, n *yaml.Node, meta ...*yaml.Node) (out envoyx.NodeSet, err error) {
return d.unmarshalApigwFilterNode(dctx, n, meta...)
}

16
server/system/envoy/yaml_encode.go
View File

@@ -33,6 +33,8 @@ func (e YamlEncoder) encode(ctx context.Context, base *yaml.Node, p envoyx.Encod
return e.encodeRbacRules(ctx, base, p, nodes, tt)
case types.ResourceTranslationResourceType:
return e.encodeResourceTranslations(ctx, base, p, nodes, tt)
case types.SettingValueResourceType:
return e.encodeSettingValues(ctx, base, p, nodes, tt)
}
return
@@ -154,6 +156,20 @@ func (e YamlEncoder) encodeRbacRules(ctx context.Context, base *yaml.Node, p env
return
}
func (e YamlEncoder) encodeSettingValues(ctx context.Context, base *yaml.Node, p envoyx.EncodeParams, nodes envoyx.NodeSet, tt envoyx.Traverser) (out *yaml.Node, err error) {
// Setting values don't have any refs
for _, n := range nodes {
sv := n.Resource.(*types.SettingValue)
base, err = y7s.AddMap(base, sv.Name, sv.Value)
if err != nil {
return
}
}
return y7s.MakeMap("settings", base)
}
func (e YamlEncoder) encodeRbacRulesByRole(p envoyx.EncodeParams, rules []rbacRuleWrap, tt envoyx.Traverser) (out *yaml.Node, err error) {
// Batch by role; make sure to keep the multiple identifier thing in mind
byRole := make(map[string]rbacRuleRoleWrap)

2
server/system/rest/settings.go
View File

@@ -85,7 +85,7 @@ func (ctrl *Settings) Set(ctx context.Context, r *request.SettingsSet) (interfac
s := &types.SettingValue{Name: r.Key, OwnedBy: r.OwnerID}
if err = s.SetValue(fmt.Sprintf("attachment:%d", att.ID)); err != nil {
if err = s.SetSetting(fmt.Sprintf("attachment:%d", att.ID)); err != nil {
return nil, err
}

2
server/system/types/app_settings.go
View File

@@ -603,7 +603,7 @@ func (eap ExternalAuthProvider) EncodeKV() (vv SettingValueSet, err error) {
for key, value := range pairs {
v := &SettingValue{Name: prefix + key}
if err = v.SetValue(value); err != nil {
if err = v.SetSetting(value); err != nil {
return
}

33
server/system/types/settings.go
View File

@@ -39,17 +39,26 @@ type (
SettingsKV map[string]types.JSONText
)
const (
settingsFilterPerPageMax = 100
)
func MakeSettingValue(name string, value interface{}) *SettingValue {
o := &SettingValue{Name: name}
_ = o.SetValue(value)
_ = o.SetSetting(value)
return o
}
func (v *SettingValue) SetRawValue(str string) error {
// These functions exist only to satisfy pkg/envoyx interfaces
func (v *SettingValue) GetID() uint64 {
return 0
}
func (v *SettingValue) GetValue(string, uint) (any, error) {
return nil, fmt.Errorf("unexpected scenario: GetValue not implemented")
}
func (v *SettingValue) SetValue(string, uint, any) error {
return fmt.Errorf("unexpected scenario: SetValue not implemented")
}
// -----------------------------------------------------------
func (v *SettingValue) SetRawSetting(str string) error {
var dummy interface{}
// Test input to be sure we can save it...
if err := json.Unmarshal([]byte(str), &dummy); err != nil {
@@ -60,7 +69,7 @@ func (v *SettingValue) SetRawValue(str string) error {
return nil
}
func (v *SettingValue) SetValue(value interface{}) (err error) {
func (v *SettingValue) SetSetting(value interface{}) (err error) {
buf := bytes.Buffer{}
enc := json.NewEncoder(&buf)
enc.SetEscapeHTML(false)
@@ -117,6 +126,16 @@ func (set SettingValueSet) FilterByPrefix(prefix string) SettingValueSet {
return pf
}
func (set SettingValueSet) FindByName(name string) *SettingValue {
for _, v := range set {
if v.Name == name {
return v
}
}
return nil
}
func (set SettingValueSet) KV() SettingsKV {
m := SettingsKV{}

19
server/system/types/settings_test.go
View File

@@ -1,9 +1,10 @@
package types
import (
"testing"
"github.com/jmoiron/sqlx/types"
"github.com/stretchr/testify/require"
"testing"
)
func TestSettingsKV_Bool(t *testing.T) {
@@ -53,12 +54,12 @@ func TestSettingsKV_Bool(t *testing.T) {
func TestSettingValueAsString(t *testing.T) {
var req = require.New(t)
req.NoError((&SettingValue{}).SetRawValue(`"string"`), "unable to set value as string")
req.NoError((&SettingValue{}).SetRawValue(`false`), "unable to set value as string")
req.NoError((&SettingValue{}).SetRawValue(`null`), "unable to set value as string")
req.NoError((&SettingValue{}).SetRawValue(`42`), "unable to set value as string")
req.NoError((&SettingValue{}).SetRawValue(`3.14`), "unable to set value as string")
req.Error((&SettingValue{}).SetRawValue(`error`), "expecting error when not setting JSON")
req.NoError((&SettingValue{}).SetRawSetting(`"string"`), "unable to set value as string")
req.NoError((&SettingValue{}).SetRawSetting(`false`), "unable to set value as string")
req.NoError((&SettingValue{}).SetRawSetting(`null`), "unable to set value as string")
req.NoError((&SettingValue{}).SetRawSetting(`42`), "unable to set value as string")
req.NoError((&SettingValue{}).SetRawSetting(`3.14`), "unable to set value as string")
req.Error((&SettingValue{}).SetRawSetting(`error`), "expecting error when not setting JSON")
}
func TestSettingValueSet_Upsert(t *testing.T) {
@@ -84,14 +85,14 @@ func TestSettingValueSet_Changed(t *testing.T) {
// make string value
msv = func(n, v string) *SettingValue {
o := &SettingValue{Name: n}
_ = o.SetValue(v)
_ = o.SetSetting(v)
return o
}
// make bool value
mbv = func(n string, v bool) *SettingValue {
o := &SettingValue{Name: n}
_ = o.SetValue(v)
_ = o.SetSetting(v)
return o
}