DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Improve KV related expr types, add r/w locking

Browse Source 
Also extended filter for compose record values
This commit is contained in:
Vivek Patel
2022-03-01 23:21:21 +05:30
parent fad8725a63
commit b41504dbe3
6 changed files with 316 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
compose/automation/expr_types.go
View File

@@ -585,6 +585,42 @@ func (t *ComposeRecordValues) Merge(nn ...expr.Iterator) (out expr.TypedValue, e
return rv, nil
}
func (t *ComposeRecordValues) Filter(keys ...string) (out expr.TypedValue, err error) {
if t.IsEmpty() {
return
}
// get cloned ComposeRecordValues
out, err = t.Merge()
if err != nil {
return
}
rv := out.(*ComposeRecordValues)
if !rv.IsEmpty() {
rv.value.Values = types.RecordValueSet{}
}
keyMap := make(map[string]string)
for _, k := range keys {
keyMap[k] = k
}
// Push the only with values with matching Name
for _, val := range t.GetValue() {
_, ok := keyMap[val.Name]
if ok {
rr := rv.GetValue().Set(val.Clone())
err = rv.Assign(rr)
if err != nil {
return out, err
}
}
}
return rv, nil
}
func (t *ComposeRecordValues) Delete(keys ...string) (out expr.TypedValue, err error) {
if t.IsEmpty() {
return

2
pkg/apigw/filter/postfilter_test.go
View File

@@ -122,7 +122,7 @@ func Test_jsonResponse(t *testing.T) {
{
name: "KV response as JSON",
expr: `{"expr": "records", "type": "KV"}`,
scope: map[string]string{"foo": "bar", "baz": "bzz"},
scope: expr.Must(expr.Any{}.Cast(map[string]string{"foo": "bar", "baz": "bzz"})),
exp: `{"baz":"bzz","foo":"bar"}`,
},
{

91
pkg/expr/expr_types.go
View File

@@ -504,8 +504,28 @@ func CastToKV(val interface{}) (out map[string]string, err error) {
}
}
func (t *KV) SelectGVal(_ context.Context, k string) (interface{}, error) {
t.mux.RLock()
defer t.mux.RUnlock()
val, err := t.Select(k)
switch c := val.(type) {
case gval.Selector:
return c, nil
default:
return UntypedValue(val), err
}
}
func (t *KV) Each(fn func(k string, v TypedValue) error) (err error) {
if t == nil || t.value == nil {
if t == nil {
return nil
}
t.mux.RLock()
defer t.mux.RUnlock()
if t.value == nil {
return
}
@@ -527,9 +547,14 @@ func (t *KV) Each(fn func(k string, v TypedValue) error) (err error) {
// Merge combines the given KVs into KV
// NOTE: It will return CLONE of the original KV, if its called without any parameters
func (t *KV) Merge(nn ...Iterator) (out TypedValue, err error) {
kv := EmptyKV()
if t != nil {
t.mux.RLock()
defer t.mux.RUnlock()
nn = append([]Iterator{t}, nn...)
nn = append([]Iterator{t}, nn...)
}
kv := EmptyKV()
for _, i := range nn {
err = i.Each(func(k string, v TypedValue) error {
@@ -547,9 +572,13 @@ func (t *KV) Merge(nn ...Iterator) (out TypedValue, err error) {
// Filter take keys returns KV with only those key value pair
func (t *KV) Filter(keys ...string) (out TypedValue, err error) {
t.mux.RLock()
defer t.mux.RUnlock()
if t.value == nil {
return
}
kv := EmptyKV()
for _, k := range keys {
@@ -564,6 +593,9 @@ func (t *KV) Filter(keys ...string) (out TypedValue, err error) {
// Delete take keys returns KV without those key value pair
func (t *KV) Delete(keys ...string) (out TypedValue, err error) {
t.mux.RLock()
defer t.mux.RUnlock()
if t.value == nil {
return
}
@@ -702,8 +734,36 @@ func CastToUrl(val interface{}) (out *url.URL, err error) {
}
}
func (t *KVV) Select(k string) (TypedValue, error) {
if v, is := t.value[k]; is {
return Must(Typify(v)), nil
} else {
return nil, errors.NotFound("no such key '%s'", k)
}
}
func (t *KVV) SelectGVal(_ context.Context, k string) (interface{}, error) {
t.mux.RLock()
defer t.mux.RUnlock()
val, err := t.Select(k)
switch c := val.(type) {
case gval.Selector:
return c, nil
default:
return UntypedValue(val), err
}
}
func (t *KVV) Each(fn func(k string, v TypedValue) error) (err error) {
if t == nil || t.value == nil {
if t == nil {
return
}
t.mux.RLock()
defer t.mux.RUnlock()
if t.value == nil {
return
}
@@ -725,19 +785,25 @@ func (t *KVV) Each(fn func(k string, v TypedValue) error) (err error) {
// Merge combines the given KVVs into KVV
// NOTE: It will return CLONE of the original KVV, if its called without any parameters
func (t *KVV) Merge(nn ...Iterator) (out TypedValue, err error) {
kvv := EmptyKVV()
if t != nil {
t.mux.RLock()
defer t.mux.RUnlock()
nn = append([]Iterator{t}, nn...)
nn = append([]Iterator{t}, nn...)
}
kvv := EmptyKVV()
for _, i := range nn {
err = i.Each(func(k string, v TypedValue) error {
ss, err := cast.ToStringSliceE(v.Get())
var ss []string
ss, err = cast.ToStringSliceE(v.Get())
if err != nil {
return err
}
_, has := kvv.value[k]
if has {
_, is := kvv.value[k]
if is {
kvv.value[k] = append(kvv.value[k], ss...)
} else {
kvv.value[k] = ss
@@ -754,9 +820,13 @@ func (t *KVV) Merge(nn ...Iterator) (out TypedValue, err error) {
// Filter take keys returns KVV with only those key value pair
func (t *KVV) Filter(keys ...string) (out TypedValue, err error) {
t.mux.RLock()
defer t.mux.RUnlock()
if t.value == nil {
return
}
kvv := EmptyKVV()
for _, k := range keys {
@@ -771,6 +841,9 @@ func (t *KVV) Filter(keys ...string) (out TypedValue, err error) {
// Delete take keys returns KVV without those key value pair
func (t *KVV) Delete(keys ...string) (out TypedValue, err error) {
t.mux.RLock()
defer t.mux.RUnlock()
if t.value == nil {
return
}

116
tests/workflows/0016_kv_expressions_step_test.go Normal file
View File

@@ -0,0 +1,116 @@
package workflows
import (
"context"
cmpTypes "github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/expr"
"github.com/stretchr/testify/require"
"testing"
"github.com/cortezaproject/corteza-server/automation/types"
)
func Test0016_kv_expressions_step(t *testing.T) {
var (
ctx = bypassRBAC(context.Background())
req = require.New(t)
)
loadScenario(ctx, t)
t.Run("KV related expressions", func(t *testing.T) {
type (
testInput struct {
Out map[string]string
}
)
var (
aux = testInput{}
vars, _ = mustExecWorkflow(ctx, t, "kv_expressions", types.WorkflowExecParams{})
expected = testInput{
Out: map[string]string{
"testString": "testing string",
},
}
)
req.NoError(vars.Decode(&aux))
req.Equal(expected, aux)
})
t.Run("KVV related expressions", func(t *testing.T) {
type (
testInput struct {
Out map[string][]string
}
)
var (
aux = testInput{}
vars, _ = mustExecWorkflow(ctx, t, "kvv_expressions", types.WorkflowExecParams{})
expected = testInput{
Out: map[string][]string{
"testString": {"foo", "bar"},
},
}
)
req.NoError(vars.Decode(&aux))
req.Equal(expected, aux)
})
t.Run("Vars related expressions", func(t *testing.T) {
type (
testInput struct {
Out map[string]expr.TypedValue
}
)
var (
aux = testInput{}
vars, _ = mustExecWorkflow(ctx, t, "vars_expressions", types.WorkflowExecParams{})
testString = expr.Must(expr.NewString("testing string"))
testInt = expr.Must(expr.NewInteger(40))
testVar = expr.Must(expr.NewVars(map[string]interface{}{
"testString": testString,
"testFloat": expr.Must(expr.NewFloat(50)),
}))
expectedVars = map[string]expr.TypedValue{
"testString": testString,
"testInt": testInt,
"testVar": testVar,
}
expected = testInput{
Out: expectedVars,
}
)
req.NoError(vars.Decode(&aux))
req.Equal(expected, aux)
})
t.Run("ComposeRecordValues related expressions", func(t *testing.T) {
type (
testInput struct {
Out *cmpTypes.Record
}
)
var (
aux = testInput{}
vars, _ = mustExecWorkflow(ctx, t, "compose_record_values_expressions", types.WorkflowExecParams{})
expected = testInput{
Out: &cmpTypes.Record{
Values: []*cmpTypes.RecordValue{
{
Name: "testFloat",
Value: "50",
},
},
},
}
)
req.NoError(vars.Decode(&aux))
req.Equal(expected, aux)
})
}

103
tests/workflows/0016_set_expression_issue_test.go
View File

@@ -1,103 +0,0 @@
package workflows
import (
"context"
cmpTypes "github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/expr"
"github.com/stretchr/testify/require"
"testing"
"github.com/cortezaproject/corteza-server/automation/types"
)
func Test0016_set_expression_issue(t *testing.T) {
var (
ctx = bypassRBAC(context.Background())
req = require.New(t)
)
loadScenario(ctx, t)
t.Run("set expressions", func(t *testing.T) {
type (
testInput struct {
Out map[string]expr.TypedValue
OutConstStr map[string]expr.TypedValue
OutConstInt map[string]expr.TypedValue
OutRv *cmpTypes.Record
}
)
var (
aux = testInput{}
vars, _ = mustExecWorkflow(ctx, t, "set_expression", types.WorkflowExecParams{})
testString = expr.Must(expr.NewString("testing string"))
testInt = expr.Must(expr.NewInteger(40))
testVar = expr.Must(expr.NewVars(map[string]interface{}{
"testString": testString,
"testFloat": expr.Must(expr.NewFloat(50)),
}))
expectedVars = map[string]expr.TypedValue{
"testString": testString,
"testInt": testInt,
"testVar": testVar,
}
expected = testInput{
Out: expectedVars,
OutConstStr: map[string]expr.TypedValue{
"testConstKey": expr.Must(expr.NewString("testConstValue")),
},
OutConstInt: map[string]expr.TypedValue{
"testInt": testInt,
},
OutRv: &cmpTypes.Record{
Values: []*cmpTypes.RecordValue{
{
Name: "testRv",
Value: "testing string",
},
{
Name: "testFloat",
Value: "50",
},
},
},
}
)
req.NoError(vars.Decode(&aux))
req.Equal(expected, aux)
})
t.Run("omit expressions", func(t *testing.T) {
type (
testInput struct {
Out map[string]expr.TypedValue
OutConstInt map[string]expr.TypedValue
OutRv *cmpTypes.Record
}
)
var (
aux = testInput{}
vars, _ = mustExecWorkflow(ctx, t, "omit_expression", types.WorkflowExecParams{})
expected = testInput{
Out: map[string]expr.TypedValue{},
OutConstInt: map[string]expr.TypedValue{},
OutRv: &cmpTypes.Record{
Values: []*cmpTypes.RecordValue{
{
Name: "testFloat",
Value: "50",
},
},
},
}
)
req.NoError(vars.Decode(&aux))
req.Equal(expected, aux)
})
}

123
tests/workflows/testdata/S0016_set_expression_issue/workflow.yaml → tests/workflows/testdata/S0016_kv_expressions_step/workflow.yaml vendored
View File

@@ -1,5 +1,70 @@
workflows:
set_expression:
kv_expressions:
enabled: true
trace: true
triggers:
- enabled: true
stepID: 1
steps:
- stepID: 1
kind: expressions
arguments: [
## Initialize empty kv
{ target: aux, type: KV, value: null },
## Initialize key and value(string) for set expression
{ target: testString, type: String, value: "testString" },
{ target: foo, type: String, value: "testing string" },
# Evaluate set for KV(1st argument) with string value(3rd argument);
# it will set string value type for given key to KV(out)
{ target: out, type: KV, expr: "set(aux, testString, foo)" },
{ target: out, type: KV, expr: "set(out, \"deleteMe\", \"foobar\")" },
# Evaluate omit for KV(1st argument) for key(Name);
# it will omit values with matching keys from KV(out)
{ target: out, type: KV, expr: "omit(out, \"deleteMe\")" },
]
- stepID: 2
kind: termination
paths:
- { parentID: 1, childID: 2 }
kvv_expressions:
enabled: true
trace: true
triggers:
- enabled: true
stepID: 1
steps:
- stepID: 1
kind: expressions
arguments: [
## Initialize empty kvv
{ target: aux, type: KVV, expr: "{\"test\": [\"z\", \"y\", \"x\"], \"deleteMe\": [\"foo\", \"bar\"], }" },
## Initialize key and value(Array) for set expression
{ target: testString, type: String, value: "testString" },
{ target: testArray, type: Any, expr: "[\"foo\", \"bar\"]" },
# Evaluate set for KVV(1st argument) with []string value(3rd argument);
# it will set []string value for given key to KVV(out)
{ target: out, type: KVV, expr: "set(aux, testString, testArray)" },
# Evaluate omit for KVV(1st argument) for key(Name);
# it will omit values with matching keys from KVV(out)
{ target: out, type: KVV, expr: "omit(out, \"test\", \"deleteMe\")" },
]
- stepID: 2
kind: termination
paths:
- { parentID: 1, childID: 2 }
vars_expressions:
enabled: true
trace: true
triggers:
@@ -45,17 +110,9 @@ workflows:
# it will set int value type for given key to Vars(outConstInt)
{ target: outConstInt, type: Vars, expr: "set({}, testInt, bar)" },
## Initialize rv(ComposeRecordValues) and key for set expression
{ target: rv, type: ComposeRecordValues, expr: "{}" },
{ target: testRv, type: String, value: "testRv" },
# Evaluate set for ComposeRecordValues(1st argument) with string value(3rd argument);
# it will set string value for given key(Name) to ComposeRecordValues(outRv)
{ target: outRv, type: ComposeRecordValues, expr: "set(rv, testRv, foo)" },
# Evaluate set for ComposeRecordValues(1st argument) with float value(3rd argument);
# it will set float value for given key(Name) to ComposeRecordValues(outRv)
{ target: outRv, type: ComposeRecordValues, expr: "set(outRv, \"testFloat\", 50)" },
# Evaluate omit for Vars(1st argument) for keys;
# it will omit values matching keys from Vars(outConstInt)
{ target: outConstInt, type: Vars, expr: "omit(outConstInt, testInt)" },
]
- stepID: 2
kind: termination
@@ -63,7 +120,7 @@ workflows:
paths:
- { parentID: 1, childID: 2 }
omit_expression:
compose_record_values_expressions:
enabled: true
trace: true
triggers:
@@ -74,41 +131,23 @@ workflows:
- stepID: 1
kind: expressions
arguments: [
# Initialize empty vars
{ target: aux, type: Vars, value: null },
## Initialize key and value(string) for set/omit expressions
{ target: testString, type: String, value: "testString" },
{ target: foo, type: String, value: "testing string" },
# Set values into Vars
{ target: out, type: Vars, expr: "set(aux, testString, foo)" },
# Evaluate omit for Vars(1st argument) for keys;
# it will omit values matching keys from Vars(outConstInt)
{ target: out, type: Vars, expr: "omit(out, testString)" },
## Set/Omit value into constant
{ target: testInt, type: String, value: "testInt" },
{ target: bar, type: Integer, expr: "40" },
{ target: outConstInt, type: Vars, expr: "set({}, testInt, bar)" },
# Evaluate omit for Vars(1st argument) for keys;
# it will omit values matching keys from Vars(outConstInt)
{ target: outConstInt, type: Vars, expr: "omit(outConstInt, testInt)" },
## Initialize rv(ComposeRecordValues) and key for set/omit expressions
## Initialize rv(ComposeRecordValues) and key for set expression
{ target: rv, type: ComposeRecordValues, expr: "{}" },
{ target: testRv, type: String, value: "testRv" },
{ target: foo, type: String, value: "testing string" },
# Set values into ComposeRecordValues
{ target: outRv, type: ComposeRecordValues, expr: "set(rv, testRv, foo)" },
{ target: outRv, type: ComposeRecordValues, expr: "set(rv, \"removeMe\", \"I will be removed\")" },
{ target: outRv, type: ComposeRecordValues, expr: "set(outRv, \"testFloat\", 50)" },
# Evaluate set for ComposeRecordValues(1st argument) with string value(3rd argument);
# it will set string value for given key(Name) to ComposeRecordValues(outRv)
{ target: out, type: ComposeRecordValues, expr: "set(rv, testRv, foo)" },
# Evaluate set for ComposeRecordValues(1st argument) with float value(3rd argument);
# it will set float value for given key(Name) to ComposeRecordValues(outRv)
{ target: out, type: ComposeRecordValues, expr: "set(out, \"testFloat\", 50)" },
{ target: out, type: ComposeRecordValues, expr: "set(out, \"removeMe\", \"I will be removed\")" },
# Evaluate omit for ComposeRecordValues(1st argument) for key(Name);
# it will omit values matching Name from ComposeRecordValues(outRv)
{ target: outRv, type: ComposeRecordValues, expr: "omit(outRv, \"testRv\", \"removeMe\")" },
{ target: out, type: ComposeRecordValues, expr: "omit(out, \"testRv\", \"removeMe\")" },
]
- stepID: 2
kind: termination