Added Response postfilter

- removed json response
 - added response postfilter
 - unit tests
This commit is contained in:
Peter Grlica
2023-01-16 13:52:44 +01:00
parent 11dccaea9f
commit 31ca5d6c29
16 changed files with 355 additions and 108 deletions
+41 -32
View File
@@ -6,6 +6,8 @@ import (
"fmt"
"net/http"
"net/url"
"reflect"
"strings"
atypes "github.com/cortezaproject/corteza/server/automation/types"
agctx "github.com/cortezaproject/corteza/server/pkg/apigw/ctx"
@@ -31,22 +33,16 @@ type (
}
}
// support for arbitrary response
// obfuscation
customResponse struct {
types.FilterMeta
params struct {
Source string `json:"source"`
}
}
jsonResponse struct {
response struct {
types.FilterMeta
reg typesRegistry
params struct {
Exp *atypes.Expr
Header http.Header `json:"header"`
Exp *atypes.Expr `json:"input"`
Evaluable expr.Evaluable
}
}
@@ -181,11 +177,11 @@ func checkStatus(typ string, status int) bool {
}
}
func NewJsonResponse(opts options.ApigwOpt, reg typesRegistry) (e *jsonResponse) {
e = &jsonResponse{}
func NewResponse(opts options.ApigwOpt, reg typesRegistry) (e *response) {
e = &response{}
e.Name = "jsonResponse"
e.Label = "JSON response"
e.Name = "response"
e.Label = "Response"
e.Kind = types.PostFilter
e.Args = []*types.FilterMetaArg{
@@ -194,6 +190,11 @@ func NewJsonResponse(opts options.ApigwOpt, reg typesRegistry) (e *jsonResponse)
Label: "input",
Options: map[string]interface{}{},
},
{
Type: "header",
Label: "header",
Options: map[string]interface{}{},
},
}
e.reg = reg
@@ -201,28 +202,28 @@ func NewJsonResponse(opts options.ApigwOpt, reg typesRegistry) (e *jsonResponse)
return
}
func (j jsonResponse) New(opts options.ApigwOpt) types.Handler {
return NewJsonResponse(opts, j.reg)
func (j response) New(opts options.ApigwOpt) types.Handler {
return NewResponse(opts, j.reg)
}
func (j jsonResponse) Enabled() bool {
func (j response) Enabled() bool {
return true
}
func (j jsonResponse) String() string {
func (j response) String() string {
return fmt.Sprintf("apigw filter %s (%s)", j.Name, j.Label)
}
func (j jsonResponse) Meta() types.FilterMeta {
func (j response) Meta() types.FilterMeta {
return j.FilterMeta
}
func (j *jsonResponse) Merge(params []byte) (h types.Handler, err error) {
func (j *response) Merge(params []byte) (h types.Handler, err error) {
var (
parser = expr.NewParser()
)
err = json.NewDecoder(bytes.NewBuffer(params)).Decode(&j.params.Exp)
err = json.NewDecoder(bytes.NewBuffer(params)).Decode(&j.params)
if err != nil {
return j, err
@@ -239,11 +240,12 @@ func (j *jsonResponse) Merge(params []byte) (h types.Handler, err error) {
return j, err
}
func (j jsonResponse) Handler() types.HandlerFunc {
func (j response) Handler() types.HandlerFunc {
return func(rw http.ResponseWriter, r *http.Request) (err error) {
var (
ctx = r.Context()
scope = agctx.ScopeFromContext(ctx)
ctx = r.Context()
scope = agctx.ScopeFromContext(ctx)
hasJsonHeader = false
evald interface{}
)
@@ -274,16 +276,23 @@ func (j jsonResponse) Handler() types.HandlerFunc {
return
}
rw.Header().Add("Content-Type", "application/json")
for h, v := range j.params.Header {
for _, vv := range v {
rw.Header().Add(h, vv)
switch v := evald.(type) {
case string:
rw.Write([]byte(v))
default:
e := json.NewEncoder(rw)
err = e.Encode(v)
if strings.ToLower(h) == "content-type" {
hasJsonHeader = vv == "application/json"
}
}
}
if hasJsonHeader && reflect.ValueOf(evald).Kind() != reflect.String {
err = (json.NewEncoder(rw)).Encode(expr.UntypedValue(evald))
return
}
fmt.Fprintf(rw, "%v", evald)
return
}
}
+36 -6
View File
@@ -114,23 +114,53 @@ func Test_jsonResponse(t *testing.T) {
var (
tcc = []tf{
{
name: "Any response as JSON",
expr: `{"expr": "records", "type": "KV"}`,
name: "String response as JSON",
expr: `{"header":{"content-type":["application/json"]},"input":{"expr": "records", "type": "String"}}`,
scope: expr.Must(expr.Any{}.Cast("foobar")),
exp: `foobar`,
},
{
name: "Array response as JSON",
expr: `{"header":{"content-type":["application/json"]},"input":{"expr": "records", "type": "Array"}}`,
scope: expr.Must(expr.Any{}.Cast([]float64{3.14, 42.690})),
exp: `[3.14,42.69]`,
},
{
name: "KV response as JSON",
expr: `{"expr": "records", "type": "KV"}`,
name: "Array response as text",
expr: `{"input":{"expr": "records", "type": "Array"}}`,
scope: expr.Must(expr.Any{}.Cast([]float64{3.14, 42.690})),
exp: `[3.14 42.69]`,
},
{
name: "Any response as JSON",
expr: `{"header":{"content-type":["application/json"]},"input": {"expr": "records", "type": "Any"}}`,
scope: expr.Must(expr.Any{}.Cast(map[string]string{"foo": "bar", "baz": "bzz"})),
exp: `{"baz":"bzz","foo":"bar"}`,
},
{
name: "Any response as text",
expr: `{"input": {"expr": "records", "type": "Any"}}`,
scope: expr.Must(expr.Any{}.Cast(map[string]string{"foo": "bar", "baz": "bzz"})),
exp: `map[baz:bzz foo:bar]`,
},
{
name: "struct array response as JSON",
expr: `{"expr": "toJSON(records)", "type": "String"}`,
expr: `{"input":{"expr": "toJSON(records)", "type": "String"}}`,
scope: []aux{{"First", "Last"}, {"Foo", "bar"}},
exp: `[{"name":"First","surname":"Last"},{"name":"Foo","surname":"bar"}]`,
},
{
name: "struct array response as text",
expr: `{"input":{"expr": "records", "type": "String"}}`,
scope: []aux{{"First", "Last"}, {"Foo", "bar"}},
exp: `[{First Last} {Foo bar}]`,
},
{
name: "string csv response as text",
expr: `{"header":{"content-type":["application/octet-stream"],"Content-Disposition":["attachment; filename=foo.txt"],"Content-Transfer-Encoding":["binary"]},"input":{"expr": "records", "type": "String"}}`,
scope: "\"header 1\",\"header 2\"\nvalue 1,value 2\nvalue 3, value 4",
exp: "\"header 1\",\"header 2\"\nvalue 1,value 2\nvalue 3, value 4",
},
}
)
@@ -145,7 +175,7 @@ func Test_jsonResponse(t *testing.T) {
r = r.WithContext(agctx.ScopeToContext(context.Background(), scope))
h := getHandler(NewJsonResponse(options.ApigwOpt{}, &mockHandlerRegistry{}))
h := getHandler(NewResponse(options.ApigwOpt{}, &mockHandlerRegistry{}))
h, err := h.Merge([]byte(tc.expr))
req.NoError(err)
+7 -3
View File
@@ -12,6 +12,7 @@ import (
atypes "github.com/cortezaproject/corteza/server/automation/types"
agctx "github.com/cortezaproject/corteza/server/pkg/apigw/ctx"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
"github.com/cortezaproject/corteza/server/pkg/auth"
pe "github.com/cortezaproject/corteza/server/pkg/errors"
"github.com/cortezaproject/corteza/server/pkg/expr"
"github.com/cortezaproject/corteza/server/pkg/jsenv"
@@ -90,6 +91,9 @@ func (h *workflow) Merge(params []byte) (types.Handler, error) {
t = struct {
Workflow string `json:"workflow"`
}{}
// ctx = context.Background()
ctx = auth.SetIdentityToContext(context.Background(), auth.ServiceUser())
)
err := json.NewDecoder(bytes.NewBuffer(params)).Decode(&t)
@@ -101,17 +105,17 @@ func (h *workflow) Merge(params []byte) (types.Handler, error) {
if err == nil {
h.params.Workflow = uint64(i)
return h, h.d.Load(context.Background())
return h, h.d.Load(ctx)
}
if wf, _, err := h.d.Search(context.Background(), atypes.WorkflowFilter{Query: fmt.Sprintf("handle='%s'", t.Workflow)}); err != nil {
if wf, _, err := h.d.Search(ctx, atypes.WorkflowFilter{Query: t.Workflow}); err != nil {
return h, err
} else {
h.params.Workflow = wf[0].ID
}
// preload workflow cache
return h, h.d.Load(context.Background())
return h, h.d.Load(ctx)
}
func (h workflow) Handler() types.HandlerFunc {
+1 -1
View File
@@ -75,7 +75,7 @@ func (r *Registry) Preload() {
// postfilters
r.Add("redirection", filter.NewRedirection(r.opts))
r.Add("jsonResponse", filter.NewJsonResponse(r.opts, service.Registry()))
r.Add("response", filter.NewResponse(r.opts, service.Registry()))
r.Add("defaultJsonResponse", filter.NewDefaultJsonResponse(r.opts))
}
+39
View File
@@ -0,0 +1,39 @@
package provision
import (
"context"
"fmt"
"net/http"
atypes "github.com/cortezaproject/corteza/server/automation/types"
"github.com/cortezaproject/corteza/server/store"
"github.com/cortezaproject/corteza/server/system/types"
"go.uber.org/zap"
)
func apigwFilters(ctx context.Context, log *zap.Logger, s store.Storer) (err error) {
var (
filters types.ApigwFilterSet
)
if filters, _, err = store.SearchApigwFilters(ctx, s, types.ApigwFilterFilter{Ref: "jsonResponse"}); err != nil {
return
}
for _, f := range filters {
h := http.Header{}
h.Add("Content-Type", "application/json")
f.Ref = "response"
f.Params = map[string]interface{}{
"header": h,
"input": &atypes.Expr{Expr: f.Params["input"].(string), Type: "String"},
}
if err = store.UpdateApigwFilter(ctx, s, f); err != nil {
log.Warn(fmt.Sprintf("could not migrate jsonResponse to response: %s", err))
}
}
return
}
+58 -3
View File
@@ -3,6 +3,10 @@ package provision
import (
"context"
"fmt"
"os"
"regexp"
"strings"
internalAuth "github.com/cortezaproject/corteza/server/pkg/auth"
"github.com/cortezaproject/corteza/server/pkg/handle"
"github.com/cortezaproject/corteza/server/pkg/id"
@@ -10,11 +14,9 @@ import (
"github.com/cortezaproject/corteza/server/pkg/options"
"github.com/cortezaproject/corteza/server/system/service"
"go.uber.org/zap"
"os"
"regexp"
"strings"
"github.com/cortezaproject/corteza/server/pkg/errors"
"github.com/cortezaproject/corteza/server/pkg/rand"
"github.com/cortezaproject/corteza/server/store"
"github.com/cortezaproject/corteza/server/system/types"
)
@@ -166,3 +168,56 @@ func createUserHandle(u *types.User) (hdl string) {
return hdl
}
// defaultAuthClient checks if default client exists (handle = AUTH_DEFAULT_CLIENT) and adds it
func defaultAuthClient(ctx context.Context, log *zap.Logger, s store.AuthClients, authOpt options.AuthOpt) error {
if authOpt.DefaultClient == "" {
// Default client not set
return nil
}
c := &types.AuthClient{
ID: id.Next(),
Handle: authOpt.DefaultClient,
Meta: &types.AuthClientMeta{
Name: "Corteza Web Applications",
},
ValidGrant: "authorization_code",
RedirectURI: func() string {
// Disabling protection by redirection URL for now, it caused too much confusion on simple setups
//baseURL, _ := url.Parse(authOpt.BaseURL)
//return fmt.Sprintf("%s://%s", baseURL.Scheme, baseURL.Hostname())
return ""
}(),
Secret: string(rand.Bytes(64)),
Scope: "profile api",
Enabled: true,
Trusted: true,
Security: &types.AuthClientSecurity{},
Labels: nil,
CreatedAt: *now(),
}
_, err := store.LookupAuthClientByHandle(ctx, s, c.Handle)
if err == nil {
return nil
}
if !errors.IsNotFound(err) {
return err
}
if err = store.CreateAuthClient(ctx, s, c); err != nil {
return err
}
log.Info(
"Added OAuth2 client",
zap.String("name", c.Meta.Name),
zap.String("redirectURI", c.RedirectURI),
zap.Uint64("clientId", c.ID),
)
return nil
}
+1 -57
View File
@@ -4,12 +4,8 @@ import (
"context"
"time"
"github.com/cortezaproject/corteza/server/pkg/errors"
"github.com/cortezaproject/corteza/server/pkg/id"
"github.com/cortezaproject/corteza/server/pkg/options"
"github.com/cortezaproject/corteza/server/pkg/rand"
"github.com/cortezaproject/corteza/server/store"
"github.com/cortezaproject/corteza/server/system/types"
"go.uber.org/zap"
)
@@ -39,6 +35,7 @@ func Run(ctx context.Context, log *zap.Logger, s store.Storer, provisionOpt opti
// Auto-discoveries and other parts that cannot be imported from static files
func() error { return emailSettings(ctx, s) },
func() error { return apigwFilters(ctx, log.Named("apigw.filters"), s) },
func() error { return authAddExternals(ctx, log.Named("auth.externals"), s) },
func() error { return oidcAutoDiscovery(ctx, log.Named("auth.oidc-auto-discovery"), s, authOpt) },
func() error { return defaultAuthClient(ctx, log.Named("auth.clients"), s, authOpt) },
@@ -53,56 +50,3 @@ func Run(ctx context.Context, log *zap.Logger, s store.Storer, provisionOpt opti
return nil
}
// defaultAuthClient checks if default client exists (handle = AUTH_DEFAULT_CLIENT) and adds it
func defaultAuthClient(ctx context.Context, log *zap.Logger, s store.AuthClients, authOpt options.AuthOpt) error {
if authOpt.DefaultClient == "" {
// Default client not set
return nil
}
c := &types.AuthClient{
ID: id.Next(),
Handle: authOpt.DefaultClient,
Meta: &types.AuthClientMeta{
Name: "Corteza Web Applications",
},
ValidGrant: "authorization_code",
RedirectURI: func() string {
// Disabling protection by redirection URL for now, it caused too much confusion on simple setups
//baseURL, _ := url.Parse(authOpt.BaseURL)
//return fmt.Sprintf("%s://%s", baseURL.Scheme, baseURL.Hostname())
return ""
}(),
Secret: string(rand.Bytes(64)),
Scope: "profile api",
Enabled: true,
Trusted: true,
Security: &types.AuthClientSecurity{},
Labels: nil,
CreatedAt: *now(),
}
_, err := store.LookupAuthClientByHandle(ctx, s, c.Handle)
if err == nil {
return nil
}
if !errors.IsNotFound(err) {
return err
}
if err = store.CreateAuthClient(ctx, s, c); err != nil {
return err
}
log.Info(
"Added OAuth2 client",
zap.String("name", c.Meta.Name),
zap.String("redirectURI", c.RedirectURI),
zap.Uint64("clientId", c.ID),
)
return nil
}
+19 -1
View File
@@ -343,6 +343,22 @@ func DefaultFilters() (f *extendedFilters) {
return ee, f, nil
}
f.ApigwFilter = func(s *Store, f systemType.ApigwFilterFilter) (ee []goqu.Expression, _ systemType.ApigwFilterFilter, err error) {
if ee, f, err = ApigwFilterFilter(s.Dialect, f); err != nil {
return
}
if len(f.Ref) > 0 {
ee = append(ee, goqu.C("ref").Eq(f.Ref))
}
if len(f.Kind) > 0 {
ee = append(ee, goqu.C("kind").Eq(f.Kind))
}
return ee, f, nil
}
return
}
@@ -405,7 +421,9 @@ func stateFalseComparison(d drivers.Dialect, lit string, fs filter.State) goqu.E
}
// @todo: Currently we have for support for MsSQL, MySql, PSQL, SQLite drivers,
// this changes is supported by all DB but we need to move to store.driver
//
// this changes is supported by all DB but we need to move to store.driver
//
// generateSorting verify and converts given sorting to literal if required
func generateSorting(sortables map[string]string, s *filter.SortExpr) (out goqu.Expression, err error) {
const COALESCE string = "coalesce"
@@ -18,7 +18,7 @@ props:
fields: [ ID, ref ]
- name: search
type: "*types.ApigwFilterFilter"
fields: []
fields: [ kind ]
actions:
- action: search
+5 -1
View File
@@ -3,9 +3,10 @@ package types
import (
"database/sql/driver"
"encoding/json"
"github.com/cortezaproject/corteza/server/pkg/sql"
"time"
"github.com/cortezaproject/corteza/server/pkg/sql"
"github.com/cortezaproject/corteza/server/pkg/filter"
)
@@ -35,6 +36,9 @@ type (
Deleted filter.State `json:"deleted"`
Disabled filter.State `json:"disabled"`
Kind string `json:"kind"`
Ref string `json:"ref"`
// Check fn is called by store backend for each resource found function can
// modify the resource and return false if store should not return it
//
+25 -3
View File
@@ -3,13 +3,16 @@ package apigw
import (
"context"
"errors"
"github.com/cortezaproject/corteza/server/pkg/dal"
"github.com/cortezaproject/corteza/server/store/adapters/rdbms/drivers/sqlite"
"fmt"
"os"
"path"
"testing"
"github.com/cortezaproject/corteza/server/pkg/dal"
"github.com/cortezaproject/corteza/server/store/adapters/rdbms/drivers/sqlite"
"github.com/cortezaproject/corteza/server/app"
as "github.com/cortezaproject/corteza/server/automation/service"
"github.com/cortezaproject/corteza/server/pkg/api/server"
"github.com/cortezaproject/corteza/server/pkg/apigw"
"github.com/cortezaproject/corteza/server/pkg/auth"
@@ -61,8 +64,8 @@ func init() {
func InitTestApp() {
ctx := cli.Context()
if testApp == nil {
if testApp == nil {
testApp = helpers.NewIntegrationTestApp(ctx, func(app *app.CortezaApp) (err error) {
service.DefaultStore, err = sqlite.ConnectInMemory(ctx)
if err != nil {
@@ -74,6 +77,10 @@ func InitTestApp() {
})
}
if err := testApp.Activate(ctx); err != nil {
panic(fmt.Errorf("could not activate corteza: %v", err))
}
if r == nil {
r = chi.NewRouter()
r.Use(server.BaseMiddleware(false, logger.Default())...)
@@ -144,11 +151,16 @@ func (h helper) apiInit() *apitest.APITest {
func setupScenario(t *testing.T) (context.Context, helper, store.Storer) {
ctx, h, s := setup(t)
loadScenario(ctx, s, t, h)
loadRbacRules(ctx, s, t, h)
_ = apigw.Service().Reload(ctx)
return ctx, h, s
}
func loadRbacRules(ctx context.Context, s store.Storer, t *testing.T, h helper) {
helpers.AllowMeWorkflowSearch(h)
}
func setup(t *testing.T) (context.Context, helper, store.Storer) {
h := newHelper(t)
s := service.DefaultStore
@@ -156,6 +168,7 @@ func setup(t *testing.T) (context.Context, helper, store.Storer) {
u := &sysTypes.User{
ID: id.Next(),
}
u.SetRoles(auth.BypassRoles().IDs()...)
ctx := auth.SetIdentityToContext(context.Background(), u)
@@ -187,11 +200,20 @@ func loadScenario(ctx context.Context, s store.Storer, t *testing.T, h helper) {
func loadScenarioWithName(ctx context.Context, s store.Storer, t *testing.T, h helper, scenario string) {
cleanup(ctx, h, s)
parseEnvoy(ctx, s, h, path.Join("testdata", scenario))
loadWorkflows(ctx, h)
}
func loadWorkflows(ctx context.Context, h helper) {
err := as.DefaultWorkflow.Load(ctx)
h.a.NoError(err)
}
func cleanup(ctx context.Context, h helper, s store.Storer) {
h.noError(s.TruncateApigwFilters(ctx))
h.noError(s.TruncateApigwRoutes(ctx))
h.noError(s.TruncateRbacRules(ctx))
h.noError(s.TruncateAutomationTriggers(ctx))
h.noError(s.TruncateAutomationWorkflows(ctx))
}
func parseEnvoy(ctx context.Context, s store.Storer, h helper, path string) {
@@ -0,0 +1,20 @@
package apigw
import (
"net/http"
"testing"
)
func Test_postfilter_string_json(t *testing.T) {
var (
_, h, _ = setupScenario(t)
)
h.apiInit().
Get("/json/string").
Header("Accept", "application/json").
Expect(t).
Status(http.StatusOK).
Body("{\"baz\":{\"@value\":{\"1\":{\"@value\":1,\"@type\":\"Float\"},\"2\":{\"@value\":2,\"@type\":\"Float\"}},\"@type\":\"Vars\"},\"foo\":{\"@value\":\"bar\",\"@type\":\"String\"}}").
End()
}
@@ -0,0 +1,20 @@
package apigw
import (
"net/http"
"testing"
)
func Test_postfilter_string_kv(t *testing.T) {
var (
_, h, _ = setupScenario(t)
)
h.apiInit().
Get("/json/kv").
Header("Accept", "application/json").
Expect(t).
Status(http.StatusOK).
Body("{\"baz\":\"123\",\"foo\":\"bar\"}").
End()
}
@@ -0,0 +1,38 @@
apigateway:
- endpoint: /json/string
method: GET
enabled: true
filters:
- ref: response
kind: postfilter
enabled: true
params:
input:
expr: toJSON(test)
type: String
header:
Content-Type: [ application/json ]
- ref: workflow
kind: processer
enabled: true
params:
workflow: 'test_variable_json'
workflows:
- id: 1
enabled: true
trace: false
handle: test_variable_json
meta:
name: Test variable
triggers:
- enabled: true
stepID: 1
steps:
- stepID: 1
kind: expressions
arguments:
- target: test
type: Any
expr: |
{"foo":"bar","baz":{"1":1, "2":2}}
@@ -0,0 +1,38 @@
apigateway:
- endpoint: /json/kv
method: GET
enabled: true
filters:
- ref: response
kind: postfilter
enabled: true
params:
input:
expr: test
type: KV
header:
Content-Type: [ application/json ]
- ref: workflow
kind: processer
enabled: true
params:
workflow: 'test_variable_kv'
workflows:
- id: 1
enabled: true
trace: false
handle: test_variable_kv
meta:
name: Test variable
triggers:
- enabled: true
stepID: 1
steps:
- stepID: 1
kind: expressions
arguments:
- target: test
type: KV
expr: |
{"foo":"bar","baz":123}
+6
View File
@@ -3,6 +3,7 @@ package helpers
import (
"context"
automationTypes "github.com/cortezaproject/corteza/server/automation/types"
composeTypes "github.com/cortezaproject/corteza/server/compose/types"
"github.com/cortezaproject/corteza/server/pkg/auth"
"github.com/cortezaproject/corteza/server/pkg/cli"
@@ -71,6 +72,11 @@ func AllowMeModuleCreate(mrg myRoleGetter) {
AllowMe(mrg, composeTypes.NamespaceRbacResource(0), "module.create")
}
func AllowMeWorkflowSearch(mrg myRoleGetter) {
AllowMe(mrg, automationTypes.WorkflowRbacResource(0), "workflows.search")
AllowMe(mrg, automationTypes.WorkflowRbacResource(0), "triggers.search")
}
func AllowMeModuleSearch(mrg myRoleGetter) {
AllowMe(mrg, composeTypes.NamespaceRbacResource(0), "modules.search")
AllowMe(mrg, composeTypes.ModuleRbacResource(0, 0), "read")