From c299a64086ae942893368346251c2b43b43665c5 Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Mon, 30 Nov 2020 07:53:15 +0100 Subject: [PATCH] Removed obsolete importer code, pkg/deinterfacer --- compose/commands/exporter.go | 51 ++++++-- pkg/deinterfacer/deinterfacer.go | 195 ---------------------------- pkg/importer/importer.go | 36 ----- pkg/importer/static_fs.go | 57 -------- pkg/rbac/importer.go | 149 --------------------- pkg/rbac/importer_test.go | 48 ------- pkg/settings/importer.go | 54 -------- pkg/settings/importer_test.go | 44 ------- pkg/settings/testdata/settings.yaml | 17 --- 9 files changed, 43 insertions(+), 608 deletions(-) delete mode 100644 pkg/deinterfacer/deinterfacer.go delete mode 100644 pkg/importer/importer.go delete mode 100644 pkg/importer/static_fs.go delete mode 100644 pkg/rbac/importer.go delete mode 100644 pkg/rbac/importer_test.go delete mode 100644 pkg/settings/importer.go delete mode 100644 pkg/settings/importer_test.go delete mode 100644 pkg/settings/testdata/settings.yaml diff --git a/compose/commands/exporter.go b/compose/commands/exporter.go index b4123193a..25623f35a 100644 --- a/compose/commands/exporter.go +++ b/compose/commands/exporter.go @@ -8,7 +8,6 @@ import ( "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/auth" "github.com/cortezaproject/corteza-server/pkg/cli" - "github.com/cortezaproject/corteza-server/pkg/deinterfacer" "github.com/cortezaproject/corteza-server/pkg/errors" "github.com/cortezaproject/corteza-server/pkg/handle" "github.com/cortezaproject/corteza-server/pkg/rbac" @@ -17,6 +16,7 @@ import ( sysService "github.com/cortezaproject/corteza-server/system/service" sysTypes "github.com/cortezaproject/corteza-server/system/types" sqlTypes "github.com/jmoiron/sqlx/types" + "github.com/spf13/cast" "github.com/spf13/cobra" "gopkg.in/yaml.v2" "regexp" @@ -516,10 +516,10 @@ func expPageBlocks(in types.PageBlocks, pages types.PageSet, modules types.Modul func expPageBlocks_Automation(b *types.PageBlock) { bb := make([]interface{}, 0) - _ = deinterfacer.Each(b.Options["buttons"], func(_ int, _ string, btn interface{}) error { + _ = each(b.Options["buttons"], func(_ int, _ string, btn interface{}) error { button := map[string]interface{}{} - _ = deinterfacer.Each(btn, func(_ int, k string, v interface{}) error { + _ = each(btn, func(_ int, k string, v interface{}) error { switch k { case "triggerID", "scriptID": // if s := scripts.FindByID(deinterfacer.ToUint64(v)); s != nil { @@ -541,10 +541,10 @@ func expPageBlocks_Automation(b *types.PageBlock) { func expPageBlocks_Calendar(b *types.PageBlock, modules types.ModuleSet) { ff := make([]interface{}, 0) - _ = deinterfacer.Each(b.Options["feeds"], func(_ int, _ string, def interface{}) error { + _ = each(b.Options["feeds"], func(_ int, _ string, def interface{}) error { feed := map[string]interface{}{} - _ = deinterfacer.Each(def, func(_ int, k string, v interface{}) error { + _ = each(def, func(_ int, k string, v interface{}) error { switch k { case "options": expEncodeModuleInMap(v, modules) @@ -565,10 +565,10 @@ func expPageBlocks_Calendar(b *types.PageBlock, modules types.ModuleSet) { func expPageBlocks_Metric(b *types.PageBlock, modules types.ModuleSet) { mm := make([]interface{}, 0) - _ = deinterfacer.Each(b.Options["metrics"], func(_ int, _ string, def interface{}) error { + _ = each(b.Options["metrics"], func(_ int, _ string, def interface{}) error { metric := map[string]interface{}{} - _ = deinterfacer.Each(def, func(_ int, k string, v interface{}) error { + _ = each(def, func(_ int, k string, v interface{}) error { switch k { case "moduleID": metric["module"] = expEncodeModule(v, modules) @@ -587,7 +587,7 @@ func expPageBlocks_Metric(b *types.PageBlock, modules types.ModuleSet) { } func expEncodeModule(id interface{}, modules types.ModuleSet) string { - if moduleID := deinterfacer.ToUint64(id); moduleID > 0 { + if moduleID := cast.ToUint64(id); moduleID > 0 { if module := modules.FindByID(moduleID); module != nil { return makeHandleFromName(module.Name, module.Handle, "module-%d", module.ID) } @@ -686,3 +686,38 @@ func makeHandleFromName(name, currentHandle, def string, id uint64) string { return fmt.Sprintf(def, id) } + +// each() tries to resolve interface{} into map[interface{}]interface{} or []interface{} +// and calls fn() for each entry +func each(i interface{}, fn func(int, string, interface{}) error) (err error) { + if kv, ok := i.(map[interface{}]interface{}); ok { + for k, v := range kv { + if key, ok := k.(string); !ok { + err = fmt.Errorf("unsupported key type: %T (%+v)", k, k) + } else { + err = fn(-1, key, v) + } + + if err != nil { + return + } + } + } else if kv, ok := i.(map[string]interface{}); ok { + for k, v := range kv { + err = fn(-1, k, v) + + if err != nil { + return + } + } + } else if slice, ok := i.([]interface{}); ok { + _ = slice + for index, i := range slice { + if err = fn(index, "", i); err != nil { + return + } + } + } + + return +} diff --git a/pkg/deinterfacer/deinterfacer.go b/pkg/deinterfacer/deinterfacer.go deleted file mode 100644 index 19d16fa81..000000000 --- a/pkg/deinterfacer/deinterfacer.go +++ /dev/null @@ -1,195 +0,0 @@ -package deinterfacer - -import ( - "strconv" - - "github.com/pkg/errors" -) - -// each() tries to resolve interface{} into map[interface{}]interface{} or []interface{} -// and calls fn() for each entry -func Each(i interface{}, fn func(int, string, interface{}) error) (err error) { - if kv, ok := i.(map[interface{}]interface{}); ok { - for k, v := range kv { - if key, ok := k.(string); !ok { - err = errors.WithStack(errors.Errorf("unsupported key type: %T (%+v)", k, k)) - } else { - err = fn(-1, key, v) - } - - if err != nil { - return - } - } - } else if kv, ok := i.(map[string]interface{}); ok { - for k, v := range kv { - err = fn(-1, k, v) - - if err != nil { - return - } - } - } else if slice, ok := i.([]interface{}); ok { - _ = slice - for index, i := range slice { - if err = fn(index, "", i); err != nil { - return - } - } - } - - return -} - -func Simplify(i interface{}) interface{} { - if IsMap(i) { - kv := map[string]interface{}{} - Each(i, func(_ int, k string, v interface{}) error { - kv[k] = Simplify(v) - return nil - }) - - return kv - } else if IsSlice(i) { - s := make([]interface{}, len(i.([]interface{}))) - - Each(i, func(n int, _ string, v interface{}) error { - s[n] = Simplify(v) - return nil - }) - - return s - } else { - return i - } -} - -// KVsetString assigns value (if exists and of type string) from src to destination -func KVsetString(dst *string, key string, src interface{}, def ...string) { - if kv, ok := src.(map[interface{}]interface{}); !ok { - return - } else if val, ok := kv[key]; ok { - *dst = ToString(val, def...) - } -} - -// ToString assigns value (if exists and of type string) from src to destination -func ToString(val interface{}, def ...string) string { - if str, ok := val.(string); ok { - return str - } else if len(def) > 0 { - return def[0] - } - return "" -} - -// ToInt assigns value (if exists and of type int) from src to destination -func ToInt(val interface{}, def ...int) int { - if i, ok := val.(int); ok { - return i - } else if len(def) > 0 { - return def[0] - } - return 0 -} - -// ToInt assigns value (if exists and of type int) from src to destination -func ToUint64(val interface{}, def ...uint64) uint64 { - if i, ok := val.(uint64); ok { - return i - } else if s, ok := val.(string); ok { - if i, _ := strconv.ParseUint(s, 10, 64); i > 0 { - return i - } - } - - if len(def) > 0 { - return def[0] - } - return 0 -} - -// ToBool assigns value (if exists and of type bool) from src to destination -func ToBool(val interface{}, def ...bool) bool { - if b, ok := val.(bool); ok { - return b - } else if len(def) > 0 { - return def[0] - } - return false -} - -func ToStrings(i interface{}) (out []string) { - var ( - ii []interface{} - str string - ok bool - n int - ) - - if ii, ok = i.([]interface{}); ok { - out = make([]string, len(ii)) - for n, i = range ii { - if str, ok = i.(string); ok { - out[n] = str - } - } - } else if str, ok = i.(string); ok { - return []string{str} - } - - return -} - -func ToSliceOfStringToInterfaceMap(def interface{}) []map[string]interface{} { - items := make([]map[string]interface{}, 0) - _ = Each(def, func(_ int, _ string, def interface{}) error { - item := make(map[string]interface{}) - _ = Each(def, func(_ int, k string, v interface{}) error { - item[k] = Simplify(v) - return nil - }) - - items = append(items, item) - return nil - }) - - return items -} - -// ToInt assigns values (if exists and of type ints) from src to destination -func ToInts(i interface{}, def ...int) (out []int) { - var ( - ii []interface{} - num int - ok bool - n int - ) - - if ii, ok = i.([]interface{}); ok { - out = make([]int, len(ii)) - for n, i = range ii { - if num, ok = i.(int); ok { - out[n] = num - } - } - } else if num, ok = i.(int); ok { - return []int{num} - } - - return -} - -func IsMap(i interface{}) (ok bool) { - _, ok = i.(map[interface{}]interface{}) - return -} - -func IsSlice(i interface{}) (ok bool) { - _, ok = i.([]interface{}) - return -} - -func IsIterable(i interface{}) (ok bool) { - return IsMap(i) || IsSlice(i) -} diff --git a/pkg/importer/importer.go b/pkg/importer/importer.go deleted file mode 100644 index 3bc363a76..000000000 --- a/pkg/importer/importer.go +++ /dev/null @@ -1,36 +0,0 @@ -package importer - -import ( - "context" - - "github.com/cortezaproject/corteza-server/pkg/rbac" - "github.com/cortezaproject/corteza-server/pkg/settings" -) - -type ( - Interface interface { - CastSet(set interface{}) error - Cast(handle string, set interface{}) error - } - - PermissionImporter interface { - CastSet(string, string, interface{}) error - CastResourcesSet(string, interface{}) error - UpdateResources(base string, handle string, ID uint64) - UpdateRoles(handle string, ID uint64) - Store(context.Context, rbac.ImportKeeper) error - } - - SettingImporter interface { - CastSet(interface{}) error - Store(context.Context, settings.ImportKeeper) error - } -) - -func NormalizeHandle(in string) string { - return in -} - -func IsValidHandle(in string) bool { - return true -} diff --git a/pkg/importer/static_fs.go b/pkg/importer/static_fs.go deleted file mode 100644 index 0a4b16b59..000000000 --- a/pkg/importer/static_fs.go +++ /dev/null @@ -1,57 +0,0 @@ -package importer - -import ( - "bytes" - "io" - "os" - "path/filepath" - "sort" - - "github.com/goware/statik/fs" - "github.com/pkg/errors" -) - -// ReadStatic reads static FS and returns slice of io.Readers -func ReadStatic(data string) ([]io.Reader, error) { - var ( - files = make([]string, 0) - - yamlFilter = func(filename string, info os.FileInfo, err error) error { - if err == nil && !info.IsDir() { - if matched, err := filepath.Match("/*.yaml", filename); matched && err == nil { - files = append(files, filename) - } - } - - return err - } - - sfs, err = fs.New(data) - - readers []io.Reader - ) - - if err != nil { - return nil, errors.Wrap(err, "could not read static filesystem") - } - - if err = fs.Walk(sfs, "/", yamlFilter); err != nil { - return nil, errors.Wrap(err, "could not filter files") - } - - if len(files) == 0 { - return nil, nil - } - - sort.Strings(files) - - for _, file := range files { - if bb, err := fs.ReadFile(sfs, file); err != nil { - return nil, errors.Wrapf(err, "could not read %s", file) - } else { - readers = append(readers, bytes.NewBuffer(bb)) - } - } - - return readers, nil -} diff --git a/pkg/rbac/importer.go b/pkg/rbac/importer.go deleted file mode 100644 index 3e0086b8d..000000000 --- a/pkg/rbac/importer.go +++ /dev/null @@ -1,149 +0,0 @@ -package rbac - -import ( - "context" - "sort" - "strings" - - "github.com/pkg/errors" - - "github.com/cortezaproject/corteza-server/pkg/deinterfacer" -) - -type ( - Importer struct { - whitelist whitelistChecker - - // Rules per role - rules map[string]RuleSet - } - - whitelistChecker interface { - Check(*Rule) bool - } - - ImportKeeper interface { - Grant(ctx context.Context, rr ...*Rule) error - } -) - -func NewImporter(wl whitelistChecker) *Importer { - return &Importer{ - whitelist: wl, - } -} - -// CastSet - resolves permission rules for specific resource: -// : [, ...] -func (imp *Importer) CastSet(resource, accessStr string, roles interface{}) (err error) { - if !deinterfacer.IsMap(roles) { - return errors.New("expecting map of roles") - } - return deinterfacer.Each(roles, func(_ int, roleHandle string, oo interface{}) error { - return imp.appendPermissionRule(roleHandle, accessStr, resource, oo) - }) -} - -// CastResourcesSet - resolves permission rules: -// { : [, ...] } -func (imp *Importer) CastResourcesSet(accessStr string, roles interface{}) (err error) { - // if !IsIterable(roles) { - // return errors.New("expecting map of roles") - // } - - return deinterfacer.Each(roles, func(_ int, roleHandle string, perResource interface{}) error { - if !deinterfacer.IsMap(perResource) { - return errors.New("expecting map of resources") - } - - // Each resource - return deinterfacer.Each(perResource, func(_ int, resource string, oo interface{}) error { - // We want to make life of the person that's preparing the import data easy, so - // let's do a little guessing instead of him: - if strings.Contains(resource, ":") { - // This is not service-level resource, trim * and : from the end - resource = strings.TrimRight(resource, ":*") - resource = Resource(resource + string(resourceDelimiter)).AppendWildcard().String() - } - - return imp.appendPermissionRule(roleHandle, accessStr, resource, oo) - }) - }) -} - -func (imp *Importer) appendPermissionRule(roleHandle, accessStr, res string, oo interface{}) (err error) { - var access Access - - if err = access.UnmarshalJSON([]byte(accessStr)); err != nil { - return - } - - if imp.rules == nil { - imp.rules = map[string]RuleSet{} - } - - if imp.rules[roleHandle] == nil { - imp.rules[roleHandle] = RuleSet{} - } - - operations := deinterfacer.ToStrings(oo) - if operations == nil { - return errors.New("could not resolve permission rule operations") - } - - sort.Strings(operations) - - for _, op := range operations { - rule := &Rule{ - Access: access, - Resource: Resource(res), - Operation: Operation(op), - } - - if imp.whitelist != nil && !imp.whitelist.Check(rule) { - return errors.Errorf("invalid rule: operation %q on resource %q", op, res) - } - - imp.rules[roleHandle] = append(imp.rules[roleHandle], rule) - } - - return nil -} - -// UpdateResources iterates over all rules and replaces resource (foo:bar => foo:42) -func (imp *Importer) UpdateResources(base, handle string, ID uint64) { - var ( - from = Resource(base).append(handle) - to = Resource(base).AppendID(ID) - ) - for _, rules := range imp.rules { - for _, rule := range rules { - if rule.Resource == from { - rule.Resource = to - } - } - } -} - -func (imp *Importer) UpdateRoles(handle string, ID uint64) { - if imp.rules[handle] != nil { - for _, rule := range imp.rules[handle] { - rule.RoleID = ID - } - } -} - -func (imp *Importer) Store(ctx context.Context, k ImportKeeper) (err error) { - for _, rr := range imp.rules { - // Make sure all rules have valid role - rr, _ = rr.Filter(func(rule *Rule) (b bool, e error) { - return rule.RoleID > 0, nil - }) - - if err = k.Grant(ctx, rr...); err != nil { - return - } - } - - return -} diff --git a/pkg/rbac/importer_test.go b/pkg/rbac/importer_test.go deleted file mode 100644 index c59fe55bd..000000000 --- a/pkg/rbac/importer_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package rbac - -import ( - "testing" -) - -func TestPermissionRulesImport_CastResourcesSet(t *testing.T) { - t.Skip() - // tests := []struct { - // name string - // resource string - // yaml string - // rules map[string]RuleSet - // }{ - // {name: "empty", yaml: ``}, - // {name: "empty map", yaml: `{}`}, - // {name: "empty slice", yaml: `[]`}, - // { - // name: "one role, one resource, two ops", - // yaml: `admins: { resource: [ read, write ] }`, - // rules: map[string]RuleSet{ - // "admins": { - // AllowRule(0, "resource", "read"), - // AllowRule(0, "resource", "write"), - // }, - // }, - // }, - // { - // name: "op as string", - // yaml: `admins: { resource: read }`, - // rules: map[string]RuleSet{ - // "admins": { - // AllowRule(0, "resource", "read"), - // }, - // }, - // }, - // } - // for _, tt := range tests { - // t.Run(tt.name, func(t *testing.T) { - // imp := &Importer{} - // - // aux, err := importer.ParseYAML([]byte(tt.yaml)) - // require.NoError(t, err) - // require.NoError(t, imp.CastResourcesSet("allow", aux)) - // require.Equal(t, tt.rules, imp.rules) - // }) - // } -} diff --git a/pkg/settings/importer.go b/pkg/settings/importer.go deleted file mode 100644 index 31d103193..000000000 --- a/pkg/settings/importer.go +++ /dev/null @@ -1,54 +0,0 @@ -package settings - -import ( - "context" - "fmt" - "github.com/cortezaproject/corteza-server/pkg/deinterfacer" - "github.com/cortezaproject/corteza-server/system/types" -) - -type ( - Importer struct { - settings types.SettingValueSet - } - - ImportKeeper interface { - BulkSet(ctx context.Context, vv types.SettingValueSet) (err error) - } -) - -func NewImporter() *Importer { - return &Importer{} -} - -// CastSet - resolves settings: -// [ , ... ] -func (imp *Importer) CastSet(settings interface{}) (err error) { - if !deinterfacer.IsMap(settings) { - return fmt.Errorf("expecting map of settings") - } - - return deinterfacer.Each(settings, func(_ int, name string, value interface{}) error { - return imp.addSetting(name, value) - }) -} - -func (imp *Importer) addSetting(name string, value interface{}) (err error) { - // Convert to interface{}, since json.Marshal cant handle map[interface{}]interface{} - setting := &types.SettingValue{Name: name} - - if err = setting.SetValue(deinterfacer.Simplify(value)); err != nil { - return err - } - - imp.settings = append(imp.settings, setting) - return nil -} - -func (imp *Importer) Store(ctx context.Context, k ImportKeeper) (err error) { - return k.BulkSet(ctx, imp.settings) -} - -func (imp *Importer) GetValues() types.SettingValueSet { - return imp.settings -} diff --git a/pkg/settings/importer_test.go b/pkg/settings/importer_test.go deleted file mode 100644 index abc9ea60b..000000000 --- a/pkg/settings/importer_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package settings - -import ( - "os" - "strings" - "testing" - - "github.com/jmoiron/sqlx/types" - "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" -) - -func TestSettingImport_CastSet(t *testing.T) { - var aux interface{} - req := require.New(t) - f, err := os.Open("testdata/settings.yaml") - defer f.Close() - req.NoError(err) - req.NoError(yaml.NewDecoder(f).Decode(&aux)) - req.NotNil(aux) - - imp := NewImporter() - err = imp.CastSet(aux) - req.Nil(err) - - tests := map[string]string{ - "v_string": "\"string\"", - "v_float": "12.34", - "v_int-as-float": "123", - "v_bool_true": "true", - "v_bool_false": "false", - "v_slice": "[1,1.23,\"string\",true,false]", - "v_map": "{\"k1\":\"string\",\"k2\":1,\"k3\":1.23,\"k4\":true,\"k5\":false}", - } - - for k, test := range tests { - t.Run(k, func(t *testing.T) { - v := imp.settings.First(k) - vv, _ := types.JSONText(test).MarshalJSON() - ee, _ := v.Value.MarshalJSON() - req.Equal(strings.TrimSpace(string(ee)), strings.TrimSpace(string(vv))) - }) - } -} diff --git a/pkg/settings/testdata/settings.yaml b/pkg/settings/testdata/settings.yaml deleted file mode 100644 index 03778e70b..000000000 --- a/pkg/settings/testdata/settings.yaml +++ /dev/null @@ -1,17 +0,0 @@ -v_string: string -v_float: 12.34 -v_int-as-float: 123 -v_bool_true: true -v_bool_false: false -v_slice: -- 1 -- 1.23 -- string -- true -- false -v_map: - k1: string - k2: 1 - k3: 1.23 - k4: true - k5: false