DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Merge branch 'feature-settings-impexp'

Browse Source
This commit is contained in:
Tomaž Jerman
2019-10-16 12:51:28 +02:00
parent 9bf78a3f40 729eae9a6e
commit 0b3a19dd0f
25 changed files with 782 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose/commands/exporter.go
+139 -155
View File
@@ -22,6 +22,9 @@ import (
"github.com/cortezaproject/corteza-server/pkg/deinterfacer"
"github.com/cortezaproject/corteza-server/pkg/handle"
"github.com/cortezaproject/corteza-server/pkg/permissions"
"github.com/cortezaproject/corteza-server/pkg/settings"
intSettings "github.com/cortezaproject/corteza-server/pkg/settings"
sysExporter "github.com/cortezaproject/corteza-server/system/exporter"
sysTypes "github.com/cortezaproject/corteza-server/system/types"
)
@@ -39,105 +42,144 @@ func Exporter(ctx context.Context, c *cli.Config) *cobra.Command {
var (
nsFlag = cmd.Flags().Lookup("namespace").Value.String()
ns *types.Namespace
err error
sFlag = cmd.Flags().Lookup("settings").Changed
pFlag = cmd.Flags().Lookup("permissions").Changed
out = Compose{
out = &Compose{
Namespaces: map[string]Namespace{},
Settings: yaml.MapSlice{},
}
nsOut = Namespace{}
)
if nsFlag == "" {
cli.HandleError(errors.New("Specify namespace to export from"))
if nsFlag == "" && !sFlag && !pFlag {
cli.HandleError(errors.New("Specify namespace or setting or permissions flag"))
}
if namespaceID, _ := strconv.ParseUint(nsFlag, 10, 64); namespaceID > 0 {
ns, err = service.DefaultNamespace.FindByID(namespaceID)
if err != repository.ErrNamespaceNotFound {
cli.HandleError(err)
}
} else if ns, err = service.DefaultNamespace.FindByHandle(nsFlag); err != nil {
if err != repository.ErrNamespaceNotFound {
cli.HandleError(err)
}
if nsFlag != "" {
nsExporter(ctx, out, nsFlag, args)
}
// roles, err = service.DefaultSystemRole.Find(ctx)
// cli.HandleError(err)
// At the moment, we can not load roles from system service
// so we'll just use static set of known roles
//
// Roles are use for resolving access control
roles = sysTypes.RoleSet{
&sysTypes.Role{ID: permissions.EveryoneRoleID, Handle: "everyone"},
&sysTypes.Role{ID: permissions.AdminsRoleID, Handle: "admins"},
if sFlag {
settingExporter(ctx, out)
}
modules, _, err := service.DefaultModule.Find(types.ModuleFilter{NamespaceID: ns.ID})
cli.HandleError(err)
pages, _, err := service.DefaultPage.Find(types.PageFilter{NamespaceID: ns.ID})
cli.HandleError(err)
charts, _, err := service.DefaultChart.Find(types.ChartFilter{NamespaceID: ns.ID})
cli.HandleError(err)
scripts, _, err := service.DefaultInternalAutomationManager.FindScripts(ctx, automation.ScriptFilter{})
cli.HandleError(err)
triggers, _, err := service.DefaultInternalAutomationManager.FindTriggers(ctx, automation.TriggerFilter{})
cli.HandleError(err)
scripts, _ = scripts.Filter(func(script *automation.Script) (b bool, e error) {
return script.NamespaceID == ns.ID, nil
})
if pFlag {
permissionExporter(ctx, out)
}
y := yaml.NewEncoder(cmd.OutOrStdout())
// nsOut.Name = ns.Name
// nsOut.Handle = ns.Slug
// nsOut.Enabled = ns.Enabled
// nsOut.Meta = ns.Meta
//
// nsOut.Allow = expResourcePermissions(permissions.Allow, ns.PermissionResource())
// nsOut.Deny = expResourcePermissions(permissions.Deny, ns.PermissionResource())
for _, arg := range args {
switch arg {
case "module", "modules":
nsOut.Modules = expModules(modules)
case "chart", "charts":
nsOut.Charts = expCharts(charts, modules)
case "page", "pages":
nsOut.Pages = expPages(0, pages, modules, charts, scripts)
case "scripts", "triggers", "automation":
nsOut.Scripts = expAutomation(scripts, triggers, modules)
case "allow", "deny", "permission", "permissions":
out.Allow = expServicePermissions(permissions.Allow)
out.Deny = expServicePermissions(permissions.Deny)
}
}
// out.Namespaces[ns.Slug] = nsOut
nsOut.Namespace = ns.Slug
_, _ = y, out
cli.HandleError(y.Encode(nsOut))
cli.HandleError(y.Encode(out))
},
}
cmd.Flags().String("namespace", "", "Export namespace resources (by ID or string)")
cmd.Flags().BoolP("settings", "s", false, "Export settings")
cmd.Flags().BoolP("permissions", "p", false, "Export system permissions")
return cmd
}
func nsExporter(ctx context.Context, out *Compose, nsFlag string, args []string) {
var (
ns *types.Namespace
err error
nsOut = Namespace{}
)
if namespaceID, _ := strconv.ParseUint(nsFlag, 10, 64); namespaceID > 0 {
ns, err = service.DefaultNamespace.FindByID(namespaceID)
if err != repository.ErrNamespaceNotFound {
cli.HandleError(err)
}
} else if ns, err = service.DefaultNamespace.FindByHandle(nsFlag); err != nil {
if err != repository.ErrNamespaceNotFound {
cli.HandleError(err)
}
}
// roles, err = service.DefaultSystemRole.Find(ctx)
// cli.HandleError(err)
// At the moment, we can not load roles from system service
// so we'll just use static set of known roles
//
// Roles are use for resolving access control
roles = sysTypes.RoleSet{
&sysTypes.Role{ID: permissions.EveryoneRoleID, Handle: "everyone"},
&sysTypes.Role{ID: permissions.AdminsRoleID, Handle: "admins"},
}
modules, _, err := service.DefaultModule.Find(types.ModuleFilter{NamespaceID: ns.ID})
cli.HandleError(err)
pages, _, err := service.DefaultPage.Find(types.PageFilter{NamespaceID: ns.ID})
cli.HandleError(err)
charts, _, err := service.DefaultChart.Find(types.ChartFilter{NamespaceID: ns.ID})
cli.HandleError(err)
scripts, _, err := service.DefaultInternalAutomationManager.FindScripts(ctx, automation.ScriptFilter{})
cli.HandleError(err)
triggers, _, err := service.DefaultInternalAutomationManager.FindTriggers(ctx, automation.TriggerFilter{})
cli.HandleError(err)
scripts, _ = scripts.Filter(func(script *automation.Script) (b bool, e error) {
return script.NamespaceID == ns.ID, nil
})
// nsOut.Name = ns.Name
// nsOut.Handle = ns.Slug
// nsOut.Enabled = ns.Enabled
// nsOut.Meta = ns.Meta
//
// nsOut.Allow = sysExporter.ExportableResourcePermissions(roles, service.DefaultPermissions, permissions.Allow, ns.PermissionResource())
// nsOut.Deny = sysExporter.ExportableResourcePermissions(roles, service.DefaultPermissions, permissions.Deny, ns.PermissionResource())
for _, arg := range args {
switch arg {
case "module", "modules":
nsOut.Modules = expModules(modules)
case "chart", "charts":
nsOut.Charts = expCharts(charts, modules)
case "page", "pages":
nsOut.Pages = expPages(0, pages, modules, charts, scripts)
case "scripts", "triggers", "automation":
nsOut.Scripts = expAutomation(scripts, triggers, modules)
}
}
nsOut.Namespace = ns.Slug
out.Namespaces[ns.Slug] = nsOut
}
func settingExporter(ctx context.Context, out *Compose) {
var (
err error
)
ss, err := service.DefaultSettings.FindByPrefix("")
cli.HandleError(err)
out.Settings = settings.Export(ss)
}
func permissionExporter(ctx context.Context, out *Compose) {
roles := sysTypes.RoleSet{
&sysTypes.Role{ID: permissions.EveryoneRoleID, Handle: "everyone"},
&sysTypes.Role{ID: permissions.AdminsRoleID, Handle: "admins"},
}
out.Allow = sysExporter.ExportableServicePermissions(roles, service.DefaultPermissions, permissions.Allow)
out.Deny = sysExporter.ExportableServicePermissions(roles, service.DefaultPermissions, permissions.Deny)
}
// This is PoC for exporting compose resources
//
type (
Compose struct {
Namespaces map[string]Namespace
Namespaces map[string]Namespace `yaml:",omitempty"`
Settings yaml.MapSlice `yaml:",omitempty"`
Allow map[string]map[string][]string `yaml:",omitempty"`
Deny map[string]map[string][]string `yaml:",omitempty"`
@@ -242,6 +284,19 @@ var (
pagesHandles = make(map[string]bool)
)
func expSettings(ss intSettings.ValueSet) (o yaml.MapSlice) {
o = yaml.MapSlice{}
for _, s := range ss {
setting := yaml.MapItem{
Key: s.Name,
Value: s.Value.String(),
}
o = append(o, setting)
}
return o
}
func expModules(mm types.ModuleSet) (o map[string]Module) {
o = map[string]Module{}
@@ -250,8 +305,8 @@ func expModules(mm types.ModuleSet) (o map[string]Module) {
Name: m.Name,
Fields: expModuleFields(m.Fields, mm),
Allow: expResourcePermissions(permissions.Allow, types.ModulePermissionResource),
Deny: expResourcePermissions(permissions.Deny, types.ModulePermissionResource),
Allow: sysExporter.ExportableResourcePermissions(roles, service.DefaultPermissions, permissions.Allow, types.ModulePermissionResource),
Deny: sysExporter.ExportableResourcePermissions(roles, service.DefaultPermissions, permissions.Deny, types.ModulePermissionResource),
}
if meta := expModuleMetaCleanup(m.Meta); len(meta) > 0 {
@@ -299,8 +354,8 @@ func expModuleFields(ff types.ModuleFieldSet, modules types.ModuleSet) (o yaml.M
Visible: f.Visible,
Multi: f.Multi,
Allow: expResourcePermissions(permissions.Allow, types.ModuleFieldPermissionResource),
Deny: expResourcePermissions(permissions.Deny, types.ModuleFieldPermissionResource),
Allow: sysExporter.ExportableResourcePermissions(roles, service.DefaultPermissions, permissions.Allow, types.ModuleFieldPermissionResource),
Deny: sysExporter.ExportableResourcePermissions(roles, service.DefaultPermissions, permissions.Deny, types.ModuleFieldPermissionResource),
},
}
}
@@ -379,8 +434,8 @@ func expPages(parentID uint64, pages types.PageSet, modules types.ModuleSet, cha
Pages: expPages(child.ID, pages, modules, charts, scripts),
Visible: child.Visible,
Allow: expResourcePermissions(permissions.Allow, types.PagePermissionResource),
Deny: expResourcePermissions(permissions.Deny, types.PagePermissionResource),
Allow: sysExporter.ExportableResourcePermissions(roles, service.DefaultPermissions, permissions.Allow, types.PagePermissionResource),
Deny: sysExporter.ExportableResourcePermissions(roles, service.DefaultPermissions, permissions.Deny, types.PagePermissionResource),
}
if child.ModuleID > 0 {
@@ -533,8 +588,8 @@ func expCharts(charts types.ChartSet, modules types.ModuleSet) (o map[string]Cha
Name: c.Name,
Config: ChartConfig{Reports: make([]map[string]interface{}, len(c.Config.Reports))},
Allow: expResourcePermissions(permissions.Allow, types.ChartPermissionResource),
Deny: expResourcePermissions(permissions.Deny, types.ChartPermissionResource),
Allow: sysExporter.ExportableResourcePermissions(roles, service.DefaultPermissions, permissions.Allow, types.ChartPermissionResource),
Deny: sysExporter.ExportableResourcePermissions(roles, service.DefaultPermissions, permissions.Deny, types.ChartPermissionResource),
}
for i, r := range c.Config.Reports {
@@ -585,8 +640,8 @@ func expAutomation(ss automation.ScriptSet, tt automation.TriggerSet, mm types.M
Triggers: []map[string]interface{}{},
Allow: expResourcePermissions(permissions.Allow, types.AutomationScriptPermissionResource),
Deny: expResourcePermissions(permissions.Deny, types.AutomationScriptPermissionResource),
Allow: sysExporter.ExportableResourcePermissions(roles, service.DefaultPermissions, permissions.Allow, types.AutomationScriptPermissionResource),
Deny: sysExporter.ExportableResourcePermissions(roles, service.DefaultPermissions, permissions.Deny, types.AutomationScriptPermissionResource),
}
handle := makeHandleFromName(s.Name, "", "automation-script-%d", s.ID)
@@ -638,77 +693,6 @@ func expAutomation(ss automation.ScriptSet, tt automation.TriggerSet, mm types.M
return out
}
func expServicePermissions(access permissions.Access) map[string]map[string][]string {
var (
has bool
res string
rules permissions.RuleSet
sp = make(map[string]map[string][]string)
)
for _, r := range roles {
rules = service.DefaultPermissions.FindRulesByRoleID(r.ID)
if len(rules) == 0 {
continue
}
for _, rule := range rules {
if rule.Resource.GetService() != rule.Resource && !rule.Resource.HasWildcard() {
continue
}
res = strings.TrimRight(rule.Resource.String(), ":*")
if _, has = sp[r.Handle]; !has {
sp[r.Handle] = map[string][]string{}
}
if _, has = sp[r.Handle][res]; !has {
sp[r.Handle][res] = make([]string, 0)
}
sp[r.Handle][res] = append(sp[r.Handle][res], rule.Operation.String())
}
}
return sp
}
func expResourcePermissions(access permissions.Access, resource permissions.Resource) map[string][]string {
var (
has bool
rules permissions.RuleSet
sp = make(map[string][]string)
)
for _, r := range roles {
rules = service.DefaultPermissions.FindRulesByRoleID(r.ID)
if len(rules) == 0 {
continue
}
for _, rule := range rules {
if rule.Resource != resource {
continue
}
if rule.Access != access {
continue
}
if _, has = sp[r.Handle]; !has {
sp[r.Handle] = make([]string, 0)
}
sp[r.Handle] = append(sp[r.Handle], rule.Operation.String())
}
}
return sp
}
func makeHandleFromName(name, currentHandle, def string, id uint64) string {
if currentHandle != "" {
return currentHandle
compose/importer/default.go
+9 -5
View File
@@ -9,6 +9,7 @@ import (
"github.com/cortezaproject/corteza-server/compose/service"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/permissions"
"github.com/cortezaproject/corteza-server/pkg/settings"
sysTypes "github.com/cortezaproject/corteza-server/system/types"
)
@@ -22,7 +23,9 @@ func Import(ctx context.Context, ns *types.Namespace, ff ...io.Reader) (err erro
service.DefaultChart.With(ctx),
service.DefaultPage.With(ctx),
service.DefaultInternalAutomationManager,
permissions.NewImporter(service.DefaultAccessControl.Whitelist()),
settings.NewImporter(),
)
// At the moment, we can not load roles from system service
@@ -41,12 +44,12 @@ func Import(ctx context.Context, ns *types.Namespace, ff ...io.Reader) (err erro
}
if ns != nil {
// If we're importing with --namespace switch,
// we're going to import all into one NS
err = imp.GetNamespaceImporter().Cast(ns.Slug, aux)
if mp, ok := aux.(map[interface{}]interface{}); ok && mp["namespaces"] != nil {
err = imp.GetNamespaceImporter().Cast(ns.Slug, mp["namespaces"])
} else {
err = imp.GetNamespaceImporter().Cast(ns.Slug, aux)
}
} else {
// importing one or more namespaces
err = imp.Cast(aux)
}
@@ -71,6 +74,7 @@ func Import(ctx context.Context, ns *types.Namespace, ff ...io.Reader) (err erro
service.DefaultRecord.With(ctx),
service.DefaultInternalAutomationManager,
service.DefaultAccessControl,
service.DefaultSettings.With(ctx),
roles,
)
}
compose/importer/importer.go
+13 -1
View File
@@ -11,6 +11,7 @@ import (
"github.com/cortezaproject/corteza-server/pkg/deinterfacer"
"github.com/cortezaproject/corteza-server/pkg/importer"
"github.com/cortezaproject/corteza-server/pkg/permissions"
"github.com/cortezaproject/corteza-server/pkg/settings"
sysTypes "github.com/cortezaproject/corteza-server/system/types"
)
@@ -25,6 +26,7 @@ type (
automationFinder automationFinder
permissions importer.PermissionImporter
settings importer.SettingImporter
}
moduleKeeper interface {
@@ -53,7 +55,7 @@ type (
}
)
func NewImporter(nsf namespaceFinder, mf moduleFinder, cf chartFinder, pf pageFinder, af automationFinder, p importer.PermissionImporter) *Importer {
func NewImporter(nsf namespaceFinder, mf moduleFinder, cf chartFinder, pf pageFinder, af automationFinder, p importer.PermissionImporter, s importer.SettingImporter) *Importer {
imp := &Importer{
namespaceFinder: nsf,
moduleFinder: mf,
@@ -62,6 +64,7 @@ func NewImporter(nsf namespaceFinder, mf moduleFinder, cf chartFinder, pf pageFi
automationFinder: af,
permissions: p,
settings: s,
}
imp.namespaces = NewNamespaceImporter(imp)
@@ -110,6 +113,9 @@ func (imp *Importer) Cast(def interface{}) (err error) {
case "namespace":
return imp.namespaces.CastSet([]interface{}{val})
case "settings":
return imp.settings.CastSet(val)
case "allow", "deny":
return imp.permissions.CastResourcesSet(key, val)
@@ -130,6 +136,7 @@ func (imp *Importer) Store(
rStore recordKeeper,
asStore automationScriptKeeper,
pk permissions.ImportKeeper,
sk settings.ImportKeeper,
roles sysTypes.RoleSet,
) (err error) {
err = imp.namespaces.Store(ctx, nsStore, mStore, cStore, pStore, rStore, asStore)
@@ -148,5 +155,10 @@ func (imp *Importer) Store(
return errors.Wrap(err, "could not import permissions")
}
err = imp.settings.Store(ctx, sk)
if err != nil {
return errors.Wrap(err, "could not import settings")
}
return nil
}
compose/importer/importer_test.go
+6
View File
@@ -21,4 +21,10 @@ func TestImporter_Cast(t *testing.T) {
req.Equal("Module1", i.namespaces.modules["foo"].set.FindByHandle("m1").Name)
})
impFixTester(t, "importer_with_settings", func(t *testing.T, i *Importer) {
req := require.New(t)
req.NotNil(i)
})
}
compose/importer/main_test.go
+4 -1
View File
@@ -11,6 +11,7 @@ import (
"github.com/cortezaproject/corteza-server/compose/service"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/permissions"
"github.com/cortezaproject/corteza-server/pkg/settings"
)
var (
@@ -23,6 +24,7 @@ var (
// Add namespace to the stack, make sure importer can find it
pi *permissions.Importer
st *settings.Importer
imp *Importer
)
@@ -35,8 +37,9 @@ func TestMain(m *testing.M) {
func resetMocks() {
// whitelist = nil, anything can be added
pi = permissions.NewImporter(service.AccessControl(nil).Whitelist())
st = settings.NewImporter()
imp = NewImporter(nil, nil, nil, nil, nil, pi)
imp = NewImporter(nil, nil, nil, nil, nil, pi, st)
}
compose/importer/testdata/importer_with_settings.yaml
Vendored
+8
View File
@@ -0,0 +1,8 @@
settings:
s1: v1
s2:
- v2
- v3
s3:
ss1: v4
ss2: v5
compose/service/service.go
+3 -3
View File
@@ -111,9 +111,9 @@ func Init(ctx context.Context, log *zap.Logger, c Config) (err error) {
zap.Error(err))
}
if err != nil {
return err
}
if err != nil {
return err
}
}
// Permissions, access control
messaging/commands/exporter.go
+93
View File
@@ -0,0 +1,93 @@
package commands
import (
"context"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/cortezaproject/corteza-server/messaging/service"
"github.com/cortezaproject/corteza-server/pkg/auth"
"github.com/cortezaproject/corteza-server/pkg/cli"
"github.com/cortezaproject/corteza-server/pkg/permissions"
"github.com/cortezaproject/corteza-server/pkg/settings"
sysExporter "github.com/cortezaproject/corteza-server/system/exporter"
sysTypes "github.com/cortezaproject/corteza-server/system/types"
)
func Exporter(ctx context.Context, c *cli.Config) *cobra.Command {
cmd := &cobra.Command{
Use: "export",
Short: "Export",
Long: `Export messaging resources`,
Run: func(cmd *cobra.Command, args []string) {
c.InitServices(ctx, c)
ctx = auth.SetSuperUserContext(ctx)
var (
sFlag = cmd.Flags().Lookup("settings").Changed
pFlag = cmd.Flags().Lookup("permissions").Changed
out = &Messaging{
Settings: yaml.MapSlice{},
}
)
if !sFlag && !pFlag {
cli.HandleError(errors.New("Specify setting or permissions flag"))
}
if pFlag {
permissionExporter(ctx, out)
}
if sFlag {
settingExporter(ctx, out)
}
y := yaml.NewEncoder(cmd.OutOrStdout())
cli.HandleError(y.Encode(out))
},
}
cmd.Flags().BoolP("settings", "s", false, "Export settings")
cmd.Flags().BoolP("permissions", "p", false, "Export system permissions")
return cmd
}
func permissionExporter(ctx context.Context, out *Messaging) {
roles := sysTypes.RoleSet{
&sysTypes.Role{ID: permissions.EveryoneRoleID, Handle: "everyone"},
&sysTypes.Role{ID: permissions.AdminsRoleID, Handle: "admins"},
}
out.Allow = sysExporter.ExportableServicePermissions(roles, service.DefaultPermissions, permissions.Allow)
out.Deny = sysExporter.ExportableServicePermissions(roles, service.DefaultPermissions, permissions.Deny)
}
func settingExporter(ctx context.Context, out *Messaging) {
var (
err error
)
ss, err := service.DefaultSettings.FindByPrefix("")
cli.HandleError(err)
out.Settings = settings.Export(ss)
}
// This is PoC for exporting messaging resources
//
type (
Messaging struct {
Settings yaml.MapSlice `yaml:",omitempty"`
Allow map[string]map[string][]string `yaml:",omitempty"`
Deny map[string]map[string][]string `yaml:",omitempty"`
}
)
messaging/commands/importer.go
+45
View File
@@ -0,0 +1,45 @@
package commands
import (
"context"
"io"
"os"
"github.com/spf13/cobra"
"github.com/cortezaproject/corteza-server/messaging/importer"
"github.com/cortezaproject/corteza-server/pkg/auth"
"github.com/cortezaproject/corteza-server/pkg/cli"
)
func Importer(ctx context.Context, c *cli.Config) *cobra.Command {
cmd := &cobra.Command{
Use: "import",
Short: "Import",
Run: func(cmd *cobra.Command, args []string) {
c.InitServices(ctx, c)
var (
ff []io.Reader
err error
)
ctx = auth.SetSuperUserContext(ctx)
if len(args) > 0 {
ff = make([]io.Reader, len(args))
for a, arg := range args {
ff[a], err = os.Open(arg)
cli.HandleError(err)
}
cli.HandleError(importer.Import(ctx, ff...))
} else {
cli.HandleError(importer.Import(ctx, os.Stdin))
}
},
}
return cmd
}
messaging/importer/default.go
+3
View File
@@ -9,6 +9,7 @@ import (
"github.com/cortezaproject/corteza-server/messaging/service"
"github.com/cortezaproject/corteza-server/messaging/types"
"github.com/cortezaproject/corteza-server/pkg/permissions"
"github.com/cortezaproject/corteza-server/pkg/settings"
sysTypes "github.com/cortezaproject/corteza-server/system/types"
)
@@ -28,6 +29,7 @@ func Import(ctx context.Context, ff ...io.Reader) (err error) {
p = permissions.NewImporter(service.DefaultAccessControl.Whitelist())
imp = NewImporter(
p,
settings.NewImporter(),
NewChannelImport(p, cc),
)
@@ -56,6 +58,7 @@ func Import(ctx context.Context, ff ...io.Reader) (err error) {
ctx,
service.DefaultChannel.With(ctx),
service.DefaultAccessControl,
service.DefaultSettings.With(ctx),
roles,
)
}
messaging/importer/importer.go
+14 -2
View File
@@ -4,6 +4,8 @@ import (
"context"
"fmt"
"github.com/cortezaproject/corteza-server/pkg/settings"
"github.com/cortezaproject/corteza-server/messaging/types"
"github.com/cortezaproject/corteza-server/pkg/deinterfacer"
"github.com/cortezaproject/corteza-server/pkg/importer"
@@ -15,6 +17,7 @@ type (
Importer struct {
channels *Channel
permissions importer.PermissionImporter
settings importer.SettingImporter
}
channelFinder interface {
@@ -22,10 +25,11 @@ type (
}
)
func NewImporter(p importer.PermissionImporter, ci *Channel) *Importer {
func NewImporter(p importer.PermissionImporter, s importer.SettingImporter, ci *Channel) *Importer {
return &Importer{
channels: ci,
permissions: p,
settings: s,
}
}
@@ -37,6 +41,9 @@ func (imp *Importer) Cast(in interface{}) (err error) {
case "channel":
return imp.channels.CastSet([]interface{}{val})
case "settings":
return imp.settings.CastSet(val)
case "allow", "deny":
return imp.permissions.CastResourcesSet(key, val)
@@ -48,7 +55,7 @@ func (imp *Importer) Cast(in interface{}) (err error) {
})
}
func (imp *Importer) Store(ctx context.Context, rk channelKeeper, pk permissions.ImportKeeper, roles sysTypes.RoleSet) (err error) {
func (imp *Importer) Store(ctx context.Context, rk channelKeeper, pk permissions.ImportKeeper, sk settings.ImportKeeper, roles sysTypes.RoleSet) (err error) {
err = imp.channels.Store(ctx, rk)
if err != nil {
return
@@ -65,5 +72,10 @@ func (imp *Importer) Store(ctx context.Context, rk channelKeeper, pk permissions
return
}
err = imp.settings.Store(ctx, sk)
if err != nil {
return
}
return nil
}
messaging/importer/main_test.go
+4 -1
View File
@@ -9,11 +9,13 @@ import (
"gopkg.in/yaml.v2"
"github.com/cortezaproject/corteza-server/pkg/permissions"
"github.com/cortezaproject/corteza-server/pkg/settings"
"github.com/cortezaproject/corteza-server/system/service"
)
var (
pi *permissions.Importer
si *settings.Importer
imp *Importer
)
@@ -26,9 +28,10 @@ func TestMain(m *testing.M) {
func resetMocks() {
// whitelist = nil, anything can be added
pi = permissions.NewImporter(service.AccessControl(nil).Whitelist())
si = settings.NewImporter()
imp = NewImporter(
pi,
si,
NewChannelImport(pi, nil),
)
}
messaging/messaging.go
+10
View File
@@ -8,6 +8,7 @@ import (
"github.com/spf13/cobra"
"github.com/titpetric/factory"
"github.com/cortezaproject/corteza-server/messaging/commands"
migrate "github.com/cortezaproject/corteza-server/messaging/db"
"github.com/cortezaproject/corteza-server/messaging/rest"
"github.com/cortezaproject/corteza-server/messaging/service"
@@ -79,6 +80,15 @@ func Configure() *cli.Config {
func(r chi.Router) { ws.ApiServerRoutes(r) },
},
AdtSubCommands: cli.CommandMakers{
func(ctx context.Context, c *cli.Config) *cobra.Command {
return commands.Importer(ctx, c)
},
func(ctx context.Context, c *cli.Config) *cobra.Command {
return commands.Exporter(ctx, c)
},
},
ProvisionMigrateDatabase: cli.Runners{
func(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
var db, err = factory.Database.Get(messaging)
pkg/importer/importer.go
+6
View File
@@ -4,6 +4,7 @@ import (
"context"
"github.com/cortezaproject/corteza-server/pkg/permissions"
"github.com/cortezaproject/corteza-server/pkg/settings"
)
type (
@@ -19,6 +20,11 @@ type (
UpdateRoles(handle string, ID uint64)
Store(context.Context, permissions.ImportKeeper) error
}
SettingImporter interface {
CastSet(interface{}) error
Store(context.Context, settings.ImportKeeper) error
}
)
func NormalizeHandle(in string) string {
pkg/settings/exporter.go
+24
View File
@@ -0,0 +1,24 @@
package settings
import (
"encoding/json"
"gopkg.in/yaml.v2"
)
// Export transforms a given ValueSet into a yaml exportable structure
func Export(ss ValueSet) (o yaml.MapSlice) {
o = yaml.MapSlice{}
for _, s := range ss {
var v interface{}
json.Unmarshal(s.Value, &v)
setting := yaml.MapItem{
Key: s.Name,
Value: v,
}
o = append(o, setting)
}
return o
}
pkg/settings/exporter_test.go
+64
View File
@@ -0,0 +1,64 @@
package settings
import (
"testing"
"github.com/jmoiron/sqlx/types"
)
func TestExport(t *testing.T) {
ss := ValueSet{}
ss = append(ss, ([]*Value{
&Value{
Name: "v_string",
Value: types.JSONText("\"string\""),
},
&Value{
Name: "v_float",
Value: types.JSONText("123"),
},
&Value{
Name: "v_float",
Value: types.JSONText("12.34"),
},
&Value{
Name: "v_bool",
Value: types.JSONText("true"),
},
&Value{
Name: "v_slice",
Value: types.JSONText("[1, \"string\", true]"),
},
&Value{
Name: "v_map",
Value: types.JSONText("{\"k1\": \"v1\",\"k2\": 2}"),
},
})...)
tt := Export(ss)
if _, ok := tt[0].Value.(string); !ok {
t.Errorf("Expecting %v to be string", tt[0].Value)
}
if _, ok := tt[1].Value.(float64); !ok {
t.Errorf("Expecting %v to be float64", tt[1].Value)
}
if _, ok := tt[2].Value.(float64); !ok {
t.Errorf("Expecting %v to be float64", tt[2].Value)
}
if _, ok := tt[3].Value.(bool); !ok {
t.Errorf("Expecting %v to be bool", tt[3].Value)
}
if _, ok := tt[4].Value.([]interface{}); !ok {
t.Errorf("Expecting %v to be []interface{}", tt[4].Value)
}
if _, ok := tt[5].Value.(map[string]interface{}); !ok {
t.Errorf("Expecting %v to be map[string]interface {}", tt[5].Value)
}
}
pkg/settings/importer.go
+56
View File
@@ -0,0 +1,56 @@
package settings
import (
"context"
"encoding/json"
"github.com/pkg/errors"
"github.com/cortezaproject/corteza-server/pkg/deinterfacer"
)
type (
Importer struct {
settings ValueSet
}
ImportKeeper interface {
BulkSet(vv ValueSet) (err error)
}
)
func NewImporter() *Importer {
return &Importer{}
}
// CastSet - resolves settings:
// <ValueSet> [ <Value>, ... ]
func (imp *Importer) CastSet(settings interface{}) (err error) {
if !deinterfacer.IsMap(settings) {
return errors.New("expecting map of settings")
}
return deinterfacer.Each(settings, func(_ int, name string, value interface{}) error {
return imp.addSetting(name, value)
})
}
func (imp *Importer) addSetting(name string, value interface{}) (err error) {
// Convert to interface{}, since json.Marshal cant handle map[interface{}]interface{}
v, err := json.Marshal(deinterfacer.Simplify(value))
if err != nil {
return err
}
setting := &Value{
Name: name,
Value: v,
}
imp.settings = append(imp.settings, setting)
return nil
}
func (imp *Importer) Store(ctx context.Context, k ImportKeeper) (err error) {
return k.BulkSet(imp.settings)
}
pkg/settings/importer_test.go
+64
View File
@@ -0,0 +1,64 @@
package settings
import (
"os"
"testing"
"github.com/jmoiron/sqlx/types"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v2"
)
func TestSettingImport_CastSet(t *testing.T) {
type tv struct {
name string
value string
}
var aux interface{}
req := require.New(t)
f, err := os.Open("testdata/settings.yaml")
defer f.Close()
req.NoError(err)
req.NoError(yaml.NewDecoder(f).Decode(&aux))
req.NotNil(aux)
imp := NewImporter()
err = imp.CastSet(aux)
req.Nil(err)
req.Len(imp.settings, 6)
tests := []tv{
tv{
name: "array",
value: "[\"v1\",\"v2\"]",
},
tv{
name: "object",
value: "{\"k1\":\"v1\",\"k2\":\"v2\"}",
},
tv{
name: "plain_b",
value: "true",
},
tv{
name: "plain_double",
value: "12.34",
},
tv{
name: "plain_double_2",
value: "12",
},
tv{
name: "plain_s",
value: "\"string\"",
},
}
for _, t := range tests {
v := imp.settings.First(t.name)
vv, _ := types.JSONText(t.value).MarshalJSON()
ee, _ := v.Value.MarshalJSON()
req.Equal(vv, ee)
}
}
pkg/settings/testdata/settings.yaml
Vendored
+10
View File
@@ -0,0 +1,10 @@
array:
- v1
- v2
object:
k1: v1
k2: v2
plain_b: true
plain_double: 12.34
plain_double_2: 12
plain_s: string
system/commands/exporter.go
+93
View File
@@ -0,0 +1,93 @@
package commands
import (
"context"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/cortezaproject/corteza-server/pkg/auth"
"github.com/cortezaproject/corteza-server/pkg/cli"
"github.com/cortezaproject/corteza-server/pkg/permissions"
"github.com/cortezaproject/corteza-server/pkg/settings"
sysExporter "github.com/cortezaproject/corteza-server/system/exporter"
"github.com/cortezaproject/corteza-server/system/service"
sysTypes "github.com/cortezaproject/corteza-server/system/types"
)
func Exporter(ctx context.Context, c *cli.Config) *cobra.Command {
cmd := &cobra.Command{
Use: "export",
Short: "Export",
Long: `Export system resources`,
Run: func(cmd *cobra.Command, args []string) {
c.InitServices(ctx, c)
ctx = auth.SetSuperUserContext(ctx)
var (
sFlag = cmd.Flags().Lookup("settings").Changed
pFlag = cmd.Flags().Lookup("permissions").Changed
out = &System{
Settings: yaml.MapSlice{},
}
)
if !sFlag && !pFlag {
cli.HandleError(errors.New("Specify setting or permissions flag"))
}
if pFlag {
permissionExporter(ctx, out)
}
if sFlag {
settingExporter(ctx, out)
}
y := yaml.NewEncoder(cmd.OutOrStdout())
cli.HandleError(y.Encode(out))
},
}
cmd.Flags().BoolP("settings", "s", false, "Export settings")
cmd.Flags().BoolP("permissions", "p", false, "Export system permissions")
return cmd
}
func permissionExporter(ctx context.Context, out *System) {
roles := sysTypes.RoleSet{
&sysTypes.Role{ID: permissions.EveryoneRoleID, Handle: "everyone"},
&sysTypes.Role{ID: permissions.AdminsRoleID, Handle: "admins"},
}
out.Allow = sysExporter.ExportableServicePermissions(roles, service.DefaultPermissions, permissions.Allow)
out.Deny = sysExporter.ExportableServicePermissions(roles, service.DefaultPermissions, permissions.Deny)
}
func settingExporter(ctx context.Context, out *System) {
var (
err error
)
ss, err := service.DefaultSettings.FindByPrefix("")
cli.HandleError(err)
out.Settings = settings.Export(ss)
}
// This is PoC for exporting system resources
//
type (
System struct {
Settings yaml.MapSlice `yaml:",omitempty"`
Allow map[string]map[string][]string `yaml:",omitempty"`
Deny map[string]map[string][]string `yaml:",omitempty"`
}
)
system/exporter/export.go
+89
View File
@@ -0,0 +1,89 @@
package exporter
import (
"strings"
"github.com/cortezaproject/corteza-server/pkg/permissions"
"github.com/cortezaproject/corteza-server/system/types"
)
type (
ruleFinder interface {
FindRulesByRoleID(uint64) permissions.RuleSet
}
)
func ExportableServicePermissions(roles types.RoleSet, rf ruleFinder, access permissions.Access) map[string]map[string][]string {
var (
has bool
res string
rules permissions.RuleSet
sp = make(map[string]map[string][]string)
)
for _, r := range roles {
rules = rf.FindRulesByRoleID(r.ID)
if len(rules) == 0 {
continue
}
for _, rule := range rules {
if rule.Resource.GetService() != rule.Resource && !rule.Resource.HasWildcard() {
continue
}
if rule.Access != access {
continue
}
res = strings.TrimRight(rule.Resource.String(), ":*")
if _, has = sp[r.Handle]; !has {
sp[r.Handle] = map[string][]string{}
}
if _, has = sp[r.Handle][res]; !has {
sp[r.Handle][res] = make([]string, 0)
}
sp[r.Handle][res] = append(sp[r.Handle][res], rule.Operation.String())
}
}
return sp
}
func ExportableResourcePermissions(roles types.RoleSet, rf ruleFinder, access permissions.Access, resource permissions.Resource) map[string][]string {
var (
has bool
rules permissions.RuleSet
sp = make(map[string][]string)
)
for _, r := range roles {
rules = rf.FindRulesByRoleID(r.ID)
if len(rules) == 0 {
continue
}
for _, rule := range rules {
if rule.Resource != resource {
continue
}
if rule.Access != access {
continue
}
if _, has = sp[r.Handle]; !has {
sp[r.Handle] = make([]string, 0)
}
sp[r.Handle] = append(sp[r.Handle], rule.Operation.String())
}
}
return sp
}
system/importer/default.go
+3
View File
@@ -7,6 +7,7 @@ import (
"gopkg.in/yaml.v2"
"github.com/cortezaproject/corteza-server/pkg/permissions"
"github.com/cortezaproject/corteza-server/pkg/settings"
"github.com/cortezaproject/corteza-server/system/service"
"github.com/cortezaproject/corteza-server/system/types"
)
@@ -26,6 +27,7 @@ func Import(ctx context.Context, ff ...io.Reader) (err error) {
pi := permissions.NewImporter(service.DefaultAccessControl.Whitelist())
imp := NewImporter(
pi,
settings.NewImporter(),
NewRoleImport(pi, roles),
)
@@ -51,6 +53,7 @@ func Import(ctx context.Context, ff ...io.Reader) (err error) {
ctx,
service.DefaultRole.With(ctx),
service.DefaultAccessControl,
service.DefaultSettings.With(ctx),
roles,
)
}
system/importer/importer.go
+14 -1
View File
@@ -7,13 +7,16 @@ import (
"github.com/cortezaproject/corteza-server/pkg/deinterfacer"
"github.com/cortezaproject/corteza-server/pkg/importer"
"github.com/cortezaproject/corteza-server/pkg/permissions"
"github.com/cortezaproject/corteza-server/pkg/settings"
"github.com/cortezaproject/corteza-server/system/types"
"github.com/pkg/errors"
)
type (
Importer struct {
roles *Role
permissions importer.PermissionImporter
settings importer.SettingImporter
}
roleFinder interface {
@@ -21,10 +24,11 @@ type (
}
)
func NewImporter(p importer.PermissionImporter, ri *Role) *Importer {
func NewImporter(p importer.PermissionImporter, s importer.SettingImporter, ri *Role) *Importer {
return &Importer{
roles: ri,
permissions: p,
settings: s,
}
}
@@ -39,6 +43,9 @@ func (imp *Importer) Cast(in interface{}) (err error) {
case "allow", "deny":
return imp.permissions.CastResourcesSet(key, val)
case "settings":
return imp.settings.CastSet(val)
default:
err = fmt.Errorf("unexpected key %q", key)
}
@@ -51,6 +58,7 @@ func (imp *Importer) Store(
ctx context.Context,
rk roleKeeper,
pk permissions.ImportKeeper,
sk settings.ImportKeeper,
roles types.RoleSet,
) (err error) {
err = imp.roles.Store(ctx, rk)
@@ -69,5 +77,10 @@ func (imp *Importer) Store(
return
}
err = imp.settings.Store(ctx, sk)
if err != nil {
return errors.Wrap(err, "could not import settings")
}
return nil
}
system/importer/main_test.go
+5 -3
View File
@@ -9,12 +9,13 @@ import (
"gopkg.in/yaml.v2"
"github.com/cortezaproject/corteza-server/pkg/permissions"
"github.com/cortezaproject/corteza-server/pkg/settings"
"github.com/cortezaproject/corteza-server/system/service"
)
var (
pi *permissions.Importer
pi *permissions.Importer
si *settings.Importer
imp *Importer
)
@@ -26,9 +27,10 @@ func TestMain(m *testing.M) {
func resetMocks() {
// whitelist = nil, anything can be added
pi = permissions.NewImporter(service.AccessControl(nil).Whitelist())
si = settings.NewImporter()
imp = NewImporter(
pi,
si,
NewRoleImport(pi, nil),
)
}
system/system.go
+3
View File
@@ -126,6 +126,9 @@ func Configure() *cli.Config {
func(ctx context.Context, c *cli.Config) *cobra.Command {
return commands.Importer(ctx, c)
},
func(ctx context.Context, c *cli.Config) *cobra.Command {
return commands.Exporter(ctx, c)
},
func(ctx context.Context, c *cli.Config) *cobra.Command {
return commands.Users(ctx, c)
},