3
0

Add tests, basic messaging importer

This commit is contained in:
Denis Arh
2019-09-25 21:08:05 +02:00
parent d8a8e311f0
commit 1302f243f4
20 changed files with 628 additions and 67 deletions

View File

@@ -7,14 +7,12 @@ import (
"strconv"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/cortezaproject/corteza-server/compose/importer"
"github.com/cortezaproject/corteza-server/compose/repository"
"github.com/cortezaproject/corteza-server/compose/service"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/internal/auth"
"github.com/cortezaproject/corteza-server/internal/permissions"
"github.com/cortezaproject/corteza-server/pkg/cli"
)
@@ -28,7 +26,6 @@ func Importer(ctx context.Context, c *cli.Config) *cobra.Command {
c.InitServices(ctx, c)
var (
aux interface{}
ff []io.Reader
nsFlag = cmd.Flags().Lookup("namespace").Value.String()
ns *types.Namespace
@@ -56,47 +53,10 @@ func Importer(ctx context.Context, c *cli.Config) *cobra.Command {
ff[a], err = os.Open(arg)
cli.HandleError(err)
}
cli.HandleError(importer.Import(ctx, ns, ff...))
} else {
args = []string{"STDIN"}
ff = []io.Reader{os.Stdin}
cli.HandleError(importer.Import(ctx, ns, os.Stdin))
}
// Initialize importer
imp := importer.NewImporter(
service.DefaultNamespace.With(ctx),
service.DefaultModule.With(ctx),
service.DefaultChart.With(ctx),
service.DefaultPage.With(ctx),
service.DefaultInternalAutomationManager,
permissions.NewImporter(service.DefaultAccessControl.Whitelist()),
)
for i, f := range ff {
cmd.Printf("Importing from %s\n", args[i])
cli.HandleError(yaml.NewDecoder(f).Decode(&aux))
if ns != nil {
// If we're importing with --namespace switch,
// we're going to import all into one NS
cli.HandleError(imp.GetNamespaceImporter().Cast(ns.Slug, aux))
} else {
// importing one or more namespaces
cli.HandleError(imp.Cast(aux))
}
}
// Store all imported
cli.HandleError(imp.Store(
ctx,
service.DefaultNamespace.With(ctx),
service.DefaultModule.With(ctx),
service.DefaultChart.With(ctx),
service.DefaultPage.With(ctx),
service.DefaultInternalAutomationManager,
service.DefaultAccessControl,
))
},
}

View File

@@ -191,7 +191,6 @@ func (asImp *AutomationScript) Get(handle string) (*automation.Script, error) {
return nil, errors.New("invalid automation script handle")
}
fmt.Printf(" => %s (?%v)\n", handle, asImp.set.FindByName(handle, asImp.namespace.ID) != nil)
return asImp.set.FindByName(handle, asImp.namespace.ID), nil
}

65
compose/importer/aux.go Normal file
View File

@@ -0,0 +1,65 @@
package importer
import (
"context"
"io"
"gopkg.in/yaml.v2"
"github.com/cortezaproject/corteza-server/compose/service"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/internal/permissions"
)
// Import performs standard import procedure with default services
func Import(ctx context.Context, ns *types.Namespace, ff ...io.Reader) (err error) {
var (
aux interface{}
imp = NewImporter(
service.DefaultNamespace.With(ctx),
service.DefaultModule.With(ctx),
service.DefaultChart.With(ctx),
service.DefaultPage.With(ctx),
service.DefaultInternalAutomationManager,
permissions.NewImporter(service.DefaultAccessControl.Whitelist()),
)
)
for _, f := range ff {
if err = yaml.NewDecoder(f).Decode(&aux); err != nil {
return
}
if ns != nil {
// If we're importing with --namespace switch,
// we're going to import all into one NS
err = imp.GetNamespaceImporter().Cast(ns.Slug, aux)
} else {
// importing one or more namespaces
err = imp.Cast(aux)
}
if err != nil {
return
}
}
// Get roles across the system
roles, err := service.DefaultSystemRole.Find(ctx)
if err != nil {
return
}
// Store all imported
return imp.Store(
ctx,
service.DefaultNamespace.With(ctx),
service.DefaultModule.With(ctx),
service.DefaultChart.With(ctx),
service.DefaultPage.With(ctx),
service.DefaultInternalAutomationManager,
service.DefaultAccessControl,
roles,
)
}

View File

@@ -6,7 +6,6 @@ import (
"github.com/pkg/errors"
"github.com/cortezaproject/corteza-server/compose/service"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/internal/permissions"
"github.com/cortezaproject/corteza-server/pkg/automation"
@@ -47,6 +46,10 @@ type (
UpdateScript(context.Context, *automation.Script) error
CreateScript(context.Context, *automation.Script) error
}
roleFinder interface {
Find(context.Context) (sysTypes.RoleSet, error)
}
)
func NewImporter(nsf namespaceFinder, mf moduleFinder, cf chartFinder, pf pageFinder, af automationFinder, p importer.PermissionImporter) *Importer {
@@ -100,21 +103,26 @@ func (imp *Importer) Cast(in interface{}) (err error) {
})
}
func (imp *Importer) Store(ctx context.Context, nsStore namespaceKeeper, mStore moduleKeeper, cStore chartKeeper, pStore pageKeeper, asStore automationScriptKeeper, pk permissions.ImportKeeper) (err error) {
func (imp *Importer) Store(
ctx context.Context,
nsStore namespaceKeeper,
mStore moduleKeeper,
cStore chartKeeper,
pStore pageKeeper,
asStore automationScriptKeeper,
pk permissions.ImportKeeper,
roles sysTypes.RoleSet,
) (err error) {
err = imp.namespaces.Store(ctx, nsStore, mStore, cStore, pStore, asStore)
if err != nil {
return errors.Wrap(err, "could not import namespaces")
}
// Make sure we properly replace role handles with IDs
if roles, err := service.DefaultSystemRole.Find(ctx); err != nil {
return err
} else {
roles.Walk(func(role *sysTypes.Role) error {
imp.permissions.UpdateRoles(role.Handle, role.ID)
return nil
})
}
_ = roles.Walk(func(role *sysTypes.Role) error {
imp.permissions.UpdateRoles(role.Handle, role.ID)
return nil
})
err = imp.permissions.Store(ctx, pk)
if err != nil {

View File

@@ -101,7 +101,7 @@ func (imp *Importer) appendPermissionRule(roleHandle, accessStr, res string, oo
}
if imp.whitelist != nil && !imp.whitelist.Check(rule) {
return errors.Errorf("invalid rule: %q on %q", res, op)
return errors.Errorf("invalid rule: operation %q on resource %q", op, res)
}
imp.rules[roleHandle] = append(imp.rules[roleHandle], rule)

View File

@@ -0,0 +1,134 @@
package importer
import (
"context"
"fmt"
"github.com/pkg/errors"
"github.com/cortezaproject/corteza-server/messaging/types"
"github.com/cortezaproject/corteza-server/pkg/deinterfacer"
"github.com/cortezaproject/corteza-server/pkg/importer"
)
type (
Channel struct {
set types.ChannelSet
dirty map[uint64]bool
permissions importer.PermissionImporter
}
channelKeeper interface {
Update(*types.Channel) (*types.Channel, error)
Create(*types.Channel) (*types.Channel, error)
}
)
func NewChannelImport(permissions importer.PermissionImporter, set types.ChannelSet) *Channel {
if set == nil {
set = types.ChannelSet{}
}
out := &Channel{
set: set,
dirty: make(map[uint64]bool),
permissions: permissions,
}
return out
}
func (cImp *Channel) CastSet(set interface{}) error {
var name string
return deinterfacer.Each(set, func(index int, _ string, def interface{}) error {
if index > -1 {
// Channels defined as collection
deinterfacer.KVsetString(&name, "name", def)
}
return cImp.Cast(name, def)
})
}
func (cImp *Channel) Cast(name string, def interface{}) (err error) {
var channel *types.Channel
// if !importer.IsValidHandle(handle) {
// return errors.New("invalid channel handle")
// }
//
// handle = importer.NormalizeHandle(handle)
if channel, err = cImp.Get(name); err != nil {
return err
} else if channel == nil {
channel = &types.Channel{
Name: name,
}
cImp.set = append(cImp.set, channel)
} else if channel.ID == 0 {
return errors.Errorf("channel name %q already defined in this import session", channel.Name)
} else {
cImp.dirty[channel.ID] = true
}
if name, ok := def.(string); ok && name != "" {
channel.Name = name
return nil
}
return deinterfacer.Each(def, func(_ int, key string, val interface{}) (err error) {
switch key {
case "name":
// already handled
case "type":
channel.Type = types.ChannelType(deinterfacer.ToString(val))
if !channel.Type.IsValid() {
return fmt.Errorf("invalid channel type %q for channel %q", channel.Type, channel.Name)
}
case "topic":
channel.Topic = deinterfacer.ToString(val)
case "allow", "deny":
return cImp.permissions.CastSet(types.ChannelPermissionResource.String()+channel.Name, key, val)
default:
return fmt.Errorf("unexpected key %q for channel %q", key, channel.Name)
}
return err
})
}
func (cImp *Channel) Get(name string) (*types.Channel, error) {
// name = importer.NormalizeHandle(name)
//
// if !importer.IsValidHandle(name) {
// return nil, errors.New("invalid channel name")
// }
return cImp.set.FindByName(name), nil
}
func (cImp *Channel) Store(ctx context.Context, k channelKeeper) error {
return cImp.set.Walk(func(channel *types.Channel) (err error) {
var handle = channel.Name
if channel.ID == 0 {
channel, err = k.Create(channel)
} else if cImp.dirty[channel.ID] {
channel, err = k.Update(channel)
}
if err != nil {
return
}
cImp.permissions.UpdateResources(types.ChannelPermissionResource.String(), handle, channel.ID)
cImp.permissions.UpdateRoles(channel.Name, channel.ID)
return
})
}

View File

@@ -0,0 +1,28 @@
package importer
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/cortezaproject/corteza-server/messaging/types"
)
func TestChannelImport_CastSet(t *testing.T) {
impFixTester(t, "channels", func(t *testing.T, ri *Channel) {
req := require.New(t)
req.NotNil(ri.set)
req.Len(ri.set, 3)
req.NotNil(ri.set.FindByName("General"))
req.Equal("Talk about anything", ri.set.FindByName("General").Topic)
req.Equal(types.ChannelTypePublic, ri.set.FindByName("General").Type)
req.NotNil(ri.set.FindByName("Random"))
req.Equal("", ri.set.FindByName("Random").Topic)
req.Equal(types.ChannelTypePublic, ri.set.FindByName("Random").Type)
req.NotNil(ri.set.FindByName("Secret"))
req.Equal(types.ChannelTypePrivate, ri.set.FindByName("Secret").Type)
})
}

View File

@@ -0,0 +1,69 @@
package importer
import (
"context"
"fmt"
"github.com/cortezaproject/corteza-server/internal/permissions"
"github.com/cortezaproject/corteza-server/messaging/types"
"github.com/cortezaproject/corteza-server/pkg/deinterfacer"
"github.com/cortezaproject/corteza-server/pkg/importer"
sysTypes "github.com/cortezaproject/corteza-server/system/types"
)
type (
Importer struct {
channels *Channel
permissions importer.PermissionImporter
}
channelFinder interface {
Find(context.Context) (types.ChannelSet, error)
}
)
func NewImporter(p importer.PermissionImporter, ci *Channel) *Importer {
return &Importer{
channels: ci,
permissions: p,
}
}
func (imp *Importer) Cast(in interface{}) (err error) {
return deinterfacer.Each(in, func(index int, key string, val interface{}) (err error) {
switch key {
case "channels":
return imp.channels.CastSet(val)
case "channel":
return imp.channels.CastSet([]interface{}{val})
case "allow", "deny":
return imp.permissions.CastResourcesSet(key, val)
default:
err = fmt.Errorf("unexpected key %q", key)
}
return err
})
}
func (imp *Importer) Store(ctx context.Context, rk channelKeeper, pk permissions.ImportKeeper, roles sysTypes.RoleSet) (err error) {
err = imp.channels.Store(ctx, rk)
if err != nil {
return
}
// Make sure we properly replace channel handles with IDs
roles.Walk(func(r *sysTypes.Role) error {
imp.permissions.UpdateRoles(r.Handle, r.ID)
return nil
})
err = imp.permissions.Store(ctx, pk)
if err != nil {
return
}
return nil
}

View File

@@ -0,0 +1,65 @@
package importer
import (
"fmt"
"os"
"testing"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v2"
"github.com/cortezaproject/corteza-server/internal/permissions"
"github.com/cortezaproject/corteza-server/system/service"
)
var (
pi *permissions.Importer
imp *Importer
)
func TestMain(m *testing.M) {
resetMocks()
os.Exit(m.Run())
}
func resetMocks() {
// whitelist = nil, anything can be added
pi = permissions.NewImporter(service.AccessControl(nil).Whitelist())
imp = NewImporter(
pi,
NewChannelImport(pi, nil),
)
}
func impFixTester(t *testing.T, name string, tester interface{}) {
t.Run(name, func(t *testing.T) {
// We're not calling reset mocks BEFORE calling tester()
// because we want to have an option to set it up as we want
defer resetMocks()
var aux interface{}
req := require.New(t)
f, err := os.Open(fmt.Sprintf("testdata/%s.yaml", name))
req.NoError(err)
req.NoError(yaml.NewDecoder(f).Decode(&aux))
req.NotNil(aux)
if reqError, ok := tester.(error); ok {
req.EqualError(imp.Cast(aux), reqError.Error())
return
} else {
req.NoError(imp.Cast(aux))
}
switch tester := tester.(type) {
case func(*testing.T, *Channel):
tester(t, imp.channels)
case func(*testing.T, *Importer):
tester(t, imp)
default:
panic("unsupported tester function signature")
}
})
}

View File

@@ -0,0 +1,8 @@
channels:
- name: General
topic: Talk about anything
type: public
- name: Random
type: public
- name: Secret
type: private

View File

@@ -111,3 +111,14 @@ func (cm ChannelMembershipPolicy) IsValid() bool {
return false
}
// FindByName finds items from slice by its name
func (set ChannelSet) FindByName(name string) *Channel {
for i := range set {
if set[i].Name == name {
return set[i]
}
}
return nil
}

38
provision/update.sh Executable file
View File

@@ -0,0 +1,38 @@
#!/bin/bash
function download {
SRC="https://raw.githubusercontent.com/cortezaproject/corteza-configs/master/${1}"
DST=${2}
echo -ne "\033[32m${DST}\033[39m (${SRC}) ..."
curl -s $SRC > ${DST}
echo "done"
}
function getCrmConfig {
NAMES="1000_namespace 1100_modules 1200_charts 1300_scripts 1400_pages"
for NAME in $NAMES; do
download "crm/${NAME}.yaml" "./compose/${NAME}.yaml"
done
}
function get {
getCrmConfig
}
function gen {
echo "generating..."
}
case ${1:-"all"} in
gen)
gen
;;
get)
get
;;
all)
get
gen
esac

View File

@@ -47,23 +47,25 @@ func Importer(ctx context.Context, c *cli.Config) *cobra.Command {
roles, err := service.DefaultRole.With(ctx).Find(&types.RoleFilter{})
cli.HandleError(err)
perm := permissions.NewImporter(service.DefaultAccessControl.Whitelist())
imp := importer.NewImporter(perm,
importer.NewRoleImport(perm, roles),
)
for i, f := range ff {
cmd.Printf("Importing from %s\n", args[i])
cli.HandleError(yaml.NewDecoder(f).Decode(&aux))
perm := permissions.NewImporter(service.DefaultAccessControl.Whitelist())
imp := importer.NewImporter(perm,
importer.NewRoleImport(perm, roles),
)
cli.HandleError(imp.Store(
ctx,
service.DefaultRole.With(ctx),
service.DefaultAccessControl,
))
cli.HandleError(imp.Cast(aux))
}
cli.HandleError(imp.Store(
ctx,
service.DefaultRole.With(ctx),
service.DefaultAccessControl,
roles,
))
},
}

View File

@@ -7,6 +7,7 @@ import (
"github.com/cortezaproject/corteza-server/internal/permissions"
"github.com/cortezaproject/corteza-server/pkg/deinterfacer"
"github.com/cortezaproject/corteza-server/pkg/importer"
"github.com/cortezaproject/corteza-server/system/types"
)
type (
@@ -14,6 +15,10 @@ type (
roles *Role
permissions importer.PermissionImporter
}
roleFinder interface {
Find(context.Context) (types.RoleSet, error)
}
)
func NewImporter(p importer.PermissionImporter, ri *Role) *Importer {
@@ -42,12 +47,23 @@ func (imp *Importer) Cast(in interface{}) (err error) {
})
}
func (imp *Importer) Store(ctx context.Context, rk roleKeeper, pk permissions.ImportKeeper) (err error) {
func (imp *Importer) Store(
ctx context.Context,
rk roleKeeper,
pk permissions.ImportKeeper,
roles types.RoleSet,
) (err error) {
err = imp.roles.Store(ctx, rk)
if err != nil {
return
}
// Make sure we properly replace role handles with IDs
roles.Walk(func(role *types.Role) error {
imp.permissions.UpdateRoles(role.Handle, role.ID)
return nil
})
err = imp.permissions.Store(ctx, pk)
if err != nil {
return

View File

@@ -0,0 +1,54 @@
package compose
import (
"os"
"testing"
"gopkg.in/yaml.v2"
"github.com/cortezaproject/corteza-server/compose/importer"
"github.com/cortezaproject/corteza-server/compose/service"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/internal/permissions"
sysTypes "github.com/cortezaproject/corteza-server/system/types"
)
func TestProvisioning(t *testing.T) {
h := newHelper(t)
var (
aux interface{}
ctx = h.secCtx()
imp = importer.NewImporter(
service.DefaultNamespace.With(ctx),
service.DefaultModule.With(ctx),
service.DefaultChart.With(ctx),
service.DefaultPage.With(ctx),
service.DefaultInternalAutomationManager,
permissions.NewImporter(service.DefaultAccessControl.Whitelist()),
)
)
h.allow(types.ComposePermissionResource, "grant")
var f, err = os.Open("../../provision/compose/001_permission_rules.yaml")
h.a.NoError(err)
h.a.NoError(yaml.NewDecoder(f).Decode(&aux))
h.a.NoError(imp.Cast(aux))
h.a.NoError(imp.Store(
ctx,
service.DefaultNamespace.With(ctx),
service.DefaultModule.With(ctx),
service.DefaultChart.With(ctx),
service.DefaultPage.With(ctx),
service.DefaultInternalAutomationManager,
service.DefaultAccessControl,
sysTypes.RoleSet{
&sysTypes.Role{ID: 1, Handle: "everyone"},
&sysTypes.Role{ID: 2, Handle: "admins"},
},
))
}

View File

@@ -10,6 +10,7 @@ import (
func RecursiveDotEnvLoad() {
for _, loc := range []string{".env", "../.env", "../../.env"} {
if _, err := os.Stat(loc); err == nil {
print("LOADING ENV", loc)
godotenv.Load(loc)
}
}

View File

@@ -121,6 +121,11 @@ func newHelper(t *testing.T) helper {
return h
}
// Returns context w/ security details
func (h helper) secCtx() context.Context {
return auth.SetIdentityToContext(context.Background(), h.cUser)
}
// apitest basics, initialize, set handler, add auth
func (h helper) apiInit() *apitest.APITest {
InitApp()

View File

@@ -0,0 +1,46 @@
package messaging
import (
"os"
"testing"
"gopkg.in/yaml.v2"
"github.com/cortezaproject/corteza-server/internal/permissions"
"github.com/cortezaproject/corteza-server/messaging/importer"
"github.com/cortezaproject/corteza-server/messaging/service"
"github.com/cortezaproject/corteza-server/messaging/types"
sysTypes "github.com/cortezaproject/corteza-server/system/types"
)
func TestProvisioning(t *testing.T) {
h := newHelper(t)
var (
aux interface{}
roles = sysTypes.RoleSet{
&sysTypes.Role{ID: 1, Handle: "everyone"},
&sysTypes.Role{ID: 2, Handle: "admins"},
}
ctx = h.secCtx()
pi = permissions.NewImporter(service.DefaultAccessControl.Whitelist())
imp = importer.NewImporter(pi, importer.NewChannelImport(pi, nil))
)
h.allow(types.MessagingPermissionResource, "grant")
var f, err = os.Open("../../provision/messaging/001_permission_rules.yaml")
h.a.NoError(err)
h.a.NoError(yaml.NewDecoder(f).Decode(&aux))
h.a.NoError(imp.Cast(aux))
h.a.NoError(imp.Store(
ctx,
service.DefaultChannel.With(ctx),
service.DefaultAccessControl,
roles,
))
}

View File

@@ -128,6 +128,11 @@ func newHelper(t *testing.T) helper {
return h
}
// Returns context w/ security details
func (h helper) secCtx() context.Context {
return auth.SetIdentityToContext(context.Background(), h.cUser)
}
// apitest basics, initialize, set handler, add auth
func (h helper) apiInit() *apitest.APITest {
InitApp()

View File

@@ -0,0 +1,47 @@
package system
import (
"os"
"testing"
"gopkg.in/yaml.v2"
"github.com/cortezaproject/corteza-server/internal/permissions"
"github.com/cortezaproject/corteza-server/system/importer"
"github.com/cortezaproject/corteza-server/system/service"
"github.com/cortezaproject/corteza-server/system/types"
)
func TestProvisioning(t *testing.T) {
h := newHelper(t)
var (
aux interface{}
roles = types.RoleSet{
&types.Role{ID: 1, Handle: "everyone"},
&types.Role{ID: 2, Handle: "admins"},
}
ctx = h.secCtx()
pi = permissions.NewImporter(service.DefaultAccessControl.Whitelist())
imp = importer.NewImporter(pi, importer.NewRoleImport(pi, roles))
)
h.allow(types.SystemPermissionResource, "grant")
h.allow(types.SystemPermissionResource, "role.create")
h.allow(types.RolePermissionResource.AppendWildcard(), "update")
var f, err = os.Open("../../provision/system/001_permission_rules.yaml")
h.a.NoError(err)
h.a.NoError(yaml.NewDecoder(f).Decode(&aux))
h.a.NoError(imp.Cast(aux))
h.a.NoError(imp.Store(
ctx,
service.DefaultRole.With(ctx),
service.DefaultAccessControl,
roles,
))
}