DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Remove all pre-2022.9.x migrations

Browse Source
This commit is contained in:
Denis Arh
2022-07-04 13:09:41 +02:00
parent 5d8f00df64
commit 23e97f96e4
13 changed files with 6 additions and 1456 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/provision/migrations.go
-7
View File
@@ -1,7 +0,0 @@
package provision
import "time"
func releaseDate(y int, m time.Month) time.Time {
return time.Date(y, m+1, 1, 0, 0, 0, 0, time.UTC)
}
pkg/provision/migrations_202103_apps.go
-34
View File
@@ -1,34 +0,0 @@
package provision
import (
"context"
"github.com/cortezaproject/corteza-server/store"
"github.com/cortezaproject/corteza-server/system/types"
"time"
)
func migrateApplications(ctx context.Context, s store.Storer) error {
rd := releaseDate(2021, time.March)
set, _, err := store.SearchApplications(ctx, s, types.ApplicationFilter{})
if err != nil {
return err
}
return set.Walk(func(app *types.Application) (err error) {
if app.Unify == nil {
return
}
if app.Unify.Url == "/messaging" && (app.UpdatedAt == nil || app.UpdatedAt.Before(rd)) {
// Disable and un-list messaging app
// but only if it was not updated after 21.3 release date
app.Unify.Listed = false
app.Enabled = false
return store.UpdateApplication(ctx, s, app)
}
return nil
})
}
pkg/provision/migrations_202103_templates.go
-107
View File
@@ -1,107 +0,0 @@
package provision
import (
"context"
"fmt"
"github.com/cortezaproject/corteza-server/pkg/errors"
"github.com/cortezaproject/corteza-server/pkg/id"
"github.com/cortezaproject/corteza-server/store"
"github.com/cortezaproject/corteza-server/system/types"
"go.uber.org/zap"
)
// Pre 2021.3 versions had email templates stored in settings
// from 2021.3 onwards we have dedicated subsystem for managing templates
//
// This migration moves email templates from settings templates.
func migrateEmailTemplates(ctx context.Context, log *zap.Logger, s store.Storer) error {
var (
// setting name => template handle
m = map[string]*types.Template{
//"general.mail.logo",
"general.mail.header.en": {
Type: "text/html",
Handle: "email_general_header",
Partial: true,
Meta: types.TemplateMeta{
Short: "General template header",
Description: "General template header to use with system email notifications",
},
},
"general.mail.footer.en": {
Type: "text/html",
Handle: "email_general_footer",
Partial: true,
Meta: types.TemplateMeta{
Short: "General template footer",
Description: "General template footer to use with system email notifications",
},
},
"auth.mail.email-confirmation.subject.en": {
Type: "text/plain",
Handle: "auth_email_confirm_subject",
Meta: types.TemplateMeta{Short: "Password reset subject"},
},
"auth.mail.email-confirmation.body.en": {
Type: "text/html",
Handle: "auth_email_confirm_body",
Meta: types.TemplateMeta{Short: "Password reset content"},
},
"auth.mail.password-reset.subject.en": {
Type: "text/plain",
Handle: "auth_email_password_reset_subject",
Meta: types.TemplateMeta{Short: "Email confirmation subject"},
},
"auth.mail.password-reset.body.en": {
Type: "text/html",
Handle: "auth_email_password_reset_body",
Meta: types.TemplateMeta{Short: "Email confirmation content"},
},
}
)
return store.Tx(ctx, s, func(ctx context.Context, s store.Storer) error {
for name, tmpl := range m {
sval, err := store.LookupSettingValueByNameOwnedBy(ctx, s, name, 0)
if errors.IsNotFound(err) {
// setting not found, that's ok, see the next one
continue
} else if err != nil {
return fmt.Errorf("failed to lookup for setting by bame: %w", err)
}
_, err = store.LookupTemplateByHandle(ctx, s, tmpl.Handle)
if err != nil && !errors.IsNotFound(err) {
// any error but not-found is fatal.
return fmt.Errorf("failed to lookup for template by handle: %w", err)
} else if err == nil {
// template exists
continue
}
tmpl.ID = id.Next()
tmpl.CreatedAt = *now()
tmpl.Template = sval.String()
err = store.CreateTemplate(ctx, s, tmpl)
if err != nil {
return fmt.Errorf("failed to store migrated template: %w", err)
}
log.Debug("migrated template from settings",
zap.String("setting", name),
zap.String("handle", tmpl.Handle),
)
}
// Go over all settings again and remove them
for name := range m {
if err := store.DeleteSettingValueByNameOwnedBy(ctx, s, name, 0); err != nil {
return err
}
}
return nil
})
}
pkg/provision/migrations_202109_rbac.go
-297
View File
@@ -1,297 +0,0 @@
package provision
import (
"context"
"fmt"
"strconv"
"strings"
composeTypes "github.com/cortezaproject/corteza-server/compose/types"
federationTypes "github.com/cortezaproject/corteza-server/federation/types"
"github.com/cortezaproject/corteza-server/pkg/filter"
"github.com/cortezaproject/corteza-server/pkg/rbac"
"github.com/cortezaproject/corteza-server/store"
systemTypes "github.com/cortezaproject/corteza-server/system/types"
"go.uber.org/zap"
)
type (
resourceIndex struct {
fields map[uint64]*composeTypes.ModuleField
modules map[uint64]*composeTypes.Module
charts map[uint64]*composeTypes.Chart
pages map[uint64]*composeTypes.Page
exposedModules map[uint64]*federationTypes.ExposedModule
sharedModules map[uint64]*federationTypes.SharedModule
}
)
// MigrateOperations creates system roles
func migratePre202109RbacRules(ctx context.Context, log *zap.Logger, s store.Storer) error {
return store.Tx(ctx, s, func(ctx context.Context, s store.Storer) error {
rr, _, err := store.SearchRbacRules(ctx, s, rbac.RuleFilter{})
if err != nil {
return err
}
log.Info("migrating RBAC rules to new format", zap.Int("rules", len(rr)))
rx, err := preloadResourceIndex(ctx, s)
if err != nil {
return err
}
var uniq = make(map[string]bool)
var uniqID = func(r *rbac.Rule) string {
return fmt.Sprintf("%s|%s|%d", r.Resource, r.Operation, r.RoleID)
}
for _, r := range rr {
var (
cr = *r
action = migratePre202109RbacRule(r, rx)
)
if action != 0 {
err = store.DeleteRbacRule(ctx, s, &cr)
if err != nil {
return fmt.Errorf("could not delete RBAC rule %s: %v", r, err)
}
if action == -1 {
log.Debug("removed obsolete RBAC rule", zap.Stringer("rule", r))
}
}
if action == 1 {
if uniq[uniqID(r)] {
log.Warn("skipping duplicate RBAC rule", zap.Stringer("rule", r))
continue
}
err = store.CreateRbacRule(ctx, s, r)
if err != nil {
return fmt.Errorf("could not create RBAC rule %s: %v", r, err)
}
uniq[uniqID(r)] = true
}
}
return nil
})
}
// 0 - no action
// -1 - remove
// 1 - update
func migratePre202109RbacRule(r *rbac.Rule, rx *resourceIndex) (op int) {
const (
nsSep = "::"
nsDef = "corteza"
)
if strings.Contains(r.Resource, nsSep) {
return
}
if r.Resource == "" {
return -1
}
// split old format
parts := strings.SplitN(r.Resource, ":", 3)
switch {
case parts[0] == "messaging":
return -1
case len(parts) > 1 && parts[1] == "automation-script":
return -1
case len(parts) == 1:
r.Resource = nsDef + nsSep + strings.Join(parts[:1], ":")
default:
r.Resource = nsDef + nsSep + strings.Join(parts[:2], ":")
}
op = 1
rType := rbac.ResourceType(r.Resource)
switch {
case rType == systemTypes.UserResourceType && strings.HasPrefix(r.Operation, "unmask."):
// flipping terms in user unmask operations
r.Operation = strings.TrimPrefix(r.Operation, "unmask.") + ".unmask"
case rType == "corteza::federation:module":
// fed. module resource was split into two resources - exposed & shared
if r.Operation == "manage" {
rType = federationTypes.ExposedModuleResourceType
} else if r.Operation == "map" {
rType = federationTypes.SharedModuleResourceType
} else {
return -1
}
case rType == composeTypes.ModuleResourceType && strings.HasPrefix(r.Operation, "record.") && r.Operation != "record.create":
// change resource type from module to record on record read, delete, update operations & remove the prefix
rType = composeTypes.RecordResourceType
r.Operation = strings.TrimPrefix(r.Operation, "record.")
}
if len(parts) == 3 {
var ID, _ = strconv.ParseUint(parts[2], 10, 64)
p1 := uint64(0)
p2 := uint64(0)
// exceptions with nested references
switch rType {
case composeTypes.ModuleFieldResourceType:
if ID > 0 {
if f, ok := rx.fields[ID]; ok {
p1 = f.NamespaceID
p2 = f.ModuleID
}
}
r.Resource = composeTypes.ModuleFieldRbacResource(p1, p2, ID)
case composeTypes.ModuleResourceType:
if ID > 0 {
if r, ok := rx.modules[ID]; ok {
p1 = r.NamespaceID
}
}
r.Resource = composeTypes.ModuleRbacResource(p1, ID)
// ID belongs to module!
case composeTypes.RecordResourceType:
if ID > 0 {
if r, ok := rx.modules[ID]; ok {
p1 = r.NamespaceID
}
}
r.Resource = composeTypes.RecordRbacResource(p1, ID, 0)
case composeTypes.ChartResourceType:
if ID > 0 {
if r, ok := rx.charts[ID]; ok {
p1 = r.NamespaceID
}
}
r.Resource = composeTypes.ChartRbacResource(p1, ID)
case composeTypes.PageResourceType:
if ID > 0 {
if r, ok := rx.pages[ID]; ok {
p1 = r.NamespaceID
}
}
r.Resource = composeTypes.PageRbacResource(p1, ID)
case federationTypes.ExposedModuleResourceType:
if ID > 0 {
if r, ok := rx.exposedModules[ID]; ok {
p1 = r.NodeID
}
}
r.Resource = federationTypes.ExposedModuleRbacResource(p1, ID)
case federationTypes.SharedModuleResourceType:
if ID > 0 {
if r, ok := rx.sharedModules[ID]; ok {
p1 = r.NodeID
}
}
r.Resource = federationTypes.SharedModuleRbacResource(p1, ID)
default:
r.Resource = rType + "/" + func() string {
if ID == 0 {
return "*"
}
return parts[2]
}()
}
} else {
r.Resource = rType + "/"
}
return
}
// helper to preloadresources that may be used when properly constructing rules
func preloadResourceIndex(ctx context.Context, s store.Storer) (*resourceIndex, error) {
rx := &resourceIndex{}
rx.modules = make(map[uint64]*composeTypes.Module)
modules, _, err := store.SearchComposeModules(ctx, s, composeTypes.ModuleFilter{
Paging: filter.Paging{Limit: 0},
Deleted: filter.StateInclusive,
})
modIDs := make([]uint64, 0, len(modules))
if err != nil {
return nil, err
}
for _, r := range modules {
rx.modules[r.ID] = r
modIDs = append(modIDs, r.ID)
}
if len(modIDs) > 0 {
rx.fields = make(map[uint64]*composeTypes.ModuleField)
fields, _, err := store.SearchComposeModuleFields(ctx, s, composeTypes.ModuleFieldFilter{
ModuleID: modIDs,
Deleted: filter.StateInclusive,
})
if err != nil {
return nil, err
}
for _, r := range fields {
rx.fields[r.ID] = r
}
}
rx.charts = make(map[uint64]*composeTypes.Chart)
chart, _, err := store.SearchComposeCharts(ctx, s, composeTypes.ChartFilter{
Paging: filter.Paging{Limit: 0},
Deleted: filter.StateInclusive,
})
if err != nil {
return nil, err
}
for _, r := range chart {
rx.charts[r.ID] = r
}
rx.pages = make(map[uint64]*composeTypes.Page)
page, _, err := store.SearchComposePages(ctx, s, composeTypes.PageFilter{
Paging: filter.Paging{Limit: 0},
Deleted: filter.StateInclusive,
})
if err != nil {
return nil, err
}
for _, r := range page {
rx.pages[r.ID] = r
}
rx.exposedModules = make(map[uint64]*federationTypes.ExposedModule)
exposedModule, _, err := store.SearchFederationExposedModules(ctx, s, federationTypes.ExposedModuleFilter{Paging: filter.Paging{Limit: 0}})
if err != nil {
return nil, err
}
for _, r := range exposedModule {
rx.exposedModules[r.ID] = r
}
rx.sharedModules = make(map[uint64]*federationTypes.SharedModule)
sharedModule, _, err := store.SearchFederationSharedModules(ctx, s, federationTypes.SharedModuleFilter{Paging: filter.Paging{Limit: 0}})
if err != nil {
return nil, err
}
for _, r := range sharedModule {
rx.sharedModules[r.ID] = r
}
return rx, nil
}
pkg/provision/migrations_202109_rbac_test.go
-43
View File
@@ -1,43 +0,0 @@
package provision
import (
"testing"
composeTypes "github.com/cortezaproject/corteza-server/compose/types"
federationTypes "github.com/cortezaproject/corteza-server/federation/types"
"github.com/cortezaproject/corteza-server/pkg/rbac"
"github.com/stretchr/testify/require"
)
func Test_migratePre202109RbacRule(t *testing.T) {
rx := &resourceIndex{
fields: make(map[uint64]*composeTypes.ModuleField),
modules: make(map[uint64]*composeTypes.Module),
charts: make(map[uint64]*composeTypes.Chart),
pages: make(map[uint64]*composeTypes.Page),
exposedModules: make(map[uint64]*federationTypes.ExposedModule),
sharedModules: make(map[uint64]*federationTypes.SharedModule),
}
tcc := []struct {
wantOp int
rule *rbac.Rule
wantRule *rbac.Rule
}{
{-1, rbac.AllowRule(0, "messaging", "whatever"), nil},
{-1, rbac.AllowRule(0, "foo:automation-script", "whatever"), nil},
{1, rbac.AllowRule(0, "abc", "whatever"), rbac.AllowRule(0, "corteza::abc/", "whatever")},
{1, rbac.AllowRule(0, "federation:module", "map"), rbac.AllowRule(0, "corteza::federation:shared-module/", "map")},
{1, rbac.AllowRule(0, "federation:module", "manage"), rbac.AllowRule(0, "corteza::federation:exposed-module/", "manage")},
{1, rbac.AllowRule(0, "compose:module:234", "record.read"), rbac.AllowRule(0, "corteza::compose:record/*/234/*", "read")},
{1, rbac.AllowRule(0, "compose:module-field:234", "op"), rbac.AllowRule(0, "corteza::compose:module-field/*/*/234", "op")},
}
for _, tc := range tcc {
t.Run(tc.rule.String(), func(t *testing.T) {
require.Equal(t, tc.wantOp, migratePre202109RbacRule(tc.rule, rx))
if tc.wantRule != nil {
require.Equal(t, tc.wantRule.String(), tc.rule.String())
}
})
}
}
pkg/provision/migrations_202109_resource_translations.go
-266
View File
@@ -1,266 +0,0 @@
package provision
import (
"context"
"fmt"
"strings"
cmpTypes "github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/filter"
"github.com/cortezaproject/corteza-server/pkg/id"
"github.com/cortezaproject/corteza-server/pkg/locale"
"github.com/cortezaproject/corteza-server/store"
sysTypes "github.com/cortezaproject/corteza-server/system/types"
"go.uber.org/zap"
)
// Migrates resource translations from the resource
// struct to the dedicated store (table)
//
// While doing this, we also modify some resource substructure:
// - page blocks (assign page block IDs
// - automation buttons on page blocks (assign automation button IDs)
// - field expressions (assign expression IDs)
//
// Note: we will migrate all translations to current default language
// If you do not like that, shut down Corteza after migrations and fix this directly in the store
func migrateResourceTranslations(ctx context.Context, log *zap.Logger, s store.Storer) (err error) {
log.Info("migrating resource locales")
var (
migrated = make(map[string]bool)
)
set, _, err := store.SearchResourceTranslations(ctx, s, sysTypes.ResourceTranslationFilter{})
set.Walk(func(r *sysTypes.ResourceTranslation) error {
var pos = strings.Index(r.Resource, "/")
if pos < 0 {
return nil
}
migrated[r.Resource[0:pos]] = true
return nil
})
if !migrated[cmpTypes.NamespaceResourceTranslationType] {
if err = migrateComposeNamespaceResourceTranslations(ctx, log, s); err != nil {
return
}
}
if !migrated[cmpTypes.ModuleResourceTranslationType] {
// @todo migrate module locales
if err = migrateComposeModuleResourceTranslations(ctx, log, s); err != nil {
return
}
}
if !migrated[cmpTypes.PageResourceTranslationType] {
if err = migrateComposePageResourceTranslations(ctx, log, s); err != nil {
return
}
}
return
}
// helper fn that creates ResourceTranslation from the given resource, key and message
// and default language
func makeResourceTranslation(lr interface{ ResourceTranslation() string }, k, m string) *sysTypes.ResourceTranslation {
return &sysTypes.ResourceTranslation{
ID: id.Next(),
Lang: sysTypes.Lang{Tag: locale.Global().Default().Tag},
Resource: lr.ResourceTranslation(),
K: k,
Message: m,
CreatedAt: *now(),
}
}
// migrate resource translations for compose namespace
func migrateComposeNamespaceResourceTranslations(ctx context.Context, log *zap.Logger, s store.Storer) error {
set, _, err := store.SearchComposeNamespaces(ctx, s, cmpTypes.NamespaceFilter{Deleted: filter.StateInclusive})
if err != nil {
return err
}
log.Info("migrating compose namespaces", zap.Int("count", len(set)))
return s.Tx(ctx, func(ctx context.Context, s store.Storer) error {
return set.Walk(func(res *cmpTypes.Namespace) error {
return store.CreateResourceTranslation(ctx, s,
makeResourceTranslation(res, "name", res.Name),
makeResourceTranslation(res, "subtitle", res.Meta.Subtitle),
makeResourceTranslation(res, "description", res.Meta.Description),
)
})
})
}
// migrate resource translations for compose module
func migrateComposeModuleResourceTranslations(ctx context.Context, log *zap.Logger, s store.Storer) error {
set, _, err := store.SearchComposeModules(ctx, s, cmpTypes.ModuleFilter{Deleted: filter.StateInclusive})
if err != nil {
return err
}
log.Info("migrating compose modules", zap.Int("count", len(set)))
return s.Tx(ctx, func(ctx context.Context, s store.Storer) error {
return set.Walk(func(res *cmpTypes.Module) (err error) {
if err = migrateComposeModuleFieldResourceTranslations(ctx, s, res.NamespaceID, res.ID); err != nil {
return err
}
err = store.CreateResourceTranslation(ctx, s,
makeResourceTranslation(res, "name", res.Name),
)
if err != nil {
return err
}
return store.UpdateComposeModule(ctx, s, res)
})
})
}
// migrate module fields translations for compose module field
//
// Adds validatorID on field expressions
func migrateComposeModuleFieldResourceTranslations(ctx context.Context, s store.Storer, namespaceID uint64, moduleIDs ...uint64) error {
set, _, err := store.SearchComposeModuleFields(ctx, s, cmpTypes.ModuleFieldFilter{ModuleID: moduleIDs})
if err != nil {
return err
}
return s.Tx(ctx, func(ctx context.Context, s store.Storer) error {
return set.Walk(func(res *cmpTypes.ModuleField) (err error) {
tt := sysTypes.ResourceTranslationSet{}
// store does not contain this info,
// but we need it to generate resource id
res.NamespaceID = namespaceID
tt = append(tt, makeResourceTranslation(res, "label", res.Label))
var update = len(res.Expressions.Validators) > 0
for i := range res.Expressions.Validators {
validatorID := i + 1
res.Expressions.Validators[i].ValidatorID = uint64(validatorID)
tt = append(tt, makeResourceTranslation(res, fmt.Sprintf("expression.validator.%d.error", validatorID), res.Expressions.Validators[i].Error))
}
if err = store.CreateResourceTranslation(ctx, s, tt...); err != nil {
return
}
if update {
if err = store.UpdateComposeModuleField(ctx, s, res); err != nil {
return
}
}
return nil
})
})
}
// migrate resource translations for compose module
func migrateComposePageResourceTranslations(ctx context.Context, log *zap.Logger, s store.Storer) error {
var tt sysTypes.ResourceTranslationSet
set, _, err := store.SearchComposePages(ctx, s, cmpTypes.PageFilter{Deleted: filter.StateInclusive})
if err != nil {
return err
}
// @todo migrate page locales
// @todo page block IDs
// @todo automation button IDs
log.Info("migrating compose pages", zap.Int("count", len(set)))
return s.Tx(ctx, func(ctx context.Context, s store.Storer) error {
return set.Walk(func(res *cmpTypes.Page) (err error) {
tt = sysTypes.ResourceTranslationSet{
makeResourceTranslation(res, "title", res.Title),
makeResourceTranslation(res, "description", res.Description),
}
if pbtt, err := convertComposePageBlockTranslations(res); err != nil {
return err
} else {
tt = append(tt, pbtt...)
}
if err = store.CreateResourceTranslation(ctx, s, tt...); err != nil {
return
}
if err = store.UpdateComposePage(ctx, s, res); err != nil {
return
}
return
})
})
}
// collects translations for compose page blocks and alters them (adding block ID)
func convertComposePageBlockTranslations(res *cmpTypes.Page) (sysTypes.ResourceTranslationSet, error) {
var tt sysTypes.ResourceTranslationSet
for i, b := range res.Blocks {
blockID := i + 1
res.Blocks[i].BlockID = uint64(blockID)
pfx := fmt.Sprintf("pageBlock.%d.", blockID)
tt = append(tt,
makeResourceTranslation(res, pfx+"title", b.Title),
makeResourceTranslation(res, pfx+"description", b.Description),
)
switch b.Kind {
case "Automation":
att, bb, err := convertComposeAutomationPageBlockTranslations(res, pfx, res.Blocks[i].Options)
if err != nil {
return nil, err
}
res.Blocks[i].Options["buttons"] = bb
tt = append(tt, att...)
}
}
return tt, nil
}
// takes automation options for page block and returns translations + fixed set of buttons (containing content ID)
func convertComposeAutomationPageBlockTranslations(res *cmpTypes.Page, pfx string, opt map[string]interface{}) (tt sysTypes.ResourceTranslationSet, bb []interface{}, err error) {
if tmp, ok := opt["buttons"]; !ok {
return nil, nil, nil
} else if bb, ok = tmp.([]interface{}); !ok {
return nil, nil, nil
}
for i := range bb {
b, is := bb[i].(map[string]interface{})
if !is {
continue
}
buttonID := i + 1
b["buttonID"] = buttonID
bb[i] = b
var label string
if _, has := b["label"]; has {
label = b["label"].(string)
}
tt = append(tt, makeResourceTranslation(res, fmt.Sprintf("%sbuttons.%d.label", pfx, buttonID), label))
}
return
}
pkg/provision/migrations_202109_roles.go
-70
View File
@@ -1,70 +0,0 @@
package provision
import (
"context"
"github.com/cortezaproject/corteza-server/pkg/id"
"github.com/cortezaproject/corteza-server/store"
"go.uber.org/zap"
)
func migratePre202109Roles(ctx context.Context, log *zap.Logger, s store.Storer) (err error) {
const (
obsoleteEveryoneID uint64 = 1
obsoleteAdminsID uint64 = 2
)
log.Info("migrating pre-2021.9 roles")
m, err := loadRoles(ctx, s)
if err != nil {
return
}
// let's see if everyone role is still here:
if m["everyone"] != nil && m["everyone"].ID == obsoleteEveryoneID {
log.Info("migrating 'everyone' role to new ID")
// everyone role still present and it is using "hardcoded" ID
// we can remove it
if err = store.DeleteRoleByID(ctx, s, obsoleteEveryoneID); err != nil {
return
}
// transfer all rbac rules
if err = s.TransferRbacRules(ctx, obsoleteEveryoneID, m["authenticated"].ID); err != nil {
return
}
}
// let's see if admin role is still here:
if m["admins"] != nil && m["admins"].ID == obsoleteAdminsID {
log.Info("migrating 'admins' role to new ID and renaming it to 'admin'")
// everyone role still present and it is using "hardcoded" ID
// we can remove it
m["admins"].ID = id.Next()
m["admins"].UpdatedAt = now()
m["admins"].Handle = "admin"
m["admins"].Name = "Administrator"
if err = store.DeleteRoleByID(ctx, s, obsoleteAdminsID); err != nil {
return
}
if err = store.CreateRole(ctx, s, m["admins"]); err != nil {
return
}
// transfer all rbac rules
if err = store.TransferRoleMembers(ctx, s, obsoleteAdminsID, m["admins"].ID); err != nil {
return
}
// transfer all rbac rules
if err = store.TransferRbacRules(ctx, s, obsoleteAdminsID, m["admins"].ID); err != nil {
return
}
}
return
}
pkg/provision/migrations_202109_settings_cleanup.go
-35
View File
@@ -1,35 +0,0 @@
package provision
import (
"context"
"github.com/cortezaproject/corteza-server/store"
"go.uber.org/zap"
)
func cleanupPre202109Settings(ctx context.Context, log *zap.Logger, s store.Storer) (err error) {
log.Info("cleaning up pre-2021.9 settings")
names := []string{
"compose.ui.namespace-switcher.enabled",
"compose.ui.namespace-switcher.default-open",
"messaging.message.attachments.enabled",
"messaging.message.attachments.max-size",
"messaging.message.attachments.mimetypes",
"messaging.message.attachments.source.camera.enabled",
"messaging.message.attachments.source.gallery.enabled",
"messaging.ui.browser-notifications.enabled",
"messaging.ui.browser-notifications.header",
"messaging.ui.browser-notifications.message-trim",
"messaging.ui.emoji.enabled",
"auth.external.session-store-secret",
"auth.external.session-store-secure",
}
for _, name := range names {
if err = store.DeleteSettingValueByNameOwnedBy(ctx, s, name, 0); err != nil {
return
}
}
return
}
pkg/provision/migrations_202203_rbac_resource_fix.go
-361
View File
@@ -1,361 +0,0 @@
package provision
import (
"context"
"fmt"
"strconv"
"strings"
composeTypes "github.com/cortezaproject/corteza-server/compose/types"
federationTypes "github.com/cortezaproject/corteza-server/federation/types"
"github.com/cortezaproject/corteza-server/pkg/filter"
"github.com/cortezaproject/corteza-server/pkg/rbac"
"github.com/cortezaproject/corteza-server/store"
"go.uber.org/zap"
)
type (
resIndex struct {
fields map[uint64]*composeTypes.ModuleField
modules map[uint64]*composeTypes.Module
records map[uint64]*composeTypes.Record
charts map[uint64]*composeTypes.Chart
pages map[uint64]*composeTypes.Page
exposedModules map[uint64]*federationTypes.ExposedModule
sharedModules map[uint64]*federationTypes.SharedModule
}
)
// MigrateOperations
func migratePost202203RbacRules(ctx context.Context, log *zap.Logger, s store.Storer) error {
return store.Tx(ctx, s, func(ctx context.Context, s store.Storer) error {
rr, _, err := store.SearchRbacRules(ctx, s, rbac.RuleFilter{})
if err != nil {
return err
}
log.Info("migrating RBAC resource rules to proper format", zap.Int("rules", len(rr)))
rx, err := preloadRbacResourceIndex(ctx, s)
if err != nil {
return err
}
var uniq = make(map[string]bool)
var uniqID = func(r *rbac.Rule) string {
return fmt.Sprintf("%s|%s|%d", r.Resource, r.Operation, r.RoleID)
}
for _, r := range rr {
var (
cr = *r
action = migratePost202203RbacRule(r, rx)
)
if action != 0 {
err = store.DeleteRbacRule(ctx, s, &cr)
if err != nil {
return fmt.Errorf("could not delete RBAC rule %s: %v", r, err)
}
if action == -1 {
log.Debug("removed obsolete RBAC rule", zap.Stringer("rule", r))
}
}
if action == 1 {
if uniq[uniqID(r)] {
log.Warn("skipping duplicate RBAC rule", zap.Stringer("rule", r))
continue
}
err = store.CreateRbacRule(ctx, s, r)
if err != nil {
return fmt.Errorf("could not create RBAC rule %s: %v", r, err)
}
uniq[uniqID(r)] = true
}
}
return nil
})
}
// 0 - no action
// -1 - remove
// 1 - update
func migratePost202203RbacRule(r *rbac.Rule, rx *resIndex) (op int) {
const (
wildCard = "*"
)
if r.Resource == "" {
return -1
}
// split the IDs
parts := strings.SplitN(r.Resource, "/", 4)
parts = parts[1:]
isWildCard := func(s string) bool { return strings.Contains(s, wildCard) }
parseUint := func(s string) uint64 {
ID, _ := strconv.ParseUint(s, 10, 64)
return ID
}
validID := func(i uint64) bool { return i > 0 }
invalid := false
for i, p := range parts {
if i == 0 || len(p) == 0 {
continue
}
if !invalid {
invalid = isWildCard(parts[i-1]) && !isWildCard(p)
}
}
if invalid {
op = 1
partsLen := len(parts)
ID := uint64(0)
p1 := uint64(0)
p2 := uint64(0)
if partsLen == 1 {
ID = parseUint(parts[0])
} else if partsLen == 2 {
p1 = parseUint(parts[0])
ID = parseUint(parts[1])
} else if partsLen == 3 {
p1 = parseUint(parts[0])
p2 = parseUint(parts[1])
ID = parseUint(parts[2])
}
rType := rbac.ResourceType(r.Resource)
switch rType {
case composeTypes.ModuleFieldResourceType:
if ID > 0 {
if f, ok := rx.fields[ID]; ok {
p2 = f.ModuleID
if m, ok := rx.modules[p2]; ok {
p1 = m.NamespaceID
if !validID(p1) {
return -1
}
} else {
return -1
}
} else {
return -1
}
}
r.Resource = composeTypes.ModuleFieldRbacResource(p1, p2, ID)
case composeTypes.ModuleResourceType:
if ID > 0 {
if r, ok := rx.modules[ID]; ok {
p1 = r.NamespaceID
if !validID(p1) {
return -1
}
} else {
return -1
}
}
r.Resource = composeTypes.ModuleRbacResource(p1, ID)
case composeTypes.RecordResourceType:
if ID > 0 {
if r, ok := rx.records[ID]; ok {
p2 = r.ModuleID
if m, ok := rx.modules[p2]; ok {
p1 = m.NamespaceID
if !validID(p1) {
return -1
}
} else {
return -1
}
} else {
return -1
}
}
r.Resource = composeTypes.RecordRbacResource(p1, p2, ID)
case composeTypes.ChartResourceType:
if ID > 0 {
if r, ok := rx.charts[ID]; ok {
p1 = r.NamespaceID
if !validID(p1) {
return -1
}
} else {
return -1
}
}
r.Resource = composeTypes.ChartRbacResource(p1, ID)
case composeTypes.PageResourceType:
if ID > 0 {
if r, ok := rx.pages[ID]; ok {
p1 = r.NamespaceID
if !validID(p1) {
return -1
}
} else {
return -1
}
}
r.Resource = composeTypes.PageRbacResource(p1, ID)
case federationTypes.ExposedModuleResourceType:
if ID > 0 {
if r, ok := rx.exposedModules[ID]; ok {
p1 = r.NodeID
if !validID(p1) {
return -1
}
} else {
return -1
}
}
r.Resource = federationTypes.ExposedModuleRbacResource(p1, ID)
case federationTypes.SharedModuleResourceType:
if ID > 0 {
if r, ok := rx.sharedModules[ID]; ok {
p1 = r.NodeID
if !validID(p1) {
return -1
}
} else {
return -1
}
}
r.Resource = federationTypes.SharedModuleRbacResource(p1, ID)
default:
r.Resource = rType + "/" + func() string {
if ID == 0 {
return "*"
}
return strconv.FormatUint(ID, 10)
}()
}
}
return
}
// helper to preload resources that may be used when properly constructing rules
func preloadRbacResourceIndex(ctx context.Context, s store.Storer) (*resIndex, error) {
rx := &resIndex{}
rx.records = make(map[uint64]*composeTypes.Record)
var getRecords func(*composeTypes.Module, *filter.PagingCursor) (composeTypes.RecordSet, composeTypes.RecordFilter, error)
getRecords = func(m *composeTypes.Module, cursor *filter.PagingCursor) (rr composeTypes.RecordSet, f composeTypes.RecordFilter, err error) {
rr, f, err = store.SearchComposeRecords(ctx, s, m, composeTypes.RecordFilter{
Paging: filter.Paging{
Limit: 10000,
PageCursor: cursor,
},
Deleted: filter.StateInclusive,
})
if err != nil {
return
}
for _, rec := range rr {
rx.records[rec.ID] = rec
}
if f.NextPage != nil {
_, _, err = getRecords(m, f.NextPage)
if err != nil {
return
}
}
return
}
rx.modules = make(map[uint64]*composeTypes.Module)
modules, _, err := store.SearchComposeModules(ctx, s, composeTypes.ModuleFilter{
Paging: filter.Paging{Limit: 0},
Deleted: filter.StateInclusive,
})
modIDs := make([]uint64, 0, len(modules))
if err != nil {
return nil, err
}
for _, r := range modules {
rx.modules[r.ID] = r
modIDs = append(modIDs, r.ID)
_, _, err = getRecords(r, nil)
if err != nil {
return nil, err
}
}
if len(modIDs) > 0 {
rx.fields = make(map[uint64]*composeTypes.ModuleField)
fields, _, err := store.SearchComposeModuleFields(ctx, s, composeTypes.ModuleFieldFilter{
ModuleID: modIDs,
Deleted: filter.StateInclusive,
})
if err != nil {
return nil, err
}
for _, r := range fields {
rx.fields[r.ID] = r
}
}
rx.charts = make(map[uint64]*composeTypes.Chart)
chart, _, err := store.SearchComposeCharts(ctx, s, composeTypes.ChartFilter{
Paging: filter.Paging{Limit: 0},
Deleted: filter.StateInclusive,
})
if err != nil {
return nil, err
}
for _, r := range chart {
rx.charts[r.ID] = r
}
rx.pages = make(map[uint64]*composeTypes.Page)
page, _, err := store.SearchComposePages(ctx, s, composeTypes.PageFilter{
Paging: filter.Paging{Limit: 0},
Deleted: filter.StateInclusive,
})
if err != nil {
return nil, err
}
for _, r := range page {
rx.pages[r.ID] = r
}
rx.exposedModules = make(map[uint64]*federationTypes.ExposedModule)
exposedModule, _, err := store.SearchFederationExposedModules(ctx, s, federationTypes.ExposedModuleFilter{Paging: filter.Paging{Limit: 0}})
if err != nil {
return nil, err
}
for _, r := range exposedModule {
rx.exposedModules[r.ID] = r
}
rx.sharedModules = make(map[uint64]*federationTypes.SharedModule)
sharedModule, _, err := store.SearchFederationSharedModules(ctx, s, federationTypes.SharedModuleFilter{Paging: filter.Paging{Limit: 0}})
if err != nil {
return nil, err
}
for _, r := range sharedModule {
rx.sharedModules[r.ID] = r
}
return rx, nil
}
pkg/provision/migrations_202203_rbac_resource_fix_test.go
-39
View File
@@ -1,39 +0,0 @@
package provision
import (
"github.com/stretchr/testify/require"
"testing"
composeTypes "github.com/cortezaproject/corteza-server/compose/types"
federationTypes "github.com/cortezaproject/corteza-server/federation/types"
"github.com/cortezaproject/corteza-server/pkg/rbac"
)
func Test_migratePost202203RbacRules(t *testing.T) {
rx := &resIndex{
fields: make(map[uint64]*composeTypes.ModuleField),
modules: make(map[uint64]*composeTypes.Module),
charts: make(map[uint64]*composeTypes.Chart),
pages: make(map[uint64]*composeTypes.Page),
exposedModules: make(map[uint64]*federationTypes.ExposedModule),
sharedModules: make(map[uint64]*federationTypes.SharedModule),
}
tcc := []struct {
wantOp int
rule *rbac.Rule
wantRule *rbac.Rule
}{
{-1, rbac.AllowRule(0, "corteza::compose:module/*/123", "op"), rbac.AllowRule(0, "corteza::compose:module/*/123", "op")},
{-1, rbac.AllowRule(0, "corteza::compose:module-field/*/*/123", "op"), rbac.AllowRule(0, "corteza::compose:module-field/*/*/123", "op")},
{-1, rbac.AllowRule(0, "corteza::compose:record/*/*/234", "op"), rbac.AllowRule(0, "corteza::compose:record/*/*/234", "op")},
}
for _, tc := range tcc {
t.Run(tc.rule.String(), func(t *testing.T) {
require.Equal(t, tc.wantOp, migratePost202203RbacRule(tc.rule, rx))
if tc.wantRule != nil {
require.Equal(t, tc.wantRule.String(), tc.rule.String())
}
})
}
}
pkg/provision/migrations_202203_report_identifiers.go
-68
View File
@@ -1,68 +0,0 @@
package provision
import (
"context"
"strconv"
"github.com/cortezaproject/corteza-server/pkg/filter"
"github.com/cortezaproject/corteza-server/pkg/id"
"github.com/cortezaproject/corteza-server/store"
sysTypes "github.com/cortezaproject/corteza-server/system/types"
"github.com/spf13/cast"
"go.uber.org/zap"
)
// Migrates existing reports to include the newly added resource identifiers
func migrateReportIdentifiers(ctx context.Context, log *zap.Logger, s store.Storer) (err error) {
log.Info("migrating report identifiers")
set, _, err := store.SearchReports(ctx, s, sysTypes.ReportFilter{Deleted: filter.StateInclusive})
if err != nil {
return err
}
return s.Tx(ctx, func(ctx context.Context, s store.Storer) error {
return set.Walk(func(r *sysTypes.Report) (err error) {
r = setIDs(r)
return store.UpdateReport(ctx, s, r)
})
})
}
// duplicate from the report service
func setIDs(r *sysTypes.Report) *sysTypes.Report {
// scenarios
for _, s := range r.Scenarios {
if s.ScenarioID == 0 {
s.ScenarioID = id.Next()
}
}
// blocks
for _, b := range r.Blocks {
if b.BlockID == 0 {
b.BlockID = id.Next()
}
// elements
for _, elRaw := range b.Elements {
el, ok := elRaw.(map[string]interface{})
if !ok {
continue
}
elID, ok := el["elementID"]
sElID := cast.ToString(elID)
if sElID != "" && sElID != "0" {
continue
}
if cast.ToUint64(elID) != 0 {
continue
}
el["elementID"] = strconv.FormatUint(id.Next(), 10)
}
}
return r
}
pkg/provision/migrations_202203_resource_translations.go
-116
View File
@@ -1,116 +0,0 @@
package provision
import (
"context"
"fmt"
cmpTypes "github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/filter"
"github.com/cortezaproject/corteza-server/pkg/id"
"github.com/cortezaproject/corteza-server/store"
sysTypes "github.com/cortezaproject/corteza-server/system/types"
"go.uber.org/zap"
"strings"
)
// Migrates resource translations from the resource
// struct to the dedicated store (table)
//
// While doing this, we also modify some resource substructure:
// - chart reports (assign report IDs)
//
// Note: we will migrate all translations to current default language
// If you do not like that, shut down Corteza after migrations and fix this directly in the store
func migratePost202203ResourceTranslations(ctx context.Context, log *zap.Logger, s store.Storer) (err error) {
log.Info("migrating post 202203 resource translations")
var (
migrated = make(map[string]bool)
)
set, _, err := store.SearchResourceTranslations(ctx, s, sysTypes.ResourceTranslationFilter{})
if err != nil {
return
}
err = set.Walk(func(r *sysTypes.ResourceTranslation) error {
var pos = strings.Index(r.Resource, "/")
if pos < 0 {
return nil
}
migrated[r.Resource[0:pos]] = true
return nil
})
if err != nil {
return
}
if !migrated[cmpTypes.ChartResourceTranslationType] {
if err = migrateComposeChartResourceTranslations(ctx, log, s); err != nil {
return
}
}
return
}
// migrate resource translations for compose chart
func migrateComposeChartResourceTranslations(ctx context.Context, log *zap.Logger, s store.Storer) error {
var (
tt sysTypes.ResourceTranslationSet
)
set, _, err := store.SearchComposeCharts(ctx, s, cmpTypes.ChartFilter{Deleted: filter.StateInclusive})
if err != nil {
return err
}
log.Info("migrating compose charts", zap.Int("count", len(set)))
return s.Tx(ctx, func(ctx context.Context, s store.Storer) error {
return set.Walk(func(res *cmpTypes.Chart) (err error) {
if tt, err = convertComposeChartReportsTranslations(res); err != nil {
return err
}
if err = store.CreateResourceTranslation(ctx, s, tt...); err != nil {
return
}
if err = store.UpdateComposeChart(ctx, s, res); err != nil {
return
}
return
})
})
}
// collects translations for compose chart blocks and alters them (adding block ID)
func convertComposeChartReportsTranslations(res *cmpTypes.Chart) (sysTypes.ResourceTranslationSet, error) {
var tt sysTypes.ResourceTranslationSet
for i, r := range res.Config.Reports {
reportID := id.Next()
res.Config.Reports[i].ReportID = reportID
if _, ok := r.YAxis["label"]; ok {
tt = append(tt,
makeResourceTranslation(res, "yAxis.label", r.YAxis["label"].(string)),
)
}
// Ensure chart report metric IDs
for j, m := range r.Metrics {
metricID := id.Next()
res.Config.Reports[i].Metrics[j]["metricID"] = fmt.Sprintf("%d", metricID)
cmx := fmt.Sprintf("metrics.%d.", metricID)
if _, ok := m["label"]; ok {
tt = append(tt,
makeResourceTranslation(res, cmx+"label", m["label"].(string)),
)
}
}
}
return tt, nil
}
pkg/provision/provision.go
+6 -13
View File
@@ -24,25 +24,18 @@ var (
func Run(ctx context.Context, log *zap.Logger, s store.Storer, provisionOpt options.ProvisionOpt, authOpt options.AuthOpt) error {
log = log.Named("provision")
// Note,
ffn := []func() error{
// Migrations:
func() error { return migrateApplications(ctx, s) },
func() error { return migrateEmailTemplates(ctx, log.Named("email-templates"), s) },
func() error { return migratePre202109Roles(ctx, log.Named("pre-202109-roles"), s) },
func() error { return migratePre202109RbacRules(ctx, log.Named("pre-202109-rbac-rules"), s) },
func() error { return cleanupPre202109Settings(ctx, log.Named("pre-202109-settings"), s) },
func() error { return migrateResourceTranslations(ctx, log.Named("resource-translations"), s) },
func() error { return migrateReportIdentifiers(ctx, log.Named("report-identifiers"), s) },
func() error {
return migratePost202203RbacRules(ctx, log.Named("post-202203-rbac-resource-rules-fix"), s)
},
func() error {
return migratePost202203ResourceTranslations(ctx, log.Named("post-202203-resource-translations"), s)
},
// (placeholder for all post 2022.3.x modifications)
// *************************************************************************************************************
// Config (full & partial)
func() error { return importConfig(ctx, log.Named("config"), s, provisionOpt.Path) },
// *************************************************************************************************************
// Auto-discoveries and other parts that cannot be imported from static files
func() error { return emailSettings(ctx, s) },
func() error { return authAddExternals(ctx, log.Named("auth.externals"), s) },