diff --git a/pkg/provision/migrations.go b/pkg/provision/migrations.go deleted file mode 100644 index 6b0b03e68..000000000 --- a/pkg/provision/migrations.go +++ /dev/null @@ -1,7 +0,0 @@ -package provision - -import "time" - -func releaseDate(y int, m time.Month) time.Time { - return time.Date(y, m+1, 1, 0, 0, 0, 0, time.UTC) -} diff --git a/pkg/provision/migrations_202103_apps.go b/pkg/provision/migrations_202103_apps.go deleted file mode 100644 index 3d999255e..000000000 --- a/pkg/provision/migrations_202103_apps.go +++ /dev/null @@ -1,34 +0,0 @@ -package provision - -import ( - "context" - "github.com/cortezaproject/corteza-server/store" - "github.com/cortezaproject/corteza-server/system/types" - "time" -) - -func migrateApplications(ctx context.Context, s store.Storer) error { - rd := releaseDate(2021, time.March) - - set, _, err := store.SearchApplications(ctx, s, types.ApplicationFilter{}) - if err != nil { - return err - } - - return set.Walk(func(app *types.Application) (err error) { - if app.Unify == nil { - return - } - - if app.Unify.Url == "/messaging" && (app.UpdatedAt == nil || app.UpdatedAt.Before(rd)) { - // Disable and un-list messaging app - // but only if it was not updated after 21.3 release date - app.Unify.Listed = false - app.Enabled = false - - return store.UpdateApplication(ctx, s, app) - } - - return nil - }) -} diff --git a/pkg/provision/migrations_202103_templates.go b/pkg/provision/migrations_202103_templates.go deleted file mode 100644 index f4eb73b73..000000000 --- a/pkg/provision/migrations_202103_templates.go +++ /dev/null @@ -1,107 +0,0 @@ -package provision - -import ( - "context" - "fmt" - - "github.com/cortezaproject/corteza-server/pkg/errors" - "github.com/cortezaproject/corteza-server/pkg/id" - "github.com/cortezaproject/corteza-server/store" - "github.com/cortezaproject/corteza-server/system/types" - "go.uber.org/zap" -) - -// Pre 2021.3 versions had email templates stored in settings -// from 2021.3 onwards we have dedicated subsystem for managing templates -// -// This migration moves email templates from settings templates. -func migrateEmailTemplates(ctx context.Context, log *zap.Logger, s store.Storer) error { - var ( - // setting name => template handle - m = map[string]*types.Template{ - //"general.mail.logo", - "general.mail.header.en": { - Type: "text/html", - Handle: "email_general_header", - Partial: true, - Meta: types.TemplateMeta{ - Short: "General template header", - Description: "General template header to use with system email notifications", - }, - }, - "general.mail.footer.en": { - Type: "text/html", - Handle: "email_general_footer", - Partial: true, - Meta: types.TemplateMeta{ - Short: "General template footer", - Description: "General template footer to use with system email notifications", - }, - }, - "auth.mail.email-confirmation.subject.en": { - Type: "text/plain", - Handle: "auth_email_confirm_subject", - Meta: types.TemplateMeta{Short: "Password reset subject"}, - }, - "auth.mail.email-confirmation.body.en": { - Type: "text/html", - Handle: "auth_email_confirm_body", - Meta: types.TemplateMeta{Short: "Password reset content"}, - }, - "auth.mail.password-reset.subject.en": { - Type: "text/plain", - Handle: "auth_email_password_reset_subject", - Meta: types.TemplateMeta{Short: "Email confirmation subject"}, - }, - "auth.mail.password-reset.body.en": { - Type: "text/html", - Handle: "auth_email_password_reset_body", - Meta: types.TemplateMeta{Short: "Email confirmation content"}, - }, - } - ) - - return store.Tx(ctx, s, func(ctx context.Context, s store.Storer) error { - for name, tmpl := range m { - sval, err := store.LookupSettingValueByNameOwnedBy(ctx, s, name, 0) - if errors.IsNotFound(err) { - // setting not found, that's ok, see the next one - continue - } else if err != nil { - return fmt.Errorf("failed to lookup for setting by bame: %w", err) - } - - _, err = store.LookupTemplateByHandle(ctx, s, tmpl.Handle) - if err != nil && !errors.IsNotFound(err) { - // any error but not-found is fatal. - return fmt.Errorf("failed to lookup for template by handle: %w", err) - } else if err == nil { - // template exists - continue - } - - tmpl.ID = id.Next() - tmpl.CreatedAt = *now() - tmpl.Template = sval.String() - - err = store.CreateTemplate(ctx, s, tmpl) - if err != nil { - return fmt.Errorf("failed to store migrated template: %w", err) - } - - log.Debug("migrated template from settings", - zap.String("setting", name), - zap.String("handle", tmpl.Handle), - ) - } - - // Go over all settings again and remove them - for name := range m { - if err := store.DeleteSettingValueByNameOwnedBy(ctx, s, name, 0); err != nil { - return err - } - } - - return nil - }) -} diff --git a/pkg/provision/migrations_202109_rbac.go b/pkg/provision/migrations_202109_rbac.go deleted file mode 100644 index 9a46c97cd..000000000 --- a/pkg/provision/migrations_202109_rbac.go +++ /dev/null @@ -1,297 +0,0 @@ -package provision - -import ( - "context" - "fmt" - "strconv" - "strings" - - composeTypes "github.com/cortezaproject/corteza-server/compose/types" - federationTypes "github.com/cortezaproject/corteza-server/federation/types" - "github.com/cortezaproject/corteza-server/pkg/filter" - "github.com/cortezaproject/corteza-server/pkg/rbac" - "github.com/cortezaproject/corteza-server/store" - systemTypes "github.com/cortezaproject/corteza-server/system/types" - "go.uber.org/zap" -) - -type ( - resourceIndex struct { - fields map[uint64]*composeTypes.ModuleField - modules map[uint64]*composeTypes.Module - charts map[uint64]*composeTypes.Chart - pages map[uint64]*composeTypes.Page - exposedModules map[uint64]*federationTypes.ExposedModule - sharedModules map[uint64]*federationTypes.SharedModule - } -) - -// MigrateOperations creates system roles -func migratePre202109RbacRules(ctx context.Context, log *zap.Logger, s store.Storer) error { - return store.Tx(ctx, s, func(ctx context.Context, s store.Storer) error { - rr, _, err := store.SearchRbacRules(ctx, s, rbac.RuleFilter{}) - if err != nil { - return err - } - - log.Info("migrating RBAC rules to new format", zap.Int("rules", len(rr))) - - rx, err := preloadResourceIndex(ctx, s) - if err != nil { - return err - } - - var uniq = make(map[string]bool) - var uniqID = func(r *rbac.Rule) string { - return fmt.Sprintf("%s|%s|%d", r.Resource, r.Operation, r.RoleID) - } - - for _, r := range rr { - var ( - cr = *r - action = migratePre202109RbacRule(r, rx) - ) - - if action != 0 { - err = store.DeleteRbacRule(ctx, s, &cr) - if err != nil { - return fmt.Errorf("could not delete RBAC rule %s: %v", r, err) - } - - if action == -1 { - log.Debug("removed obsolete RBAC rule", zap.Stringer("rule", r)) - } - } - - if action == 1 { - if uniq[uniqID(r)] { - log.Warn("skipping duplicate RBAC rule", zap.Stringer("rule", r)) - continue - } - - err = store.CreateRbacRule(ctx, s, r) - if err != nil { - return fmt.Errorf("could not create RBAC rule %s: %v", r, err) - } - - uniq[uniqID(r)] = true - } - } - - return nil - }) -} - -// 0 - no action -// -1 - remove -// 1 - update -func migratePre202109RbacRule(r *rbac.Rule, rx *resourceIndex) (op int) { - const ( - nsSep = "::" - nsDef = "corteza" - ) - - if strings.Contains(r.Resource, nsSep) { - return - } - - if r.Resource == "" { - return -1 - } - - // split old format - parts := strings.SplitN(r.Resource, ":", 3) - - switch { - case parts[0] == "messaging": - return -1 - case len(parts) > 1 && parts[1] == "automation-script": - return -1 - case len(parts) == 1: - r.Resource = nsDef + nsSep + strings.Join(parts[:1], ":") - default: - r.Resource = nsDef + nsSep + strings.Join(parts[:2], ":") - } - - op = 1 - - rType := rbac.ResourceType(r.Resource) - switch { - case rType == systemTypes.UserResourceType && strings.HasPrefix(r.Operation, "unmask."): - // flipping terms in user unmask operations - r.Operation = strings.TrimPrefix(r.Operation, "unmask.") + ".unmask" - - case rType == "corteza::federation:module": - // fed. module resource was split into two resources - exposed & shared - if r.Operation == "manage" { - rType = federationTypes.ExposedModuleResourceType - } else if r.Operation == "map" { - rType = federationTypes.SharedModuleResourceType - } else { - return -1 - } - - case rType == composeTypes.ModuleResourceType && strings.HasPrefix(r.Operation, "record.") && r.Operation != "record.create": - // change resource type from module to record on record read, delete, update operations & remove the prefix - rType = composeTypes.RecordResourceType - r.Operation = strings.TrimPrefix(r.Operation, "record.") - } - - if len(parts) == 3 { - var ID, _ = strconv.ParseUint(parts[2], 10, 64) - - p1 := uint64(0) - p2 := uint64(0) - - // exceptions with nested references - switch rType { - case composeTypes.ModuleFieldResourceType: - if ID > 0 { - if f, ok := rx.fields[ID]; ok { - p1 = f.NamespaceID - p2 = f.ModuleID - } - } - r.Resource = composeTypes.ModuleFieldRbacResource(p1, p2, ID) - - case composeTypes.ModuleResourceType: - if ID > 0 { - if r, ok := rx.modules[ID]; ok { - p1 = r.NamespaceID - } - } - r.Resource = composeTypes.ModuleRbacResource(p1, ID) - - // ID belongs to module! - case composeTypes.RecordResourceType: - if ID > 0 { - if r, ok := rx.modules[ID]; ok { - p1 = r.NamespaceID - } - } - r.Resource = composeTypes.RecordRbacResource(p1, ID, 0) - - case composeTypes.ChartResourceType: - if ID > 0 { - if r, ok := rx.charts[ID]; ok { - p1 = r.NamespaceID - } - } - r.Resource = composeTypes.ChartRbacResource(p1, ID) - - case composeTypes.PageResourceType: - if ID > 0 { - if r, ok := rx.pages[ID]; ok { - p1 = r.NamespaceID - } - } - r.Resource = composeTypes.PageRbacResource(p1, ID) - - case federationTypes.ExposedModuleResourceType: - if ID > 0 { - if r, ok := rx.exposedModules[ID]; ok { - p1 = r.NodeID - } - } - r.Resource = federationTypes.ExposedModuleRbacResource(p1, ID) - - case federationTypes.SharedModuleResourceType: - if ID > 0 { - if r, ok := rx.sharedModules[ID]; ok { - p1 = r.NodeID - } - } - r.Resource = federationTypes.SharedModuleRbacResource(p1, ID) - - default: - r.Resource = rType + "/" + func() string { - if ID == 0 { - return "*" - } - return parts[2] - }() - } - - } else { - r.Resource = rType + "/" - } - - return -} - -// helper to preloadresources that may be used when properly constructing rules -func preloadResourceIndex(ctx context.Context, s store.Storer) (*resourceIndex, error) { - rx := &resourceIndex{} - - rx.modules = make(map[uint64]*composeTypes.Module) - modules, _, err := store.SearchComposeModules(ctx, s, composeTypes.ModuleFilter{ - Paging: filter.Paging{Limit: 0}, - Deleted: filter.StateInclusive, - }) - modIDs := make([]uint64, 0, len(modules)) - if err != nil { - return nil, err - } - for _, r := range modules { - rx.modules[r.ID] = r - modIDs = append(modIDs, r.ID) - } - - if len(modIDs) > 0 { - rx.fields = make(map[uint64]*composeTypes.ModuleField) - fields, _, err := store.SearchComposeModuleFields(ctx, s, composeTypes.ModuleFieldFilter{ - ModuleID: modIDs, - Deleted: filter.StateInclusive, - }) - if err != nil { - return nil, err - } - for _, r := range fields { - rx.fields[r.ID] = r - } - } - - rx.charts = make(map[uint64]*composeTypes.Chart) - chart, _, err := store.SearchComposeCharts(ctx, s, composeTypes.ChartFilter{ - Paging: filter.Paging{Limit: 0}, - Deleted: filter.StateInclusive, - }) - if err != nil { - return nil, err - } - for _, r := range chart { - rx.charts[r.ID] = r - } - - rx.pages = make(map[uint64]*composeTypes.Page) - page, _, err := store.SearchComposePages(ctx, s, composeTypes.PageFilter{ - Paging: filter.Paging{Limit: 0}, - Deleted: filter.StateInclusive, - }) - if err != nil { - return nil, err - } - for _, r := range page { - rx.pages[r.ID] = r - } - - rx.exposedModules = make(map[uint64]*federationTypes.ExposedModule) - exposedModule, _, err := store.SearchFederationExposedModules(ctx, s, federationTypes.ExposedModuleFilter{Paging: filter.Paging{Limit: 0}}) - if err != nil { - return nil, err - } - for _, r := range exposedModule { - rx.exposedModules[r.ID] = r - } - - rx.sharedModules = make(map[uint64]*federationTypes.SharedModule) - sharedModule, _, err := store.SearchFederationSharedModules(ctx, s, federationTypes.SharedModuleFilter{Paging: filter.Paging{Limit: 0}}) - if err != nil { - return nil, err - } - for _, r := range sharedModule { - rx.sharedModules[r.ID] = r - } - - return rx, nil -} diff --git a/pkg/provision/migrations_202109_rbac_test.go b/pkg/provision/migrations_202109_rbac_test.go deleted file mode 100644 index 42788fe88..000000000 --- a/pkg/provision/migrations_202109_rbac_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package provision - -import ( - "testing" - - composeTypes "github.com/cortezaproject/corteza-server/compose/types" - federationTypes "github.com/cortezaproject/corteza-server/federation/types" - "github.com/cortezaproject/corteza-server/pkg/rbac" - "github.com/stretchr/testify/require" -) - -func Test_migratePre202109RbacRule(t *testing.T) { - rx := &resourceIndex{ - fields: make(map[uint64]*composeTypes.ModuleField), - modules: make(map[uint64]*composeTypes.Module), - charts: make(map[uint64]*composeTypes.Chart), - pages: make(map[uint64]*composeTypes.Page), - exposedModules: make(map[uint64]*federationTypes.ExposedModule), - sharedModules: make(map[uint64]*federationTypes.SharedModule), - } - - tcc := []struct { - wantOp int - rule *rbac.Rule - wantRule *rbac.Rule - }{ - {-1, rbac.AllowRule(0, "messaging", "whatever"), nil}, - {-1, rbac.AllowRule(0, "foo:automation-script", "whatever"), nil}, - {1, rbac.AllowRule(0, "abc", "whatever"), rbac.AllowRule(0, "corteza::abc/", "whatever")}, - {1, rbac.AllowRule(0, "federation:module", "map"), rbac.AllowRule(0, "corteza::federation:shared-module/", "map")}, - {1, rbac.AllowRule(0, "federation:module", "manage"), rbac.AllowRule(0, "corteza::federation:exposed-module/", "manage")}, - {1, rbac.AllowRule(0, "compose:module:234", "record.read"), rbac.AllowRule(0, "corteza::compose:record/*/234/*", "read")}, - {1, rbac.AllowRule(0, "compose:module-field:234", "op"), rbac.AllowRule(0, "corteza::compose:module-field/*/*/234", "op")}, - } - for _, tc := range tcc { - t.Run(tc.rule.String(), func(t *testing.T) { - require.Equal(t, tc.wantOp, migratePre202109RbacRule(tc.rule, rx)) - if tc.wantRule != nil { - require.Equal(t, tc.wantRule.String(), tc.rule.String()) - } - }) - } -} diff --git a/pkg/provision/migrations_202109_resource_translations.go b/pkg/provision/migrations_202109_resource_translations.go deleted file mode 100644 index 81c77f9da..000000000 --- a/pkg/provision/migrations_202109_resource_translations.go +++ /dev/null @@ -1,266 +0,0 @@ -package provision - -import ( - "context" - "fmt" - "strings" - - cmpTypes "github.com/cortezaproject/corteza-server/compose/types" - "github.com/cortezaproject/corteza-server/pkg/filter" - "github.com/cortezaproject/corteza-server/pkg/id" - "github.com/cortezaproject/corteza-server/pkg/locale" - "github.com/cortezaproject/corteza-server/store" - sysTypes "github.com/cortezaproject/corteza-server/system/types" - "go.uber.org/zap" -) - -// Migrates resource translations from the resource -// struct to the dedicated store (table) -// -// While doing this, we also modify some resource substructure: -// - page blocks (assign page block IDs -// - automation buttons on page blocks (assign automation button IDs) -// - field expressions (assign expression IDs) -// -// Note: we will migrate all translations to current default language -// If you do not like that, shut down Corteza after migrations and fix this directly in the store -func migrateResourceTranslations(ctx context.Context, log *zap.Logger, s store.Storer) (err error) { - log.Info("migrating resource locales") - - var ( - migrated = make(map[string]bool) - ) - - set, _, err := store.SearchResourceTranslations(ctx, s, sysTypes.ResourceTranslationFilter{}) - set.Walk(func(r *sysTypes.ResourceTranslation) error { - var pos = strings.Index(r.Resource, "/") - if pos < 0 { - return nil - } - - migrated[r.Resource[0:pos]] = true - return nil - }) - - if !migrated[cmpTypes.NamespaceResourceTranslationType] { - if err = migrateComposeNamespaceResourceTranslations(ctx, log, s); err != nil { - return - } - } - - if !migrated[cmpTypes.ModuleResourceTranslationType] { - // @todo migrate module locales - if err = migrateComposeModuleResourceTranslations(ctx, log, s); err != nil { - return - } - } - - if !migrated[cmpTypes.PageResourceTranslationType] { - if err = migrateComposePageResourceTranslations(ctx, log, s); err != nil { - return - } - } - - return -} - -// helper fn that creates ResourceTranslation from the given resource, key and message -// and default language -func makeResourceTranslation(lr interface{ ResourceTranslation() string }, k, m string) *sysTypes.ResourceTranslation { - return &sysTypes.ResourceTranslation{ - ID: id.Next(), - Lang: sysTypes.Lang{Tag: locale.Global().Default().Tag}, - Resource: lr.ResourceTranslation(), - K: k, - Message: m, - CreatedAt: *now(), - } -} - -// migrate resource translations for compose namespace -func migrateComposeNamespaceResourceTranslations(ctx context.Context, log *zap.Logger, s store.Storer) error { - set, _, err := store.SearchComposeNamespaces(ctx, s, cmpTypes.NamespaceFilter{Deleted: filter.StateInclusive}) - if err != nil { - return err - } - - log.Info("migrating compose namespaces", zap.Int("count", len(set))) - - return s.Tx(ctx, func(ctx context.Context, s store.Storer) error { - return set.Walk(func(res *cmpTypes.Namespace) error { - return store.CreateResourceTranslation(ctx, s, - makeResourceTranslation(res, "name", res.Name), - makeResourceTranslation(res, "subtitle", res.Meta.Subtitle), - makeResourceTranslation(res, "description", res.Meta.Description), - ) - }) - }) -} - -// migrate resource translations for compose module -func migrateComposeModuleResourceTranslations(ctx context.Context, log *zap.Logger, s store.Storer) error { - set, _, err := store.SearchComposeModules(ctx, s, cmpTypes.ModuleFilter{Deleted: filter.StateInclusive}) - if err != nil { - return err - } - - log.Info("migrating compose modules", zap.Int("count", len(set))) - - return s.Tx(ctx, func(ctx context.Context, s store.Storer) error { - return set.Walk(func(res *cmpTypes.Module) (err error) { - if err = migrateComposeModuleFieldResourceTranslations(ctx, s, res.NamespaceID, res.ID); err != nil { - return err - } - - err = store.CreateResourceTranslation(ctx, s, - makeResourceTranslation(res, "name", res.Name), - ) - - if err != nil { - return err - } - - return store.UpdateComposeModule(ctx, s, res) - }) - }) -} - -// migrate module fields translations for compose module field -// -// Adds validatorID on field expressions -func migrateComposeModuleFieldResourceTranslations(ctx context.Context, s store.Storer, namespaceID uint64, moduleIDs ...uint64) error { - set, _, err := store.SearchComposeModuleFields(ctx, s, cmpTypes.ModuleFieldFilter{ModuleID: moduleIDs}) - if err != nil { - return err - } - - return s.Tx(ctx, func(ctx context.Context, s store.Storer) error { - return set.Walk(func(res *cmpTypes.ModuleField) (err error) { - tt := sysTypes.ResourceTranslationSet{} - - // store does not contain this info, - // but we need it to generate resource id - res.NamespaceID = namespaceID - tt = append(tt, makeResourceTranslation(res, "label", res.Label)) - - var update = len(res.Expressions.Validators) > 0 - - for i := range res.Expressions.Validators { - validatorID := i + 1 - res.Expressions.Validators[i].ValidatorID = uint64(validatorID) - tt = append(tt, makeResourceTranslation(res, fmt.Sprintf("expression.validator.%d.error", validatorID), res.Expressions.Validators[i].Error)) - } - - if err = store.CreateResourceTranslation(ctx, s, tt...); err != nil { - return - } - - if update { - if err = store.UpdateComposeModuleField(ctx, s, res); err != nil { - return - } - } - - return nil - }) - }) -} - -// migrate resource translations for compose module -func migrateComposePageResourceTranslations(ctx context.Context, log *zap.Logger, s store.Storer) error { - var tt sysTypes.ResourceTranslationSet - set, _, err := store.SearchComposePages(ctx, s, cmpTypes.PageFilter{Deleted: filter.StateInclusive}) - if err != nil { - return err - } - - // @todo migrate page locales - // @todo page block IDs - // @todo automation button IDs - - log.Info("migrating compose pages", zap.Int("count", len(set))) - - return s.Tx(ctx, func(ctx context.Context, s store.Storer) error { - return set.Walk(func(res *cmpTypes.Page) (err error) { - tt = sysTypes.ResourceTranslationSet{ - makeResourceTranslation(res, "title", res.Title), - makeResourceTranslation(res, "description", res.Description), - } - - if pbtt, err := convertComposePageBlockTranslations(res); err != nil { - return err - } else { - tt = append(tt, pbtt...) - } - - if err = store.CreateResourceTranslation(ctx, s, tt...); err != nil { - return - } - - if err = store.UpdateComposePage(ctx, s, res); err != nil { - return - } - - return - }) - }) -} - -// collects translations for compose page blocks and alters them (adding block ID) -func convertComposePageBlockTranslations(res *cmpTypes.Page) (sysTypes.ResourceTranslationSet, error) { - var tt sysTypes.ResourceTranslationSet - - for i, b := range res.Blocks { - blockID := i + 1 - res.Blocks[i].BlockID = uint64(blockID) - pfx := fmt.Sprintf("pageBlock.%d.", blockID) - - tt = append(tt, - makeResourceTranslation(res, pfx+"title", b.Title), - makeResourceTranslation(res, pfx+"description", b.Description), - ) - - switch b.Kind { - case "Automation": - att, bb, err := convertComposeAutomationPageBlockTranslations(res, pfx, res.Blocks[i].Options) - if err != nil { - return nil, err - } - - res.Blocks[i].Options["buttons"] = bb - tt = append(tt, att...) - } - } - - return tt, nil -} - -// takes automation options for page block and returns translations + fixed set of buttons (containing content ID) -func convertComposeAutomationPageBlockTranslations(res *cmpTypes.Page, pfx string, opt map[string]interface{}) (tt sysTypes.ResourceTranslationSet, bb []interface{}, err error) { - if tmp, ok := opt["buttons"]; !ok { - return nil, nil, nil - } else if bb, ok = tmp.([]interface{}); !ok { - return nil, nil, nil - } - - for i := range bb { - b, is := bb[i].(map[string]interface{}) - if !is { - continue - } - - buttonID := i + 1 - - b["buttonID"] = buttonID - bb[i] = b - - var label string - if _, has := b["label"]; has { - label = b["label"].(string) - } - - tt = append(tt, makeResourceTranslation(res, fmt.Sprintf("%sbuttons.%d.label", pfx, buttonID), label)) - } - - return -} diff --git a/pkg/provision/migrations_202109_roles.go b/pkg/provision/migrations_202109_roles.go deleted file mode 100644 index bdeb8d7b1..000000000 --- a/pkg/provision/migrations_202109_roles.go +++ /dev/null @@ -1,70 +0,0 @@ -package provision - -import ( - "context" - - "github.com/cortezaproject/corteza-server/pkg/id" - "github.com/cortezaproject/corteza-server/store" - "go.uber.org/zap" -) - -func migratePre202109Roles(ctx context.Context, log *zap.Logger, s store.Storer) (err error) { - const ( - obsoleteEveryoneID uint64 = 1 - obsoleteAdminsID uint64 = 2 - ) - - log.Info("migrating pre-2021.9 roles") - m, err := loadRoles(ctx, s) - if err != nil { - return - } - - // let's see if everyone role is still here: - if m["everyone"] != nil && m["everyone"].ID == obsoleteEveryoneID { - log.Info("migrating 'everyone' role to new ID") - - // everyone role still present and it is using "hardcoded" ID - // we can remove it - if err = store.DeleteRoleByID(ctx, s, obsoleteEveryoneID); err != nil { - return - } - - // transfer all rbac rules - if err = s.TransferRbacRules(ctx, obsoleteEveryoneID, m["authenticated"].ID); err != nil { - return - } - } - - // let's see if admin role is still here: - if m["admins"] != nil && m["admins"].ID == obsoleteAdminsID { - log.Info("migrating 'admins' role to new ID and renaming it to 'admin'") - - // everyone role still present and it is using "hardcoded" ID - // we can remove it - m["admins"].ID = id.Next() - m["admins"].UpdatedAt = now() - m["admins"].Handle = "admin" - m["admins"].Name = "Administrator" - - if err = store.DeleteRoleByID(ctx, s, obsoleteAdminsID); err != nil { - return - } - - if err = store.CreateRole(ctx, s, m["admins"]); err != nil { - return - } - - // transfer all rbac rules - if err = store.TransferRoleMembers(ctx, s, obsoleteAdminsID, m["admins"].ID); err != nil { - return - } - - // transfer all rbac rules - if err = store.TransferRbacRules(ctx, s, obsoleteAdminsID, m["admins"].ID); err != nil { - return - } - } - - return -} diff --git a/pkg/provision/migrations_202109_settings_cleanup.go b/pkg/provision/migrations_202109_settings_cleanup.go deleted file mode 100644 index 18d053356..000000000 --- a/pkg/provision/migrations_202109_settings_cleanup.go +++ /dev/null @@ -1,35 +0,0 @@ -package provision - -import ( - "context" - - "github.com/cortezaproject/corteza-server/store" - "go.uber.org/zap" -) - -func cleanupPre202109Settings(ctx context.Context, log *zap.Logger, s store.Storer) (err error) { - log.Info("cleaning up pre-2021.9 settings") - names := []string{ - "compose.ui.namespace-switcher.enabled", - "compose.ui.namespace-switcher.default-open", - "messaging.message.attachments.enabled", - "messaging.message.attachments.max-size", - "messaging.message.attachments.mimetypes", - "messaging.message.attachments.source.camera.enabled", - "messaging.message.attachments.source.gallery.enabled", - "messaging.ui.browser-notifications.enabled", - "messaging.ui.browser-notifications.header", - "messaging.ui.browser-notifications.message-trim", - "messaging.ui.emoji.enabled", - "auth.external.session-store-secret", - "auth.external.session-store-secure", - } - - for _, name := range names { - if err = store.DeleteSettingValueByNameOwnedBy(ctx, s, name, 0); err != nil { - return - } - } - - return -} diff --git a/pkg/provision/migrations_202203_rbac_resource_fix.go b/pkg/provision/migrations_202203_rbac_resource_fix.go deleted file mode 100644 index f62bc2f34..000000000 --- a/pkg/provision/migrations_202203_rbac_resource_fix.go +++ /dev/null @@ -1,361 +0,0 @@ -package provision - -import ( - "context" - "fmt" - "strconv" - "strings" - - composeTypes "github.com/cortezaproject/corteza-server/compose/types" - federationTypes "github.com/cortezaproject/corteza-server/federation/types" - "github.com/cortezaproject/corteza-server/pkg/filter" - "github.com/cortezaproject/corteza-server/pkg/rbac" - "github.com/cortezaproject/corteza-server/store" - "go.uber.org/zap" -) - -type ( - resIndex struct { - fields map[uint64]*composeTypes.ModuleField - modules map[uint64]*composeTypes.Module - records map[uint64]*composeTypes.Record - charts map[uint64]*composeTypes.Chart - pages map[uint64]*composeTypes.Page - exposedModules map[uint64]*federationTypes.ExposedModule - sharedModules map[uint64]*federationTypes.SharedModule - } -) - -// MigrateOperations -func migratePost202203RbacRules(ctx context.Context, log *zap.Logger, s store.Storer) error { - return store.Tx(ctx, s, func(ctx context.Context, s store.Storer) error { - rr, _, err := store.SearchRbacRules(ctx, s, rbac.RuleFilter{}) - if err != nil { - return err - } - - log.Info("migrating RBAC resource rules to proper format", zap.Int("rules", len(rr))) - - rx, err := preloadRbacResourceIndex(ctx, s) - if err != nil { - return err - } - - var uniq = make(map[string]bool) - var uniqID = func(r *rbac.Rule) string { - return fmt.Sprintf("%s|%s|%d", r.Resource, r.Operation, r.RoleID) - } - - for _, r := range rr { - var ( - cr = *r - action = migratePost202203RbacRule(r, rx) - ) - - if action != 0 { - err = store.DeleteRbacRule(ctx, s, &cr) - if err != nil { - return fmt.Errorf("could not delete RBAC rule %s: %v", r, err) - } - - if action == -1 { - log.Debug("removed obsolete RBAC rule", zap.Stringer("rule", r)) - } - } - - if action == 1 { - if uniq[uniqID(r)] { - log.Warn("skipping duplicate RBAC rule", zap.Stringer("rule", r)) - continue - } - - err = store.CreateRbacRule(ctx, s, r) - if err != nil { - return fmt.Errorf("could not create RBAC rule %s: %v", r, err) - } - - uniq[uniqID(r)] = true - } - } - - return nil - }) -} - -// 0 - no action -// -1 - remove -// 1 - update -func migratePost202203RbacRule(r *rbac.Rule, rx *resIndex) (op int) { - const ( - wildCard = "*" - ) - - if r.Resource == "" { - return -1 - } - - // split the IDs - parts := strings.SplitN(r.Resource, "/", 4) - parts = parts[1:] - - isWildCard := func(s string) bool { return strings.Contains(s, wildCard) } - parseUint := func(s string) uint64 { - ID, _ := strconv.ParseUint(s, 10, 64) - return ID - } - validID := func(i uint64) bool { return i > 0 } - - invalid := false - for i, p := range parts { - if i == 0 || len(p) == 0 { - continue - } - - if !invalid { - invalid = isWildCard(parts[i-1]) && !isWildCard(p) - } - } - - if invalid { - op = 1 - - partsLen := len(parts) - ID := uint64(0) - p1 := uint64(0) - p2 := uint64(0) - - if partsLen == 1 { - ID = parseUint(parts[0]) - } else if partsLen == 2 { - p1 = parseUint(parts[0]) - ID = parseUint(parts[1]) - } else if partsLen == 3 { - p1 = parseUint(parts[0]) - p2 = parseUint(parts[1]) - ID = parseUint(parts[2]) - } - - rType := rbac.ResourceType(r.Resource) - switch rType { - case composeTypes.ModuleFieldResourceType: - if ID > 0 { - if f, ok := rx.fields[ID]; ok { - p2 = f.ModuleID - if m, ok := rx.modules[p2]; ok { - p1 = m.NamespaceID - if !validID(p1) { - return -1 - } - } else { - return -1 - } - } else { - return -1 - } - } - - r.Resource = composeTypes.ModuleFieldRbacResource(p1, p2, ID) - - case composeTypes.ModuleResourceType: - if ID > 0 { - if r, ok := rx.modules[ID]; ok { - p1 = r.NamespaceID - if !validID(p1) { - return -1 - } - } else { - return -1 - } - } - r.Resource = composeTypes.ModuleRbacResource(p1, ID) - - case composeTypes.RecordResourceType: - if ID > 0 { - if r, ok := rx.records[ID]; ok { - p2 = r.ModuleID - if m, ok := rx.modules[p2]; ok { - p1 = m.NamespaceID - if !validID(p1) { - return -1 - } - } else { - return -1 - } - } else { - return -1 - } - } - r.Resource = composeTypes.RecordRbacResource(p1, p2, ID) - - case composeTypes.ChartResourceType: - if ID > 0 { - if r, ok := rx.charts[ID]; ok { - p1 = r.NamespaceID - if !validID(p1) { - return -1 - } - } else { - return -1 - } - } - r.Resource = composeTypes.ChartRbacResource(p1, ID) - - case composeTypes.PageResourceType: - if ID > 0 { - if r, ok := rx.pages[ID]; ok { - p1 = r.NamespaceID - if !validID(p1) { - return -1 - } - } else { - return -1 - } - } - r.Resource = composeTypes.PageRbacResource(p1, ID) - - case federationTypes.ExposedModuleResourceType: - if ID > 0 { - if r, ok := rx.exposedModules[ID]; ok { - p1 = r.NodeID - if !validID(p1) { - return -1 - } - } else { - return -1 - } - } - r.Resource = federationTypes.ExposedModuleRbacResource(p1, ID) - - case federationTypes.SharedModuleResourceType: - if ID > 0 { - if r, ok := rx.sharedModules[ID]; ok { - p1 = r.NodeID - if !validID(p1) { - return -1 - } - } else { - return -1 - } - } - r.Resource = federationTypes.SharedModuleRbacResource(p1, ID) - - default: - r.Resource = rType + "/" + func() string { - if ID == 0 { - return "*" - } - return strconv.FormatUint(ID, 10) - }() - } - } - return -} - -// helper to preload resources that may be used when properly constructing rules -func preloadRbacResourceIndex(ctx context.Context, s store.Storer) (*resIndex, error) { - rx := &resIndex{} - - rx.records = make(map[uint64]*composeTypes.Record) - var getRecords func(*composeTypes.Module, *filter.PagingCursor) (composeTypes.RecordSet, composeTypes.RecordFilter, error) - getRecords = func(m *composeTypes.Module, cursor *filter.PagingCursor) (rr composeTypes.RecordSet, f composeTypes.RecordFilter, err error) { - rr, f, err = store.SearchComposeRecords(ctx, s, m, composeTypes.RecordFilter{ - Paging: filter.Paging{ - Limit: 10000, - PageCursor: cursor, - }, - Deleted: filter.StateInclusive, - }) - - if err != nil { - return - } - for _, rec := range rr { - rx.records[rec.ID] = rec - } - - if f.NextPage != nil { - _, _, err = getRecords(m, f.NextPage) - if err != nil { - return - } - } - - return - } - - rx.modules = make(map[uint64]*composeTypes.Module) - modules, _, err := store.SearchComposeModules(ctx, s, composeTypes.ModuleFilter{ - Paging: filter.Paging{Limit: 0}, - Deleted: filter.StateInclusive, - }) - modIDs := make([]uint64, 0, len(modules)) - if err != nil { - return nil, err - } - for _, r := range modules { - rx.modules[r.ID] = r - modIDs = append(modIDs, r.ID) - _, _, err = getRecords(r, nil) - if err != nil { - return nil, err - } - } - - if len(modIDs) > 0 { - rx.fields = make(map[uint64]*composeTypes.ModuleField) - fields, _, err := store.SearchComposeModuleFields(ctx, s, composeTypes.ModuleFieldFilter{ - ModuleID: modIDs, - Deleted: filter.StateInclusive, - }) - if err != nil { - return nil, err - } - for _, r := range fields { - rx.fields[r.ID] = r - } - } - - rx.charts = make(map[uint64]*composeTypes.Chart) - chart, _, err := store.SearchComposeCharts(ctx, s, composeTypes.ChartFilter{ - Paging: filter.Paging{Limit: 0}, - Deleted: filter.StateInclusive, - }) - if err != nil { - return nil, err - } - for _, r := range chart { - rx.charts[r.ID] = r - } - - rx.pages = make(map[uint64]*composeTypes.Page) - page, _, err := store.SearchComposePages(ctx, s, composeTypes.PageFilter{ - Paging: filter.Paging{Limit: 0}, - Deleted: filter.StateInclusive, - }) - if err != nil { - return nil, err - } - for _, r := range page { - rx.pages[r.ID] = r - } - - rx.exposedModules = make(map[uint64]*federationTypes.ExposedModule) - exposedModule, _, err := store.SearchFederationExposedModules(ctx, s, federationTypes.ExposedModuleFilter{Paging: filter.Paging{Limit: 0}}) - if err != nil { - return nil, err - } - for _, r := range exposedModule { - rx.exposedModules[r.ID] = r - } - - rx.sharedModules = make(map[uint64]*federationTypes.SharedModule) - sharedModule, _, err := store.SearchFederationSharedModules(ctx, s, federationTypes.SharedModuleFilter{Paging: filter.Paging{Limit: 0}}) - if err != nil { - return nil, err - } - for _, r := range sharedModule { - rx.sharedModules[r.ID] = r - } - - return rx, nil -} diff --git a/pkg/provision/migrations_202203_rbac_resource_fix_test.go b/pkg/provision/migrations_202203_rbac_resource_fix_test.go deleted file mode 100644 index 9a51af9c9..000000000 --- a/pkg/provision/migrations_202203_rbac_resource_fix_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package provision - -import ( - "github.com/stretchr/testify/require" - "testing" - - composeTypes "github.com/cortezaproject/corteza-server/compose/types" - federationTypes "github.com/cortezaproject/corteza-server/federation/types" - "github.com/cortezaproject/corteza-server/pkg/rbac" -) - -func Test_migratePost202203RbacRules(t *testing.T) { - rx := &resIndex{ - fields: make(map[uint64]*composeTypes.ModuleField), - modules: make(map[uint64]*composeTypes.Module), - charts: make(map[uint64]*composeTypes.Chart), - pages: make(map[uint64]*composeTypes.Page), - exposedModules: make(map[uint64]*federationTypes.ExposedModule), - sharedModules: make(map[uint64]*federationTypes.SharedModule), - } - - tcc := []struct { - wantOp int - rule *rbac.Rule - wantRule *rbac.Rule - }{ - {-1, rbac.AllowRule(0, "corteza::compose:module/*/123", "op"), rbac.AllowRule(0, "corteza::compose:module/*/123", "op")}, - {-1, rbac.AllowRule(0, "corteza::compose:module-field/*/*/123", "op"), rbac.AllowRule(0, "corteza::compose:module-field/*/*/123", "op")}, - {-1, rbac.AllowRule(0, "corteza::compose:record/*/*/234", "op"), rbac.AllowRule(0, "corteza::compose:record/*/*/234", "op")}, - } - for _, tc := range tcc { - t.Run(tc.rule.String(), func(t *testing.T) { - require.Equal(t, tc.wantOp, migratePost202203RbacRule(tc.rule, rx)) - if tc.wantRule != nil { - require.Equal(t, tc.wantRule.String(), tc.rule.String()) - } - }) - } -} diff --git a/pkg/provision/migrations_202203_report_identifiers.go b/pkg/provision/migrations_202203_report_identifiers.go deleted file mode 100644 index 32d6a99bd..000000000 --- a/pkg/provision/migrations_202203_report_identifiers.go +++ /dev/null @@ -1,68 +0,0 @@ -package provision - -import ( - "context" - "strconv" - - "github.com/cortezaproject/corteza-server/pkg/filter" - "github.com/cortezaproject/corteza-server/pkg/id" - "github.com/cortezaproject/corteza-server/store" - sysTypes "github.com/cortezaproject/corteza-server/system/types" - "github.com/spf13/cast" - "go.uber.org/zap" -) - -// Migrates existing reports to include the newly added resource identifiers -func migrateReportIdentifiers(ctx context.Context, log *zap.Logger, s store.Storer) (err error) { - log.Info("migrating report identifiers") - - set, _, err := store.SearchReports(ctx, s, sysTypes.ReportFilter{Deleted: filter.StateInclusive}) - if err != nil { - return err - } - - return s.Tx(ctx, func(ctx context.Context, s store.Storer) error { - return set.Walk(func(r *sysTypes.Report) (err error) { - r = setIDs(r) - return store.UpdateReport(ctx, s, r) - }) - }) -} - -// duplicate from the report service -func setIDs(r *sysTypes.Report) *sysTypes.Report { - // scenarios - for _, s := range r.Scenarios { - if s.ScenarioID == 0 { - s.ScenarioID = id.Next() - } - } - - // blocks - for _, b := range r.Blocks { - if b.BlockID == 0 { - b.BlockID = id.Next() - } - - // elements - for _, elRaw := range b.Elements { - el, ok := elRaw.(map[string]interface{}) - if !ok { - continue - } - - elID, ok := el["elementID"] - sElID := cast.ToString(elID) - if sElID != "" && sElID != "0" { - continue - } - if cast.ToUint64(elID) != 0 { - continue - } - - el["elementID"] = strconv.FormatUint(id.Next(), 10) - } - } - - return r -} diff --git a/pkg/provision/migrations_202203_resource_translations.go b/pkg/provision/migrations_202203_resource_translations.go deleted file mode 100644 index fb61acc2b..000000000 --- a/pkg/provision/migrations_202203_resource_translations.go +++ /dev/null @@ -1,116 +0,0 @@ -package provision - -import ( - "context" - "fmt" - cmpTypes "github.com/cortezaproject/corteza-server/compose/types" - "github.com/cortezaproject/corteza-server/pkg/filter" - "github.com/cortezaproject/corteza-server/pkg/id" - "github.com/cortezaproject/corteza-server/store" - sysTypes "github.com/cortezaproject/corteza-server/system/types" - "go.uber.org/zap" - "strings" -) - -// Migrates resource translations from the resource -// struct to the dedicated store (table) -// -// While doing this, we also modify some resource substructure: -// - chart reports (assign report IDs) -// -// Note: we will migrate all translations to current default language -// If you do not like that, shut down Corteza after migrations and fix this directly in the store -func migratePost202203ResourceTranslations(ctx context.Context, log *zap.Logger, s store.Storer) (err error) { - log.Info("migrating post 202203 resource translations") - - var ( - migrated = make(map[string]bool) - ) - - set, _, err := store.SearchResourceTranslations(ctx, s, sysTypes.ResourceTranslationFilter{}) - if err != nil { - return - } - err = set.Walk(func(r *sysTypes.ResourceTranslation) error { - var pos = strings.Index(r.Resource, "/") - if pos < 0 { - return nil - } - - migrated[r.Resource[0:pos]] = true - return nil - }) - if err != nil { - return - } - - if !migrated[cmpTypes.ChartResourceTranslationType] { - if err = migrateComposeChartResourceTranslations(ctx, log, s); err != nil { - return - } - } - - return -} - -// migrate resource translations for compose chart -func migrateComposeChartResourceTranslations(ctx context.Context, log *zap.Logger, s store.Storer) error { - var ( - tt sysTypes.ResourceTranslationSet - ) - set, _, err := store.SearchComposeCharts(ctx, s, cmpTypes.ChartFilter{Deleted: filter.StateInclusive}) - if err != nil { - return err - } - - log.Info("migrating compose charts", zap.Int("count", len(set))) - - return s.Tx(ctx, func(ctx context.Context, s store.Storer) error { - return set.Walk(func(res *cmpTypes.Chart) (err error) { - if tt, err = convertComposeChartReportsTranslations(res); err != nil { - return err - } - - if err = store.CreateResourceTranslation(ctx, s, tt...); err != nil { - return - } - - if err = store.UpdateComposeChart(ctx, s, res); err != nil { - return - } - - return - }) - }) -} - -// collects translations for compose chart blocks and alters them (adding block ID) -func convertComposeChartReportsTranslations(res *cmpTypes.Chart) (sysTypes.ResourceTranslationSet, error) { - var tt sysTypes.ResourceTranslationSet - - for i, r := range res.Config.Reports { - reportID := id.Next() - res.Config.Reports[i].ReportID = reportID - - if _, ok := r.YAxis["label"]; ok { - tt = append(tt, - makeResourceTranslation(res, "yAxis.label", r.YAxis["label"].(string)), - ) - } - - // Ensure chart report metric IDs - for j, m := range r.Metrics { - metricID := id.Next() - res.Config.Reports[i].Metrics[j]["metricID"] = fmt.Sprintf("%d", metricID) - cmx := fmt.Sprintf("metrics.%d.", metricID) - - if _, ok := m["label"]; ok { - tt = append(tt, - makeResourceTranslation(res, cmx+"label", m["label"].(string)), - ) - } - } - } - - return tt, nil -} diff --git a/pkg/provision/provision.go b/pkg/provision/provision.go index c0afeefdb..9f7021557 100644 --- a/pkg/provision/provision.go +++ b/pkg/provision/provision.go @@ -24,25 +24,18 @@ var ( func Run(ctx context.Context, log *zap.Logger, s store.Storer, provisionOpt options.ProvisionOpt, authOpt options.AuthOpt) error { log = log.Named("provision") + // Note, ffn := []func() error{ // Migrations: - func() error { return migrateApplications(ctx, s) }, - func() error { return migrateEmailTemplates(ctx, log.Named("email-templates"), s) }, - func() error { return migratePre202109Roles(ctx, log.Named("pre-202109-roles"), s) }, - func() error { return migratePre202109RbacRules(ctx, log.Named("pre-202109-rbac-rules"), s) }, - func() error { return cleanupPre202109Settings(ctx, log.Named("pre-202109-settings"), s) }, - func() error { return migrateResourceTranslations(ctx, log.Named("resource-translations"), s) }, - func() error { return migrateReportIdentifiers(ctx, log.Named("report-identifiers"), s) }, - func() error { - return migratePost202203RbacRules(ctx, log.Named("post-202203-rbac-resource-rules-fix"), s) - }, - func() error { - return migratePost202203ResourceTranslations(ctx, log.Named("post-202203-resource-translations"), s) - }, + // (placeholder for all post 2022.3.x modifications) + + // ************************************************************************************************************* // Config (full & partial) func() error { return importConfig(ctx, log.Named("config"), s, provisionOpt.Path) }, + // ************************************************************************************************************* + // Auto-discoveries and other parts that cannot be imported from static files func() error { return emailSettings(ctx, s) }, func() error { return authAddExternals(ctx, log.Named("auth.externals"), s) },