Add integration tests for namespace import/export

compose/service/namespace.go

@@ -375,7 +375,9 @@ func (svc namespace) Export(ctx context.Context, namespaceID uint64, archive str
 		// initial validation
 		// - target namespace
 		targetNs, err := store.LookupComposeNamespaceByID(ctx, svc.store, namespaceID)
-		if err != nil && err != store.ErrNotFound {
+		if errors.IsNotFound(err) {
+			return NamespaceErrNotFound()
+		} else if err != nil {
 			return err
 		}
 		aProps.setNamespace(targetNs)
@@ -489,6 +491,10 @@ func (svc namespace) ImportInit(ctx context.Context, f multipart.File, size int6
 		nn := make([]resource.Interface, 0, 10)
 
 		for _, f := range archive.File {
+			if f.FileInfo().IsDir() {
+				continue
+			}
+
 			a, err := f.Open()
 			if err != nil {
 				return err

pkg/envoy/resource/compose_page.go

@@ -154,23 +154,31 @@ outer:
 		switch b.Kind {
 		// Implement the rest when support is needed
 		case "Automation":
-			bb, _ := b.Options["buttons"].([]interface{})
-			for _, b := range bb {
-				button, _ := b.(map[string]interface{})
-				auxRef = r.pbAutomation(button)
-
+			if b.Options["buttons"] == nil {
 				// In case the block isn't connected to a workflow (placeholder, script)
 				if auxRef == nil {
 					r.removeBlock(i)
 					continue outer
 				}
+			} else {
+				bb, _ := b.Options["buttons"].([]interface{})
+				for _, b := range bb {
+					button, _ := b.(map[string]interface{})
+					auxRef = r.pbAutomation(button)
 
-				// In case we are removing it
-				if auxRef.equals(ref) {
-					r.ReplaceRef(ref, nil)
-					r.WfRefs = r.WfRefs.replaceRef(ref, nil)
-					r.removeBlock(i)
-					continue outer
+					// In case the block isn't connected to a workflow (placeholder, script)
+					if auxRef == nil {
+						r.removeBlock(i)
+						continue outer
+					}
+
+					// In case we are removing it
+					if auxRef.equals(ref) {
+						r.ReplaceRef(ref, nil)
+						r.WfRefs = r.WfRefs.replaceRef(ref, nil)
+						r.removeBlock(i)
+						continue outer
+					}
 				}
 			}
 		}

tests/compose/main_test.go

@@ -1,15 +1,23 @@
 package compose
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
+	"fmt"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
 	"os"
 	"path"
+	"strings"
 	"testing"
 
 	"github.com/cortezaproject/corteza-server/app"
 	"github.com/cortezaproject/corteza-server/compose/rest"
 	"github.com/cortezaproject/corteza-server/compose/service"
+	"github.com/cortezaproject/corteza-server/compose/types"
 	"github.com/cortezaproject/corteza-server/pkg/api/server"
 	"github.com/cortezaproject/corteza-server/pkg/auth"
 	"github.com/cortezaproject/corteza-server/pkg/cli"
@@ -192,8 +200,12 @@ func cleanup(t *testing.T) {
 	}
 }
 
+func scenario(t *testing.T) string {
+	return t.Name()[5:]
+}
+
 func loadScenario(ctx context.Context, s store.Storer, t *testing.T, h helper) {
-	loadScenarioWithName(ctx, s, t, h, t.Name()[5:])
+	loadScenarioWithName(ctx, s, t, h, scenario(t))
 }
 
 func loadScenarioWithName(ctx context.Context, s store.Storer, t *testing.T, h helper, scenario string) {
@@ -248,3 +260,132 @@ func setup(t *testing.T) (context.Context, helper, store.Storer) {
 
 	return ctx, h, s
 }
+
+func grantImportExport(h helper) {
+	helpers.AllowMe(h, types.ComponentRbacResource(), "namespace.create")
+	helpers.AllowMe(h, types.ComponentRbacResource(), "module.create")
+	helpers.AllowMe(h, types.ComponentRbacResource(), "page.create")
+	helpers.AllowMe(h, types.ComponentRbacResource(), "chart.create")
+	helpers.AllowMe(h, types.NamespaceRbacResource(0), "read")
+	helpers.AllowMe(h, types.ModuleRbacResource(0, 0), "read")
+	helpers.AllowMe(h, types.PageRbacResource(0, 0), "read")
+	helpers.AllowMe(h, types.ChartRbacResource(0, 0), "read")
+}
+
+func namespaceExportSafe(t *testing.T, h helper, namespaceID uint64) []byte {
+	bb, err := namespaceExport(t, h, namespaceID)
+	h.a.NoError(err)
+
+	return bb
+}
+
+func namespaceExport(t *testing.T, h helper, namespaceID uint64) ([]byte, error) {
+	out := h.apiInit().
+		Get(fmt.Sprintf("/namespace/%d/export/out.zip?jwt=%s", namespaceID, h.token)).
+		Expect(t).
+		Status(http.StatusOK).
+		End()
+
+	defer out.Response.Body.Close()
+
+	bb, err := ioutil.ReadAll(out.Response.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if strings.HasPrefix(string(bb), "Error:") {
+		return nil, errors.New(string(bb))
+	}
+
+	return bb, nil
+}
+
+func namespaceImportInitPathSafe(t *testing.T, h helper, pp ...string) uint64 {
+	sessionID, err := namespaceImportInitPath(t, h, pp...)
+	h.a.NoError(err)
+
+	return sessionID
+}
+
+func namespaceImportInitPath(t *testing.T, h helper, pp ...string) (uint64, error) {
+	f, err := os.Open(testSource(t, pp...))
+	if err != nil {
+		return 0, err
+	}
+
+	defer f.Close()
+	bb, err := ioutil.ReadAll(f)
+	if err != nil {
+		return 0, err
+	}
+
+	return namespaceImportInit(t, h, bb)
+}
+
+func namespaceImportInitSafe(t *testing.T, h helper, arch []byte) uint64 {
+	sessionID, err := namespaceImportInit(t, h, arch)
+	h.a.NoError(err)
+
+	return sessionID
+}
+
+func namespaceImportInit(t *testing.T, h helper, arch []byte) (uint64, error) {
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("upload", "archive.zip")
+	h.noError(err)
+
+	_, err = part.Write(arch)
+	h.noError(err)
+	h.noError(writer.Close())
+
+	out := h.apiInit().
+		Post("/namespace/import").
+		Header("Accept", "application/json").
+		Body(body.String()).
+		ContentType(writer.FormDataContentType()).
+		Expect(h.t).
+		Status(http.StatusOK).
+		End()
+
+	defer out.Response.Body.Close()
+	bb, err := ioutil.ReadAll(out.Response.Body)
+	h.a.NoError(err)
+
+	var aux struct {
+		Error struct {
+			Message string
+		}
+		Response struct {
+			ID uint64 `json:"sessionID,string"`
+		}
+	}
+	h.a.NoError(json.Unmarshal(bb, &aux))
+
+	if aux.Error.Message != "" {
+		return 0, errors.New(aux.Error.Message)
+	}
+
+	return aux.Response.ID, nil
+}
+
+func namespaceImportRun(ctx context.Context, s store.Storer, t *testing.T, h helper, sessionID uint64, name, slug string) (*types.Namespace, types.ModuleSet, types.PageSet, types.ChartSet) {
+	h.apiInit().
+		Post(fmt.Sprintf("/namespace/import/%d", sessionID)).
+		Header("Accept", "application/json").
+		FormData("name", name).
+		FormData("slug", slug).
+		Expect(h.t).
+		Status(http.StatusOK).
+		Assert(helpers.AssertNoErrors).
+		End()
+
+	ns, mm, pp, cc, _, err := fetchEntireNamespace(ctx, s, slug)
+	h.a.NoError(err)
+
+	return ns, mm, pp, cc
+}
+
+func testSource(t *testing.T, pp ...string) string {
+	return path.Join(append([]string{"testdata", scenario(t)}, pp...)...)
+}

tests/compose/namespace_export_automation_test.go

@@ -0,0 +1,29 @@
+package compose
+
+import (
+	"testing"
+)
+
+func Test_namespace_export_automation(t *testing.T) {
+	ctx, h, s := setup(t)
+	loadScenario(ctx, defStore, t, h)
+	grantImportExport(h)
+
+	ns, _, _, _, _, err := fetchEntireNamespace(ctx, s, "ns1")
+	h.a.NoError(err)
+
+	arch := namespaceExportSafe(t, h, ns.ID)
+	sessionID := namespaceImportInitSafe(t, h, arch)
+	ns, _, pp, _ := namespaceImportRun(ctx, s, t, h, sessionID, "imported", "imported")
+
+	h.a.Equal("imported", ns.Slug)
+	h.a.NotEqual(0, ns.ID)
+
+	p := pp.FindByHandle("pg1")
+	h.a.Len(p.Blocks, 2)
+	for _, b := range p.Blocks {
+		h.a.NotEqual("Automation", b.Kind)
+	}
+
+	cleanup(t)
+}

tests/compose/namespace_export_empty_test.go

@@ -0,0 +1,27 @@
+package compose
+
+import (
+	"testing"
+)
+
+func Test_namespace_export_empty(t *testing.T) {
+	ctx, h, s := setup(t)
+	loadScenario(ctx, defStore, t, h)
+	grantImportExport(h)
+
+	ns, _, _, _, _, err := fetchEntireNamespace(ctx, s, "ns1")
+	h.a.NoError(err)
+
+	arch := namespaceExportSafe(t, h, ns.ID)
+	sessionID := namespaceImportInitSafe(t, h, arch)
+	ns, mm, pp, cc := namespaceImportRun(ctx, s, t, h, sessionID, "imported", "imported")
+
+	h.a.Equal("imported", ns.Slug)
+	h.a.NotEqual(0, ns.ID)
+
+	h.a.Len(mm, 0)
+	h.a.Len(pp, 0)
+	h.a.Len(cc, 0)
+
+	cleanup(t)
+}

tests/compose/namespace_export_missing_res_test.go

@@ -0,0 +1,24 @@
+package compose
+
+import (
+	"testing"
+
+	"github.com/cortezaproject/corteza-server/store"
+)
+
+func Test_namespace_export_missing_res(t *testing.T) {
+	ctx, h, s := setup(t)
+	loadScenario(ctx, defStore, t, h)
+	grantImportExport(h)
+
+	ns, mm, _, _, _, err := fetchEntireNamespace(ctx, s, "ns1")
+	h.a.NoError(err)
+
+	// Removeing one of the resources
+	h.a.NoError(store.DeleteComposeModuleByID(ctx, s, mm.FindByHandle("mod2").ID))
+
+	_, err = namespaceExport(t, h, ns.ID)
+	h.a.Error(err)
+
+	cleanup(t)
+}

tests/compose/namespace_export_simple_test.go

@@ -0,0 +1,53 @@
+package compose
+
+import (
+	"testing"
+)
+
+func Test_namespace_export_simple(t *testing.T) {
+	ctx, h, s := setup(t)
+	loadScenario(ctx, defStore, t, h)
+	grantImportExport(h)
+
+	ns, _, _, _, _, err := fetchEntireNamespace(ctx, s, "ns1")
+	h.a.NoError(err)
+
+	arch := namespaceExportSafe(t, h, ns.ID)
+	sessionID := namespaceImportInitSafe(t, h, arch)
+	ns, mm, pp, cc := namespaceImportRun(ctx, s, t, h, sessionID, "imported", "imported")
+
+	h.a.Equal("imported", ns.Slug)
+	h.a.NotEqual(0, ns.ID)
+
+	h.a.Len(mm, 3)
+	exp := map[string]bool{"mod1": true, "mod2": true, "mod3": true}
+
+	for i, m := range mm {
+		h.a.True(exp[m.Handle])
+		if i > 0 {
+			h.a.NotEqual(m.Handle, mm[i-1].Handle)
+		}
+	}
+
+	h.a.Len(pp, 3)
+	exp = map[string]bool{
+		"pg1":  true,
+		"rpg2": true,
+		"pg2":  true,
+	}
+	for i, p := range pp {
+		h.a.True(exp[p.Handle])
+		if i > 0 {
+			h.a.NotEqual(p.Handle, mm[i-1].Handle)
+		}
+	}
+
+	parent := pp.FindByHandle("rpg2")
+	child := pp.FindByHandle("pg2")
+	h.a.Equal(child.SelfID, parent.ID)
+
+	h.a.Len(cc, 1)
+	h.a.Equal("chr1", cc[0].Handle)
+
+	cleanup(t)
+}

tests/compose/namespace_export_unexisting_test.go

@@ -0,0 +1,18 @@
+package compose
+
+import (
+	"testing"
+)
+
+func Test_namespace_export_unexisting(t *testing.T) {
+	ctx, h, s := setup(t)
+	grantImportExport(h)
+	_ = ctx
+	_ = s
+
+	_, err := namespaceExport(t, h, 42)
+	h.a.Error(err)
+	h.a.Contains(err.Error(), "namespace does not exist")
+
+	cleanup(t)
+}

tests/compose/namespace_import_test.go

@@ -0,0 +1,24 @@
+package compose
+
+import (
+	"testing"
+)
+
+func Test_namespace_import(t *testing.T) {
+	t.Run("nested", func(_ *testing.T) {
+		_, h, _ := setup(t)
+		grantImportExport(h)
+
+		sessionID := namespaceImportInitPathSafe(t, h, "nested.zip")
+		h.a.NotEqual(0, sessionID)
+	})
+
+	t.Run("no namespace", func(_ *testing.T) {
+		_, h, _ := setup(t)
+		grantImportExport(h)
+
+		_, err := namespaceImportInitPath(t, h, "no-ns.zip")
+		h.a.Error(err)
+		h.a.Contains(err.Error(), "namespace.errors.importMissingNamespace")
+	})
+}

tests/compose/testdata/namespace_export_automation/data_model/1000_namespace.yaml

@@ -0,0 +1,3 @@
+namespaces:
+  ns1:
+    name: ns1 name

tests/compose/testdata/namespace_export_automation/data_model/1100_modules.yaml

@@ -0,0 +1,17 @@
+namespace: ns1
+modules:
+  mod1:
+    name: mod1 name
+    fields:
+      f1:
+        label: f1 label
+        kind: String
+        required: true
+      f2:
+        label: f2 label
+        kind: Select
+        options:
+          options:
+          - f2 opt 1
+          - f2 opt 2
+          - f2 opt 3

tests/compose/testdata/namespace_export_automation/data_model/1200_pages.yaml

@@ -0,0 +1,60 @@
+namespace: ns1
+pages:
+  pg1:
+    title: pg1 title
+    blocks:
+    - title: pg1 b1
+      kind: Content
+      xywh: [0, 0, 1, 1]
+      options:
+        body: pg1 b1 content body
+
+    - title: pg1 b2
+      kind: RecordList
+      xywh: [0, 1, 1, 1]
+      options:
+        module: mod1
+        fields:
+        - name: f1
+        - name: f2
+
+    - title: pg1 b3 automation 1
+      kind: Automation
+      xywh: [0, 0, 0, 0]
+      options:
+        buttons:
+        - label: workflow button
+          resourceType: compose:record
+          workflow: wf1
+          stepID: 4
+
+    - title: pg1 b3 automation 2
+      kind: Automation
+      xywh: [0, 0, 0, 0]
+      options:
+        buttons:
+        - label: script button
+          script: /client-scripts/compose/test.js:default
+
+    - title: pg1 b3 automation 3
+      kind: Automation
+      xywh: [0, 0, 0, 0]
+      options:
+        buttons:
+        - label: placeholder button
+
+    - title: pg1 b3 automation 4
+      kind: Automation
+      xywh: [0, 0, 0, 0]
+
+  rpg2:
+    handle: rpg2
+    module: mod1
+    title: Record page for module "mod1"
+    blocks:
+    - title: rpg2 b1 title
+      kind: Record
+      options:
+        fields:
+        - name: f1
+        - name: f2

tests/compose/testdata/namespace_export_automation/data_model/1300_workflows.yaml

@@ -0,0 +1,36 @@
+workflows:
+  wf1:
+    meta:
+      name: wf1
+    enabled: true
+    keepSessions: 0
+    triggers:
+      - resourceType: compose:record
+        eventType: beforeCreate
+        constraints:
+          - name: namespace.handle
+            op: =
+            values:
+              - ns1
+        enabled: true
+        stepID: 4
+    steps:
+      - id: 4
+        kind: prompt
+        ref: alert
+        arguments:
+          - target: message
+            source: ""
+            expr: ""
+            value: Test
+            type: String
+            tests: []
+        results: []
+        meta:
+          name: ""
+          description: ""
+          visual:
+            id: "4"
+            parent: "1"
+            value: null
+            xywh: [0, 0, 0, 0]

tests/compose/testdata/namespace_export_empty/data_model/1000_namespace.yaml

@@ -0,0 +1,3 @@
+namespaces:
+  ns1:
+    name: ns1 name

tests/compose/testdata/namespace_export_missing_res/data_model/1000_namespace.yaml

@@ -0,0 +1,3 @@
+namespaces:
+  ns1:
+    name: ns1 name

tests/compose/testdata/namespace_export_missing_res/data_model/1100_modules.yaml

@@ -0,0 +1,42 @@
+namespace: ns1
+modules:
+  mod1:
+    name: mod1 name
+    fields:
+      f1:
+        label: f1 label
+        kind: String
+        required: true
+      f2:
+        label: f2 label
+        kind: Select
+        options:
+          options:
+          - f2 opt 1
+          - f2 opt 2
+          - f2 opt 3
+      f3:
+        label: f3 label
+        kind: Record
+        options:
+          labelField: f_label
+          module: mod2
+          queryFields:
+          - f1
+
+  mod2:
+    name: mod2 name
+    fields:
+      f_label:
+        label: f_label record label
+      f1:
+        label: f1 label
+        kind: String
+        required: true
+
+  mod3:
+    name: mod3 name
+    fields:
+      f1:
+        label: f1 label
+        kind: String

tests/compose/testdata/namespace_export_missing_res/data_model/1200_pages.yaml

@@ -0,0 +1,47 @@
+namespace: ns1
+pages:
+  pg1:
+    title: pg1 title
+    blocks:
+    - title: pg1 b1
+      kind: Content
+      xywh: [0, 0, 1, 1]
+      options:
+        body: pg1 b1 content body
+
+    - title: pg1 b2
+      kind: RecordList
+      xywh: [0, 1, 1, 1]
+      options:
+        module: mod1
+        fields:
+        - name: f1
+        - name: f2
+
+    - title: pg1 b3
+      kind: Chart
+      xywh: [0, 2, 1, 1]
+      options:
+        chart: chr1
+
+  rpg2:
+    handle: rpg2
+    module: mod1
+    title: Record page for module "mod1"
+    blocks:
+    - title: rpg2 b1 title
+      kind: Record
+      options:
+        fields:
+        - name: f1
+        - name: f2
+
+    pages:
+      pg2:
+        title: pg2 title
+        blocks:
+        - title: pg2 b1
+          kind: Content
+          xywh: [0, 0, 1, 1]
+          options:
+            body: pg2 b1 content body

tests/compose/testdata/namespace_export_missing_res/data_model/1300_charts.yaml

@@ -0,0 +1,18 @@
+namespace: ns1
+charts:
+  chr1:
+    name: chr1 name
+    config:
+      reports:
+      - dimensions:
+        - field: Status
+          modifier: (no grouping / buckets)
+        filter: ""
+        metrics:
+        - backgroundColor: '#11ff57'
+          field: count
+          fixTooltips: true
+          type: pie
+        module: mod1
+        renderer: {}
+      colorScheme: tableau.Tableau10

tests/compose/testdata/namespace_export_simple/data_model/1000_namespace.yaml

@@ -0,0 +1,3 @@
+namespaces:
+  ns1:
+    name: ns1 name

tests/compose/testdata/namespace_export_simple/data_model/1100_modules.yaml

@@ -0,0 +1,42 @@
+namespace: ns1
+modules:
+  mod1:
+    name: mod1 name
+    fields:
+      f1:
+        label: f1 label
+        kind: String
+        required: true
+      f2:
+        label: f2 label
+        kind: Select
+        options:
+          options:
+          - f2 opt 1
+          - f2 opt 2
+          - f2 opt 3
+      f3:
+        label: f3 label
+        kind: Record
+        options:
+          labelField: f_label
+          module: mod2
+          queryFields:
+          - f1
+
+  mod2:
+    name: mod2 name
+    fields:
+      f_label:
+        label: f_label record label
+      f1:
+        label: f1 label
+        kind: String
+        required: true
+
+  mod3:
+    name: mod3 name
+    fields:
+      f1:
+        label: f1 label
+        kind: String

tests/compose/testdata/namespace_export_simple/data_model/1200_pages.yaml

@@ -0,0 +1,47 @@
+namespace: ns1
+pages:
+  pg1:
+    title: pg1 title
+    blocks:
+    - title: pg1 b1
+      kind: Content
+      xywh: [0, 0, 1, 1]
+      options:
+        body: pg1 b1 content body
+
+    - title: pg1 b2
+      kind: RecordList
+      xywh: [0, 1, 1, 1]
+      options:
+        module: mod1
+        fields:
+        - name: f1
+        - name: f2
+
+    - title: pg1 b3
+      kind: Chart
+      xywh: [0, 2, 1, 1]
+      options:
+        chart: chr1
+
+  rpg2:
+    handle: rpg2
+    module: mod1
+    title: Record page for module "mod1"
+    blocks:
+    - title: rpg2 b1 title
+      kind: Record
+      options:
+        fields:
+        - name: f1
+        - name: f2
+
+    pages:
+      pg2:
+        title: pg2 title
+        blocks:
+        - title: pg2 b1
+          kind: Content
+          xywh: [0, 0, 1, 1]
+          options:
+            body: pg2 b1 content body

tests/compose/testdata/namespace_export_simple/data_model/1300_charts.yaml

@@ -0,0 +1,18 @@
+namespace: ns1
+charts:
+  chr1:
+    name: chr1 name
+    config:
+      reports:
+      - dimensions:
+        - field: Status
+          modifier: (no grouping / buckets)
+        filter: ""
+        metrics:
+        - backgroundColor: '#11ff57'
+          field: count
+          fixTooltips: true
+          type: pie
+        module: mod1
+        renderer: {}
+      colorScheme: tableau.Tableau10