From 040ad02e7598b673a723d47f153a0143b5e3df1a Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Sun, 22 May 2022 18:39:35 +0200 Subject: [PATCH] Refactor and fix DAL tests DAL tests are now moved to /tests/dal --- store/adapters/rdbms/dal/connection.go | 9 + store/adapters/rdbms/dal/model.go | 4 + .../adapters/rdbms/drivers/mysql/dal_test.go | 111 -------- .../rdbms/drivers/postgres/dal_test.go | 101 -------- .../adapters/rdbms/drivers/sqlite/dal_test.go | 105 -------- pkg/dal/test/driver.go => tests/dal/codec.go | 208 +-------------- tests/dal/connection_wrap.go | 34 --- tests/dal/dal_test.go | 239 +++++------------- tests/dal/helpers.go | 26 ++ tests/dal/search.go | 184 ++++++++++++++ tests/dal/setup/mysql/setup.go | 84 ++++++ tests/dal/setup/postgres/setup.go | 80 ++++++ tests/dal/setup/sqlite/setup.go | 82 ++++++ 13 files changed, 540 insertions(+), 727 deletions(-) delete mode 100644 store/adapters/rdbms/drivers/mysql/dal_test.go delete mode 100644 store/adapters/rdbms/drivers/postgres/dal_test.go delete mode 100644 store/adapters/rdbms/drivers/sqlite/dal_test.go rename pkg/dal/test/driver.go => tests/dal/codec.go (54%) delete mode 100644 tests/dal/connection_wrap.go create mode 100644 tests/dal/helpers.go create mode 100644 tests/dal/search.go create mode 100644 tests/dal/setup/mysql/setup.go create mode 100644 tests/dal/setup/postgres/setup.go create mode 100644 tests/dal/setup/sqlite/setup.go diff --git a/store/adapters/rdbms/dal/connection.go b/store/adapters/rdbms/dal/connection.go index 77cbfafc7..bfbabaed3 100644 --- a/store/adapters/rdbms/dal/connection.go +++ b/store/adapters/rdbms/dal/connection.go @@ -32,6 +32,15 @@ func Connection(db *sqlx.DB, dialect drivers.Dialect, cc ...capabilities.Capabil } func (c *connection) model(m *dal.Model) *model { + if m.Resource == "" { + // if resource is empty, use ident + m.Resource = m.Ident + } + + if m.Resource == "" { + panic("can not add model with empty resource") + } + c.mux.RLock() if c.models[m.Resource] == nil { c.mux.RUnlock() diff --git a/store/adapters/rdbms/dal/model.go b/store/adapters/rdbms/dal/model.go index a52dd5fbd..db59ff8c8 100644 --- a/store/adapters/rdbms/dal/model.go +++ b/store/adapters/rdbms/dal/model.go @@ -54,6 +54,10 @@ func Model(m *dal.Model, c queryRunner, d drivers.Dialect) *model { ms.queryParser = ql.Converter( ql.SymHandler(func(node *ql.ASTNode) (exp.Expression, error) { attr := ms.model.Attributes.FindByIdent(node.Symbol) + if attr == nil { + return nil, fmt.Errorf("unknown attribute %q used in query expression", node.Symbol) + } + if !attr.Filterable { return nil, fmt.Errorf("attribute %q can not be used in query expression", attr.Ident) } diff --git a/store/adapters/rdbms/drivers/mysql/dal_test.go b/store/adapters/rdbms/drivers/mysql/dal_test.go deleted file mode 100644 index 4e736b0da..000000000 --- a/store/adapters/rdbms/drivers/mysql/dal_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package mysql - -import ( - "context" - "strings" - "testing" - - "github.com/cortezaproject/corteza-server/pkg/dal/test" - "github.com/cortezaproject/corteza-server/pkg/logger" - "github.com/cortezaproject/corteza-server/store/adapters/rdbms" - "github.com/cortezaproject/corteza-server/store/adapters/rdbms/dal" - "github.com/jmoiron/sqlx" - "github.com/stretchr/testify/require" -) - -func connect(req *require.Assertions) (db *sqlx.DB) { - var ( - err error - ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) - cfg *rdbms.ConnConfig - ) - - cfg, err = NewConfig("mysql+debug://crust:crust@tcp(localhost:3306)/crust_store_test?collation=utf8mb4_general_ci&charset=utf8mb4") - req.NoError(err) - - db, err = rdbms.Connect(ctx, logger.MakeDebugLogger(), cfg) - req.NoError(err) - - return -} - -func TestMySQL(t *testing.T) { - var ( - req = require.New(t) - db = connect(req) - ) - - defer db.Close() - - setupCodecTest(db, req) - setupRecordSearchTest(db, req) - test.All(t, dal.Connection(db, &dialect{})) -} - -// remove when store support for table creation is added to CRS -// -// when support is added, test.TestRecordCodec -func setupCodecTest(db sqlx.ExecerContext, req *require.Assertions) { - var ( - err error - ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) - ) - - _, err = db.ExecContext(ctx, `DROP TABLE IF EXISTS crs_test_codec`) - req.NoError(err) - - tblCreate := ` - CREATE TABLE IF NOT EXISTS crs_test_codec ( - "id" BIGINT UNSIGNED NOT NULL, - "created_at" DATETIME, - "updated_at" DATETIME, - "meta" JSON, - "pID" BIGINT UNSIGNED, - "pRef" BIGINT UNSIGNED, - "pTimestamp_TZT" DATETIME, - "pTimestamp_TZF" DATETIME, - "pTime" TIME, - "pDate" DATE, - "pNumber" NUMERIC, - "pText" TEXT, - "pBoolean_T" BOOLEAN, - "pBoolean_F" BOOLEAN, - "pEnum" TEXT, - "pGeometry" TEXT, - "pJSON" TEXT, - "pBlob" BLOB, - "pUUID" VARCHAR(36), - - PRIMARY KEY("id") - )` - - _, err = db.ExecContext(ctx, strings.ReplaceAll(tblCreate, "\"", "`")) - req.NoError(err) -} - -// remove when store support for table creation is added to CRS -func setupRecordSearchTest(db sqlx.ExecerContext, req *require.Assertions) { - var ( - err error - ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) - ) - - _, err = db.ExecContext(ctx, `DROP TABLE IF EXISTS crs_test_search`) - req.NoError(err) - tblCreate := ` - CREATE TABLE IF NOT EXISTS crs_test_search ( - "id" INT NOT NULL, - "created_at" DATETIME, - "updated_at" DATETIME, - "meta" JSON, - "p_string" TEXT, - "p_number" NUMERIC, - "p_is_odd" BOOLEAN, - - PRIMARY KEY("id") - ) - ` - - _, err = db.ExecContext(ctx, strings.ReplaceAll(tblCreate, "\"", "`")) - req.NoError(err) -} diff --git a/store/adapters/rdbms/drivers/postgres/dal_test.go b/store/adapters/rdbms/drivers/postgres/dal_test.go deleted file mode 100644 index ae12cda50..000000000 --- a/store/adapters/rdbms/drivers/postgres/dal_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package postgres - -import ( - "context" - "testing" - - "github.com/cortezaproject/corteza-server/pkg/dal/test" - "github.com/cortezaproject/corteza-server/pkg/logger" - "github.com/cortezaproject/corteza-server/store/adapters/rdbms" - "github.com/cortezaproject/corteza-server/store/adapters/rdbms/dal" - "github.com/jmoiron/sqlx" - "github.com/stretchr/testify/require" -) - -func connect(req *require.Assertions) (db *sqlx.DB) { - var ( - err error - ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) - cfg *rdbms.ConnConfig - ) - - cfg, err = NewConfig("postgres+debug://darh@localhost:5432/corteza_2022_3?sslmode=disable&") - req.NoError(err) - - db, err = rdbms.Connect(ctx, logger.MakeDebugLogger(), cfg) - req.NoError(err) - - return -} - -func TestPostgres(t *testing.T) { - var ( - req = require.New(t) - db = connect(req) - ) - - defer db.Close() - - setupCodecTest(db, req) - setupRecordSearchTest(db, req) - test.All(t, dal.Connection(db, Dialect())) -} - -// remove when store support for table creation is added to CRS -// -// when support is added, test.TestRecordCodec -func setupCodecTest(db sqlx.ExecerContext, req *require.Assertions) { - var ( - err error - ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) - ) - - _, err = db.ExecContext(ctx, ` - CREATE TEMPORARY TABLE IF NOT EXISTS crs_test_codec ( - "id" BIGINT NOT NULL, - "created_at" TIMESTAMP NOT NULL, - "updated_at" TIMESTAMP, - "meta" JSON, - "pID" BIGINT, - "pRef" BIGINT, - "pTimestamp_TZT" TIMESTAMPTZ, - "pTimestamp_TZF" TIMESTAMP, - "pTime" TIME, - "pDate" DATE, - "pNumber" NUMERIC, - "pText" TEXT, - "pBoolean_T" BOOLEAN, - "pBoolean_F" BOOLEAN, - "pEnum" TEXT, - "pGeometry" TEXT, - "pJSON" TEXT, - "pBlob" BYTEA, - "pUUID" UUID, - - PRIMARY KEY(id) - )`) - req.NoError(err) -} - -// remove when store support for table creation is added to CRS -func setupRecordSearchTest(db sqlx.ExecerContext, req *require.Assertions) { - var ( - err error - ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) - ) - - _, err = db.ExecContext(ctx, ` - CREATE TEMPORARY TABLE IF NOT EXISTS crs_test_search ( - "id" BIGINT NOT NULL, - "created_at" TIMESTAMPTZ NOT NULL, - "updated_at" TIMESTAMPTZ, - "meta" JSON, - "p_string" TEXT, - "p_number" NUMERIC, - "p_is_odd" BOOLEAN, - - PRIMARY KEY(id ) - ) - `) - req.NoError(err) -} diff --git a/store/adapters/rdbms/drivers/sqlite/dal_test.go b/store/adapters/rdbms/drivers/sqlite/dal_test.go deleted file mode 100644 index 846cf3ee0..000000000 --- a/store/adapters/rdbms/drivers/sqlite/dal_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package sqlite - -import ( - "context" - "testing" - - "github.com/cortezaproject/corteza-server/pkg/dal/test" - "github.com/cortezaproject/corteza-server/pkg/logger" - "github.com/cortezaproject/corteza-server/store/adapters/rdbms" - "github.com/cortezaproject/corteza-server/store/adapters/rdbms/dal" - "github.com/jmoiron/sqlx" - "github.com/stretchr/testify/require" -) - -func connect(req *require.Assertions) (db *sqlx.DB) { - var ( - err error - ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) - cfg *rdbms.ConnConfig - ) - - cfg, err = NewConfig("sqlite3+debug://file::memory:?cache=shared&mode=memory") - //cfg, err = NewConfig("sqlite3://file:/tmp/test.db") - req.NoError(err) - db, err = rdbms.Connect(ctx, logger.MakeDebugLogger(), cfg) - req.NoError(err) - - return -} - -func TestSQLite(t *testing.T) { - var ( - req = require.New(t) - db = connect(req) - ) - - defer db.Close() - - setupCodecTest(db, req) - setupRecordSearchTest(db, req) - test.All(t, dal.Connection(db, Dialect())) -} - -// remove when store support for table creation is added to CRS -// -// when support is added, test.TestRecordCodec -func setupCodecTest(db sqlx.ExecerContext, req *require.Assertions) { - var ( - err error - ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) - ) - - _, err = db.ExecContext(ctx, `DROP TABLE IF EXISTS crs_test_codec`) - req.NoError(err) - _, err = db.ExecContext(ctx, ` - CREATE TABLE IF NOT EXISTS crs_test_codec ( - id UNSIGNED BIG INT NOT NULL, - created_at TIMESTAMP, - updated_at TIMESTAMP, - meta JSON, - pID UNSIGNED BIG INT, - pRef UNSIGNED BIG INT, - pTimestamp_TZT TIMESTAMP, - pTimestamp_TZF TIMESTAMP, - pTime TIME, - pDate DATE, - pNumber NUMERIC, - pText TEXT, - pBoolean_T BOOLEAN, - pBoolean_F BOOLEAN, - pEnum TEXT, - pGeometry TEXT, - pJSON TEXT, - pBlob BLOB, - pUUID UUID, - - PRIMARY KEY(id) - )`) - req.NoError(err) -} - -// remove when store support for table creation is added to CRS -func setupRecordSearchTest(db sqlx.ExecerContext, req *require.Assertions) { - var ( - err error - ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) - ) - - _, err = db.ExecContext(ctx, `DROP TABLE IF EXISTS crs_test_search`) - req.NoError(err) - _, err = db.ExecContext(ctx, ` - CREATE TABLE IF NOT EXISTS crs_test_search ( - id INT NOT NULL, - created_at TIMESTAMP, - updated_at TIMESTAMP, - meta JSON, - p_string TEXT, - p_number NUMERIC, - p_is_odd BOOLEAN, - - PRIMARY KEY(id ) - ) - `) - req.NoError(err) -} diff --git a/pkg/dal/test/driver.go b/tests/dal/codec.go similarity index 54% rename from pkg/dal/test/driver.go rename to tests/dal/codec.go index a8979b490..5e378dd78 100644 --- a/pkg/dal/test/driver.go +++ b/tests/dal/codec.go @@ -1,31 +1,19 @@ -package test +package dal import ( "context" - "fmt" - "strconv" "testing" "time" "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/dal" - "github.com/cortezaproject/corteza-server/pkg/filter" - "github.com/cortezaproject/corteza-server/pkg/logger" "github.com/stretchr/testify/require" ) -func All(t *testing.T, d dal.Connection) { - t.Run("RecordCodec", func(t *testing.T) { RecordCodec(t, d) }) - t.Run("RecordSearch", func(t *testing.T) { RecordSearch(t, d) }) -} - -func RecordCodec(t *testing.T, d dal.Connection) { +func RecordCodec(t *testing.T, ctx context.Context, d dal.Connection) { var ( req = require.New(t) - // enable query logging when +debug is used on DSN schema - ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) - m = &dal.Model{ Ident: "crs_test_codec", Attributes: dal.AttributeSet{ @@ -133,195 +121,3 @@ func RecordCodec(t *testing.T, d dal.Connection) { req.Equal(vIn, vOut, "values for attribute %q are not equal", attr.Ident) } } - -func RecordSearch(t *testing.T, d dal.Connection) { - const ( - totalRecords = 10 - ) - - var ( - req = require.New(t) - - // enable query logging when +debug is used on DSN schema - ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) - - m = &dal.Model{ - Ident: "crs_test_search", - Attributes: dal.AttributeSet{ - &dal.Attribute{Ident: "ID", Type: &dal.TypeID{}, Store: &dal.CodecAlias{Ident: "id"}, PrimaryKey: true}, - &dal.Attribute{Ident: "createdAt", Type: &dal.TypeTimestamp{}, Store: &dal.CodecAlias{Ident: "created_at"}}, - &dal.Attribute{Ident: "updatedAt", Type: &dal.TypeTimestamp{}, Store: &dal.CodecAlias{Ident: "updated_at"}}, - - &dal.Attribute{Ident: "v_string", Type: &dal.TypeText{}, Store: &dal.CodecRecordValueSetJSON{Ident: "meta"}}, - &dal.Attribute{Ident: "v_number", Type: &dal.TypeNumber{}, Store: &dal.CodecRecordValueSetJSON{Ident: "meta"}}, - &dal.Attribute{Ident: "v_is_odd", Type: &dal.TypeBoolean{}, Store: &dal.CodecRecordValueSetJSON{Ident: "meta"}}, - &dal.Attribute{Ident: "p_string", Type: &dal.TypeText{}, Store: &dal.CodecPlain{}}, - &dal.Attribute{Ident: "p_number", Type: &dal.TypeNumber{}, Store: &dal.CodecPlain{}}, - &dal.Attribute{Ident: "p_is_odd", Type: &dal.TypeBoolean{}, Store: &dal.CodecPlain{}}, - }, - } - ) - - for ID := uint64(1); ID <= totalRecords; ID++ { - r := &types.Record{ID: ID, CreatedAt: time.Now()} - - i := int(ID) - r.Values = r.Values.Set(&types.RecordValue{Name: "v_string", Value: "tens_" + strconv.Itoa(i%10)}) - r.Values = r.Values.Set(&types.RecordValue{Name: "v_number", Value: strconv.Itoa(i)}) - r.Values = r.Values.Set(&types.RecordValue{Name: "v_is_odd", Value: strconv.FormatBool(i%2 == 1)}) - r.Values = r.Values.Set(&types.RecordValue{Name: "p_string", Value: "tens_" + strconv.Itoa(i%10)}) - r.Values = r.Values.Set(&types.RecordValue{Name: "p_number", Value: strconv.Itoa(i)}) - r.Values = r.Values.Set(&types.RecordValue{Name: "p_is_odd", Value: strconv.FormatBool(i%2 == 1)}) - - req.NoError(d.Create(ctx, m, r)) - } - - cases := []struct { - f types.RecordFilter - total int - }{ - { - total: totalRecords, - }, - { - f: types.RecordFilter{Query: "v_string == p_string"}, - total: totalRecords, - }, - { - f: types.RecordFilter{Query: "v_number == p_number"}, - total: totalRecords, - }, - { - f: types.RecordFilter{Query: "p_is_odd"}, - total: totalRecords / 2, - }, - { - f: types.RecordFilter{Query: "true = p_is_odd"}, - total: totalRecords / 2, - }, - { - f: types.RecordFilter{Query: "p_is_odd = true"}, - total: totalRecords / 2, - }, - { - f: types.RecordFilter{Query: "!p_is_odd"}, - total: totalRecords / 2, - }, - { - f: types.RecordFilter{Query: "p_number = 1"}, - total: 1, - }, - } - - for _, c := range cases { - t.Run(c.f.Query, func(t *testing.T) { - var ( - req = require.New(t) - ) - - i, err := d.Search(ctx, m, c.f.ToFilter()) - req.NoError(err) - - rr, err := drain(ctx, i) - req.NoError(err) - req.Len(rr, c.total) - }) - } - - t.Run("paging", func(t *testing.T) { - var ( - req = require.New(t) - ids string - fwd, bck *filter.PagingCursor - - search = func(where, orderBy string, lim uint, cur *filter.PagingCursor) (ids string, fwd, bck *filter.PagingCursor) { - f := types.RecordFilter{Query: where} - f.PageCursor = cur - f.Limit = lim - req.NoError(f.Sort.Set(orderBy)) - i, err := d.Search(ctx, m, f.ToFilter()) - req.NoError(err) - req.NoError(i.Err()) - - if !i.Next(ctx) { - req.NoError(i.Err()) - return - } - - r := new(types.Record) - req.NoError(i.Scan(r)) - - bck, err = i.BackCursor(r) - req.NoError(err) - t.Logf("bck-cursor (from the 1st fetched record): %v", bck) - - rr, err := drain(ctx, i) - req.NoError(err) - - if len(rr) > 0 { - fwd, err = i.ForwardCursor(rr[len(rr)-1]) - req.NoError(err) - t.Logf("fwd-cursor (from the lst fetched record): %v", fwd) - } - - ids = fmt.Sprintf("%d", r.ID) - for _, r = range rr { - ids += fmt.Sprintf(",%d", r.ID) - } - - return - } - ) - - ids, fwd, _ = search("", "", 3, nil) - req.Equal("1,2,3", ids) - ids, fwd, _ = search("", "", 3, fwd) - req.Equal("4,5,6", ids) - ids, fwd, _ = search("", "", 3, fwd) - req.Equal("7,8,9", ids) - ids, _, bck = search("", "", 3, fwd) - req.Equal("10", ids) - - ids, _, bck = search("", "", 3, bck) - req.Equal("7,8,9", ids) - ids, _, bck = search("", "", 3, bck) - req.Equal("4,5,6", ids) - ids, _, bck = search("", "", 3, bck) - req.Equal("1,2,3", ids) - - ids, fwd, _ = search("", "p_is_odd", 3, nil) - req.Equal("2,4,6", ids) - ids, _, bck = search("", "", 3, fwd) - req.Equal("8,10,1", ids) - ids, _, _ = search("", "", 3, bck) - req.Equal("2,4,6", ids) - - ids, fwd, _ = search("", "v_is_odd", 3, nil) - req.Equal("2,4,6", ids) - ids, _, bck = search("", "", 3, fwd) - req.Equal("8,10,1", ids) - ids, _, _ = search("", "", 3, bck) - req.Equal("2,4,6", ids) - - _, _ = fwd, bck // avoiding unused var. error - }) -} - -func drain(ctx context.Context, i dal.Iterator) (rr []*types.Record, err error) { - var r *types.Record - rr = make([]*types.Record, 0, 100) - for i.Next(ctx) { - if i.Err() != nil { - return nil, i.Err() - } - - r = new(types.Record) - if err = i.Scan(r); err != nil { - return - } - - rr = append(rr, r) - } - - return rr, i.Err() -} diff --git a/tests/dal/connection_wrap.go b/tests/dal/connection_wrap.go deleted file mode 100644 index 818ded859..000000000 --- a/tests/dal/connection_wrap.go +++ /dev/null @@ -1,34 +0,0 @@ -package dal - -import "github.com/cortezaproject/corteza-server/pkg/dal/capabilities" - -type ( - connection struct { - id uint64 - dsn string - capabilities capabilities.Set - } -) - -// Connection is a utility to define generic store connections -func Connection(id uint64, dsn string, cc ...capabilities.Capability) connection { - return connection{ - id: id, - dsn: dsn, - capabilities: cc, - } -} - -// Receivers to conform to interface - -func (s connection) ComposeRecordStoreID() uint64 { - return s.id -} - -func (s connection) StoreDSN() string { - return s.dsn -} - -func (s connection) Capabilities() capabilities.Set { - return s.capabilities -} diff --git a/tests/dal/dal_test.go b/tests/dal/dal_test.go index 9d2d20e0d..afdd17784 100644 --- a/tests/dal/dal_test.go +++ b/tests/dal/dal_test.go @@ -2,184 +2,83 @@ package dal import ( "context" - "testing" - "time" - - "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/dal" - "github.com/cortezaproject/corteza-server/pkg/dal/capabilities" - "github.com/cortezaproject/corteza-server/pkg/id" - - "github.com/stretchr/testify/require" - - _ "github.com/cortezaproject/corteza-server/store/adapters/rdbms/drivers/mysql" + "github.com/cortezaproject/corteza-server/pkg/logger" + "github.com/cortezaproject/corteza-server/tests/dal/setup/mysql" + "github.com/cortezaproject/corteza-server/tests/dal/setup/postgres" + "github.com/cortezaproject/corteza-server/tests/dal/setup/sqlite" + "github.com/cortezaproject/corteza-server/tests/helpers" + _ "github.com/joho/godotenv/autoload" + "os" + "strings" + "testing" ) -/* -See here for my table definitions +// Tests DAL functionalities for all supported databases +// +// Tests for drivers are executed if DSN env variable (DAL_TEST_DSN_) is present. +// Example: DAL_TEST_DSN_MYSQL, DAL_TEST_DSN_POSTGRES +// +// Multiple space delimited DSNs are supported. +// +// Tests scans current and two parent folders for presence of .env file +// and loads the first one found. +// +// +func TestDAL(t *testing.T) { + helpers.RecursiveDotEnvLoad() -CREATE TABLE `the_cake` ( - `id` bigint unsigned NOT NULL, - `name` VARCHAR(45), - `want` BOOLEAN, - `ownedBy` bigint unsigned NOT NULL, - `createdAt` datetime NOT NULL, - `updatedAt` datetime DEFAULT NULL, - `deletedAt` datetime DEFAULT NULL, - `createdBy` bigint unsigned NOT NULL, - `updatedBy` bigint unsigned NOT NULL DEFAULT '0', - `deletedBy` bigint unsigned NOT NULL DEFAULT '0', - PRIMARY KEY (`id`), - KEY `compose_record_owner` (`ownedBy`) -) ENGINE = InnoDB DEFAULT CHARSET = utf8mb3; + var ( + // enrich context with debug logger + // + // this will enable us to log driver commands + // when +debug is used on DSN schema + ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) -CREATE TABLE `the_cookie` ( - `id` bigint unsigned NOT NULL, - `name` VARCHAR(45), - `is_good` BOOLEAN, - `ownedBy` bigint unsigned NOT NULL, - `createdAt` datetime NOT NULL, - `updatedAt` datetime DEFAULT NULL, - `deletedAt` datetime DEFAULT NULL, - `createdBy` bigint unsigned NOT NULL, - `updatedBy` bigint unsigned NOT NULL DEFAULT '0', - `deletedBy` bigint unsigned NOT NULL DEFAULT '0', - PRIMARY KEY (`id`), - KEY `compose_record_owner` (`ownedBy`) -) ENGINE = InnoDB DEFAULT CHARSET = utf8mb3; -*/ + conn dal.Connection + err error -func TestHello(t *testing.T) { - ctx := context.Background() - - c, err := dal.Service( - ctx, - nil, - false, - // Primary... - Connection(0, "mysql://envoy:envoy@tcp(localhost:3306)/crs?collation=utf8mb4_general_ci", capabilities.FullCapabilities()...), - - // Others... - Connection(1, "mysql://envoy:envoy@tcp(localhost:3306)/crs?collation=utf8mb4_general_ci", capabilities.FullCapabilities()...), + drivers = []struct { + name string + dsn string + connect func(string) (dal.Connection, error) + }{ + { + name: "sqlite", + dsn: "sqlite3+debug://file::memory:?cache=shared&mode=memory", + connect: sqlite.Setup, + }, + { + name: "mysql", + dsn: os.Getenv("DAL_TEST_DSN_MYSQL"), + connect: mysql.Setup, + }, + { + name: "postgres", + dsn: os.Getenv("DAL_TEST_DSN_POSTGRES"), + connect: postgres.Setup, + }, + } ) - require.NoError(t, err) - _ = c - // --- + for _, driver := range drivers { + t.Run(driver.name, func(t *testing.T) { + if driver.dsn == "" { + t.Skip("DSN for DAL test not set") + } - cookieModule := &types.Module{ - ID: 10001, - Handle: "cookie", - Store: types.DalDef{ - ComposeRecordStoreID: 0, - Capabilities: capabilities.FullCapabilities(), - Partitioned: true, - PartitionFormat: "the_{{module}}", - }, - Fields: types.ModuleFieldSet{&types.ModuleField{ - ModuleID: 10001, - ID: 20001, - Name: "name", - Encoding: types.EncodingStrategy{ - EncodingStrategyAlias: &types.EncodingStrategyAlias{ - Ident: "name", - }, - }, - }, &types.ModuleField{ - ModuleID: 10001, - ID: 20002, - Name: "is_good", - Kind: "Bool", - Encoding: types.EncodingStrategy{ - EncodingStrategyAlias: &types.EncodingStrategyAlias{ - Ident: "is_good", - }, - }, - }}, + for _, dsn := range strings.Split(driver.dsn, " ") { + t.Run("", func(t *testing.T) { + t.Log("Connecting to ", dsn) + if conn, err = driver.connect(dsn); err != nil { + t.Fatal(err) + } + + t.Run("RecordCodec", func(t *testing.T) { RecordCodec(t, ctx, conn) }) + t.Run("RecordSearch", func(t *testing.T) { RecordSearch(t, ctx, conn) }) + }) + } + + }) } - - cakeModule := &types.Module{ - ID: 10002, - Handle: "cake", - Store: types.DalDef{ - ComposeRecordStoreID: 1, - Capabilities: capabilities.FullCapabilities(), - Partitioned: true, - PartitionFormat: "the_{{module}}", - }, - Fields: types.ModuleFieldSet{&types.ModuleField{ - ModuleID: 10002, - ID: 20003, - Name: "name", - Encoding: types.EncodingStrategy{ - EncodingStrategyAlias: &types.EncodingStrategyAlias{ - // this doesn't work - Ident: "name", - }, - }, - }, &types.ModuleField{ - ModuleID: 10002, - ID: 20004, - Name: "want", - Kind: "Bool", - Encoding: types.EncodingStrategy{ - EncodingStrategyAlias: &types.EncodingStrategyAlias{ - Ident: "want", - }, - }, - }}, - } - - err = c.ReloadModules(ctx, cookieModule, cakeModule) - require.NoError(t, err) - - // --- - - a := time.Now() - - cookieRecord := &types.Record{ - ID: id.Next(), - ModuleID: 10001, - Values: types.RecordValueSet{{ - Name: "name", - Value: "SOME COOKIE HERE", - }, { - Name: "is_good", - Value: "1", - }}, - CreatedAt: time.Now(), - UpdatedAt: &a, - DeletedAt: &a, - OwnedBy: 20003, - CreatedBy: 20003, - UpdatedBy: 20003, - DeletedBy: 20003, - } - - cakeRecord := &types.Record{ - ID: id.Next(), - ModuleID: 10002, - Values: types.RecordValueSet{{ - Name: "name", - Value: "DOME CAKE HERE", - }, { - Name: "want", - Value: "1", - }}, - CreatedAt: time.Now(), - UpdatedAt: &a, - DeletedAt: &a, - OwnedBy: 20003, - CreatedBy: 20003, - UpdatedBy: 20003, - DeletedBy: 20003, - } - - err = c.ComposeRecordCreate(ctx, cookieModule, cookieRecord) - require.NoError(t, err) - - err = c.ComposeRecordCreate(ctx, cakeModule, cakeRecord) - require.NoError(t, err) - - require.FailNow(t, "the test dies a failNow so I can see logs!!!") } diff --git a/tests/dal/helpers.go b/tests/dal/helpers.go new file mode 100644 index 000000000..32d707f45 --- /dev/null +++ b/tests/dal/helpers.go @@ -0,0 +1,26 @@ +package dal + +import ( + "context" + "github.com/cortezaproject/corteza-server/compose/types" + "github.com/cortezaproject/corteza-server/pkg/dal" +) + +func drain(ctx context.Context, i dal.Iterator) (rr []*types.Record, err error) { + var r *types.Record + rr = make([]*types.Record, 0, 100) + for i.Next(ctx) { + if i.Err() != nil { + return nil, i.Err() + } + + r = new(types.Record) + if err = i.Scan(r); err != nil { + return + } + + rr = append(rr, r) + } + + return rr, i.Err() +} diff --git a/tests/dal/search.go b/tests/dal/search.go new file mode 100644 index 000000000..86b18ff68 --- /dev/null +++ b/tests/dal/search.go @@ -0,0 +1,184 @@ +package dal + +import ( + "context" + "fmt" + "strconv" + "testing" + "time" + + "github.com/cortezaproject/corteza-server/compose/types" + "github.com/cortezaproject/corteza-server/pkg/dal" + "github.com/cortezaproject/corteza-server/pkg/filter" + "github.com/stretchr/testify/require" +) + +func RecordSearch(t *testing.T, ctx context.Context, d dal.Connection) { + const ( + totalRecords = 10 + ) + + var ( + req = require.New(t) + + m = &dal.Model{ + Ident: "crs_test_search", + Attributes: dal.AttributeSet{ + &dal.Attribute{Ident: "ID", Type: &dal.TypeID{}, Store: &dal.CodecAlias{Ident: "id"}, PrimaryKey: true}, + &dal.Attribute{Ident: "createdAt", Type: &dal.TypeTimestamp{}, Store: &dal.CodecAlias{Ident: "created_at"}}, + &dal.Attribute{Ident: "updatedAt", Type: &dal.TypeTimestamp{}, Store: &dal.CodecAlias{Ident: "updated_at"}}, + + &dal.Attribute{Ident: "v_string", Filterable: true, Type: &dal.TypeText{}, Store: &dal.CodecRecordValueSetJSON{Ident: "meta"}}, + &dal.Attribute{Ident: "v_number", Filterable: true, Type: &dal.TypeNumber{}, Store: &dal.CodecRecordValueSetJSON{Ident: "meta"}}, + &dal.Attribute{Ident: "v_is_odd", Filterable: true, Type: &dal.TypeBoolean{}, Store: &dal.CodecRecordValueSetJSON{Ident: "meta"}}, + &dal.Attribute{Ident: "p_string", Filterable: true, Type: &dal.TypeText{}, Store: &dal.CodecPlain{}}, + &dal.Attribute{Ident: "p_number", Filterable: true, Type: &dal.TypeNumber{}, Store: &dal.CodecPlain{}}, + &dal.Attribute{Ident: "p_is_odd", Filterable: true, Type: &dal.TypeBoolean{}, Store: &dal.CodecPlain{}}, + }, + } + ) + + for ID := uint64(1); ID <= totalRecords; ID++ { + r := &types.Record{ID: ID, CreatedAt: time.Now()} + + i := int(ID) + r.Values = r.Values.Set(&types.RecordValue{Name: "v_string", Value: "tens_" + strconv.Itoa(i%10)}) + r.Values = r.Values.Set(&types.RecordValue{Name: "v_number", Value: strconv.Itoa(i)}) + r.Values = r.Values.Set(&types.RecordValue{Name: "v_is_odd", Value: strconv.FormatBool(i%2 == 1)}) + r.Values = r.Values.Set(&types.RecordValue{Name: "p_string", Value: "tens_" + strconv.Itoa(i%10)}) + r.Values = r.Values.Set(&types.RecordValue{Name: "p_number", Value: strconv.Itoa(i)}) + r.Values = r.Values.Set(&types.RecordValue{Name: "p_is_odd", Value: strconv.FormatBool(i%2 == 1)}) + + req.NoError(d.Create(ctx, m, r)) + } + + cases := []struct { + f types.RecordFilter + total int + }{ + { + total: totalRecords, + }, + { + f: types.RecordFilter{Query: "v_string == p_string"}, + total: totalRecords, + }, + { + f: types.RecordFilter{Query: "v_number == p_number"}, + total: totalRecords, + }, + { + f: types.RecordFilter{Query: "p_is_odd"}, + total: totalRecords / 2, + }, + { + f: types.RecordFilter{Query: "true = p_is_odd"}, + total: totalRecords / 2, + }, + { + f: types.RecordFilter{Query: "p_is_odd = true"}, + total: totalRecords / 2, + }, + { + f: types.RecordFilter{Query: "!p_is_odd"}, + total: totalRecords / 2, + }, + { + f: types.RecordFilter{Query: "p_number = 1"}, + total: 1, + }, + } + + for _, c := range cases { + t.Run(c.f.Query, func(t *testing.T) { + var ( + req = require.New(t) + ) + + i, err := d.Search(ctx, m, c.f.ToFilter()) + req.NoError(err) + + rr, err := drain(ctx, i) + req.NoError(err) + req.Len(rr, c.total) + }) + } + + t.Run("paging", func(t *testing.T) { + var ( + req = require.New(t) + ids string + fwd, bck *filter.PagingCursor + + search = func(where, orderBy string, lim uint, cur *filter.PagingCursor) (ids string, fwd, bck *filter.PagingCursor) { + f := types.RecordFilter{Query: where} + f.PageCursor = cur + f.Limit = lim + req.NoError(f.Sort.Set(orderBy)) + i, err := d.Search(ctx, m, f.ToFilter()) + req.NoError(err) + req.NoError(i.Err()) + + if !i.Next(ctx) { + req.NoError(i.Err()) + return + } + + r := new(types.Record) + req.NoError(i.Scan(r)) + + bck, err = i.BackCursor(r) + req.NoError(err) + t.Logf("bck-cursor (from the 1st fetched record): %v", bck) + + rr, err := drain(ctx, i) + req.NoError(err) + + if len(rr) > 0 { + fwd, err = i.ForwardCursor(rr[len(rr)-1]) + req.NoError(err) + t.Logf("fwd-cursor (from the lst fetched record): %v", fwd) + } + + ids = fmt.Sprintf("%d", r.ID) + for _, r = range rr { + ids += fmt.Sprintf(",%d", r.ID) + } + + return + } + ) + + ids, fwd, _ = search("", "", 3, nil) + req.Equal("1,2,3", ids) + ids, fwd, _ = search("", "", 3, fwd) + req.Equal("4,5,6", ids) + ids, fwd, _ = search("", "", 3, fwd) + req.Equal("7,8,9", ids) + ids, _, bck = search("", "", 3, fwd) + req.Equal("10", ids) + + ids, _, bck = search("", "", 3, bck) + req.Equal("7,8,9", ids) + ids, _, bck = search("", "", 3, bck) + req.Equal("4,5,6", ids) + ids, _, bck = search("", "", 3, bck) + req.Equal("1,2,3", ids) + + ids, fwd, _ = search("", "p_is_odd", 3, nil) + req.Equal("2,4,6", ids) + ids, _, bck = search("", "", 3, fwd) + req.Equal("8,10,1", ids) + ids, _, _ = search("", "", 3, bck) + req.Equal("2,4,6", ids) + + ids, fwd, _ = search("", "v_is_odd", 3, nil) + req.Equal("2,4,6", ids) + ids, _, bck = search("", "", 3, fwd) + req.Equal("8,10,1", ids) + ids, _, _ = search("", "", 3, bck) + req.Equal("2,4,6", ids) + + _, _ = fwd, bck // avoiding unused var. error + }) +} diff --git a/tests/dal/setup/mysql/setup.go b/tests/dal/setup/mysql/setup.go new file mode 100644 index 000000000..28c013231 --- /dev/null +++ b/tests/dal/setup/mysql/setup.go @@ -0,0 +1,84 @@ +package mysql + +import ( + "context" + "github.com/cortezaproject/corteza-server/pkg/dal" + "github.com/cortezaproject/corteza-server/pkg/logger" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms" + rdbmsDAL "github.com/cortezaproject/corteza-server/store/adapters/rdbms/dal" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/drivers/mysql" + "github.com/jmoiron/sqlx" + "strings" +) + +func Setup(dsn string) (_ dal.Connection, err error) { + var ( + ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) + cfg *rdbms.ConnConfig + db *sqlx.DB + ) + + cfg, err = mysql.NewConfig(dsn) + if err != nil { + return + } + + db, err = rdbms.Connect(ctx, logger.MakeDebugLogger(), cfg) + if err != nil { + return + } + + if err = tables(ctx, db); err != nil { + return + } + + return rdbmsDAL.Connection(db, mysql.Dialect()), nil +} + +// remove when store support for table creation is added to CRS +// +// When support for creating DDL commands (creating tables) from DAL models and attributes +// is added, this can be removed! +func tables(ctx context.Context, db sqlx.ExecerContext) (err error) { + return ddl.Exec(ctx, db, + `DROP TABLE IF EXISTS crs_test_codec`, + strings.ReplaceAll( + `CREATE TABLE IF NOT EXISTS crs_test_codec ( + "id" BIGINT UNSIGNED, + "created_at" DATETIME, + "updated_at" DATETIME, + "meta" JSON, + "pID" BIGINT UNSIGNED, + "pRef" BIGINT UNSIGNED, + "pTimestamp_TZT" DATETIME, + "pTimestamp_TZF" DATETIME, + "pTime" TIME, + "pDate" DATE, + "pNumber" NUMERIC, + "pText" TEXT, + "pBoolean_T" BOOLEAN, + "pBoolean_F" BOOLEAN, + "pEnum" TEXT, + "pGeometry" TEXT, + "pJSON" TEXT, + "pBlob" BLOB, + "pUUID" VARCHAR(36), + + PRIMARY KEY("id") + )`, "\"", "`"), + `DROP TABLE IF EXISTS crs_test_search`, + strings.ReplaceAll( + `CREATE TABLE IF NOT EXISTS crs_test_search ( + "id" INT NOT NULL, + "created_at" DATETIME, + "updated_at" DATETIME, + "meta" JSON, + "p_string" TEXT, + "p_number" NUMERIC, + "p_is_odd" BOOLEAN, + + PRIMARY KEY("id") + )`, "\"", "`"), + ) +} diff --git a/tests/dal/setup/postgres/setup.go b/tests/dal/setup/postgres/setup.go new file mode 100644 index 000000000..c53ed144a --- /dev/null +++ b/tests/dal/setup/postgres/setup.go @@ -0,0 +1,80 @@ +package postgres + +import ( + "context" + "github.com/cortezaproject/corteza-server/pkg/dal" + "github.com/cortezaproject/corteza-server/pkg/logger" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms" + rdbmsDAL "github.com/cortezaproject/corteza-server/store/adapters/rdbms/dal" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/drivers/postgres" + "github.com/jmoiron/sqlx" +) + +func Setup(dsn string) (_ dal.Connection, err error) { + var ( + ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) + cfg *rdbms.ConnConfig + db *sqlx.DB + ) + + cfg, err = postgres.NewConfig(dsn) + if err != nil { + return + } + + db, err = rdbms.Connect(ctx, logger.MakeDebugLogger(), cfg) + if err != nil { + return + } + + if err = tables(ctx, db); err != nil { + return + } + + return rdbmsDAL.Connection(db, postgres.Dialect()), nil +} + +// remove when store support for table creation is added to CRS +// +// When support for creating DDL commands (creating tables) from DAL models and attributes +// is added, this can be removed! +func tables(ctx context.Context, db sqlx.ExecerContext) (err error) { + return ddl.Exec(ctx, db, + `CREATE TEMPORARY TABLE IF NOT EXISTS crs_test_codec ( + "id" BIGINT NOT NULL, + "created_at" TIMESTAMP NOT NULL, + "updated_at" TIMESTAMP, + "meta" JSON, + "pID" BIGINT, + "pRef" BIGINT, + "pTimestamp_TZT" TIMESTAMPTZ, + "pTimestamp_TZF" TIMESTAMP, + "pTime" TIME, + "pDate" DATE, + "pNumber" NUMERIC, + "pText" TEXT, + "pBoolean_T" BOOLEAN, + "pBoolean_F" BOOLEAN, + "pEnum" TEXT, + "pGeometry" TEXT, + "pJSON" TEXT, + "pBlob" BYTEA, + "pUUID" UUID, + + PRIMARY KEY(id) + )`, + + `CREATE TEMPORARY TABLE IF NOT EXISTS crs_test_search ( + "id" BIGINT NOT NULL, + "created_at" TIMESTAMPTZ NOT NULL, + "updated_at" TIMESTAMPTZ, + "meta" JSON, + "p_string" TEXT, + "p_number" NUMERIC, + "p_is_odd" BOOLEAN, + + PRIMARY KEY(id ) + )`, + ) +} diff --git a/tests/dal/setup/sqlite/setup.go b/tests/dal/setup/sqlite/setup.go new file mode 100644 index 000000000..4cc131f90 --- /dev/null +++ b/tests/dal/setup/sqlite/setup.go @@ -0,0 +1,82 @@ +package sqlite + +import ( + "context" + "github.com/cortezaproject/corteza-server/pkg/dal" + "github.com/cortezaproject/corteza-server/pkg/logger" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms" + rdbmsDAL "github.com/cortezaproject/corteza-server/store/adapters/rdbms/dal" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/drivers/sqlite" + "github.com/jmoiron/sqlx" +) + +func Setup(dsn string) (_ dal.Connection, err error) { + var ( + ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) + cfg *rdbms.ConnConfig + db *sqlx.DB + ) + + cfg, err = sqlite.NewConfig(dsn) + if err != nil { + return + } + + db, err = rdbms.Connect(ctx, logger.MakeDebugLogger(), cfg) + if err != nil { + return + } + + if err = tables(ctx, db); err != nil { + return + } + + return rdbmsDAL.Connection(db, sqlite.Dialect()), nil +} + +// remove when store support for table creation is added to CRS +// +// When support for creating DDL commands (creating tables) from DAL models and attributes +// is added, this can be removed! +func tables(ctx context.Context, db sqlx.ExecerContext) (err error) { + return ddl.Exec(ctx, db, + `DROP TABLE IF EXISTS crs_test_codec`, + `CREATE TABLE IF NOT EXISTS crs_test_codec ( + id UNSIGNED BIG INT NOT NULL, + created_at TIMESTAMP, + updated_at TIMESTAMP, + meta JSON, + pID UNSIGNED BIG INT, + pRef UNSIGNED BIG INT, + pTimestamp_TZT TIMESTAMP, + pTimestamp_TZF TIMESTAMP, + pTime TIME, + pDate DATE, + pNumber NUMERIC, + pText TEXT, + pBoolean_T BOOLEAN, + pBoolean_F BOOLEAN, + pEnum TEXT, + pGeometry TEXT, + pJSON TEXT, + pBlob BLOB, + pUUID UUID, + + PRIMARY KEY(id) + )`, + + `DROP TABLE IF EXISTS crs_test_search`, + `CREATE TABLE IF NOT EXISTS crs_test_search ( + id INT NOT NULL, + created_at TIMESTAMP, + updated_at TIMESTAMP, + meta JSON, + p_string TEXT, + p_number NUMERIC, + p_is_odd BOOLEAN, + + PRIMARY KEY(id ) + )`, + ) +}