From c1796529ebecc9d1f699a8ed4b636b4433c76b61 Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Tue, 10 May 2022 12:56:31 +0200 Subject: [PATCH] Refactor types, add support for sortable --- compose/crs/test/driver.go | 12 +- pkg/data/model.go | 245 ++------------------------ pkg/data/types.go | 153 ++++++++++++++++ store/adapters/rdbms/crs/model.go | 84 ++++----- store/adapters/rdbms/drivers/table.go | 3 +- 5 files changed, 207 insertions(+), 290 deletions(-) create mode 100644 pkg/data/types.go diff --git a/compose/crs/test/driver.go b/compose/crs/test/driver.go index 7a8af9cbb..f43d0afcc 100644 --- a/compose/crs/test/driver.go +++ b/compose/crs/test/driver.go @@ -30,9 +30,9 @@ func RecordCodec(t *testing.T, d crs.StoreConnection) { m = &data.Model{ Ident: "crs_test_codec", Attributes: data.AttributeSet{ - &data.Attribute{Ident: data.SysID, Type: &data.TypeID{}, Store: &data.StoreCodecAlias{Ident: "id"}, PrimaryKey: true}, - &data.Attribute{Ident: data.SysCreatedAt, Type: &data.TypeTimestamp{}, Store: &data.StoreCodecAlias{Ident: "created_at"}}, - &data.Attribute{Ident: data.SysUpdatedAt, Type: &data.TypeTimestamp{}, Store: &data.StoreCodecAlias{Ident: "updated_at"}}, + &data.Attribute{Ident: "ID", Type: &data.TypeID{}, Store: &data.StoreCodecAlias{Ident: "id"}, PrimaryKey: true}, + &data.Attribute{Ident: "createdAt", Type: &data.TypeTimestamp{}, Store: &data.StoreCodecAlias{Ident: "created_at"}}, + &data.Attribute{Ident: "updatedAt", Type: &data.TypeTimestamp{}, Store: &data.StoreCodecAlias{Ident: "updated_at"}}, &data.Attribute{Ident: "vID", Type: &data.TypeID{}, Store: &data.StoreCodecStdRecordValueJSON{Ident: "meta"}}, &data.Attribute{Ident: "vRef", Type: &data.TypeRef{}, Store: &data.StoreCodecStdRecordValueJSON{Ident: "meta"}}, @@ -149,9 +149,9 @@ func RecordSearch(t *testing.T, d crs.StoreConnection) { m = &data.Model{ Ident: "crs_test_search", Attributes: data.AttributeSet{ - &data.Attribute{Ident: data.SysID, Type: &data.TypeID{}, Store: &data.StoreCodecAlias{Ident: "id"}, PrimaryKey: true}, - &data.Attribute{Ident: data.SysCreatedAt, Type: &data.TypeTimestamp{}, Store: &data.StoreCodecAlias{Ident: "created_at"}}, - &data.Attribute{Ident: data.SysUpdatedAt, Type: &data.TypeTimestamp{}, Store: &data.StoreCodecAlias{Ident: "updated_at"}}, + &data.Attribute{Ident: "ID", Type: &data.TypeID{}, Store: &data.StoreCodecAlias{Ident: "id"}, PrimaryKey: true}, + &data.Attribute{Ident: "createdAt", Type: &data.TypeTimestamp{}, Store: &data.StoreCodecAlias{Ident: "created_at"}}, + &data.Attribute{Ident: "updatedAt", Type: &data.TypeTimestamp{}, Store: &data.StoreCodecAlias{Ident: "updated_at"}}, &data.Attribute{Ident: "v_string", Type: &data.TypeText{}, Store: &data.StoreCodecStdRecordValueJSON{Ident: "meta"}}, &data.Attribute{Ident: "v_number", Type: &data.TypeNumber{}, Store: &data.StoreCodecStdRecordValueJSON{Ident: "meta"}}, diff --git a/pkg/data/model.go b/pkg/data/model.go index 9627560a4..1521b19ed 100644 --- a/pkg/data/model.go +++ b/pkg/data/model.go @@ -27,174 +27,25 @@ type ( MultiValue bool - PrimaryKey bool + PrimaryKey bool + + // If attribute has SoftDeleteFlag=true we use it + // when filtering out deleted items SoftDeleteFlag bool - Sortable bool + + // Is column sortable? + // Note: all primary keys are sortable + Sortable bool // Store describes the strategy the underlying storage system should // apply to the underlying value Store StoreCodec // Type describes what the value represents and how it should be // encoded/decoded - Type attributeType + Type Type } + AttributeSet []*Attribute - - AttributeType string - - // temp - Type interface { - attributeType - } - - attributeType interface { - Type() AttributeType - } - - // TypeID handles ID (uint64) coding - // - // Encoding/decoding might be different depending on - // 1) underlying store (and dialect) - // 2) value codec (raw, json ...) - TypeID struct { - // @todo need to figure out how to support when IDs - // generated/provided by store (SERIAL/AUTOINCREMENT) - GeneratedByStore bool - Nullable bool - } - - // TypeRef handles ID (uint64) coding + reference info - // - // Encoding/decoding might be different depending on - // 1) underlying store (and dialect) - // 2) value codec (raw, json ...) - TypeRef struct { - RefModel *Model - RefAttribute *Attribute - Nullable bool - } - - // TypeTimestamp handles timestamp coding - // - // Encoding/decoding might be different depending on - // 1) underlying store (and dialect) - // 2) value codec (raw, json ...) - TypeTimestamp struct { - Timezone bool - Precision uint - Nullable bool - } - - // TypeTime handles time coding - // - // Encoding/decoding might be different depending on - // 1) underlying store (and dialect) - // 2) value codec (raw, json ...) - TypeTime struct { - Timezone bool - Precision uint - Nullable bool - } - - // TypeDate handles date coding - // - // Encoding/decoding might be different depending on - // 1) underlying store (and dialect) - // 2) value codec (raw, json ...) - TypeDate struct { - // - Nullable bool - } - - // TypeNumber handles number coding - // - // Encoding/decoding might be different depending on - // 1) underlying store (and dialect) - // 2) value codec (raw, json ...) - TypeNumber struct { - Precision uint - Scale uint - Nullable bool - } - - // TypeText handles string coding - // - // Encoding/decoding might be different depending on - // 1) underlying store (and dialect) - // 2) value codec (raw, json ...) - TypeText struct { - Length uint - Nullable bool - } - - // TypeBoolean - TypeBoolean struct { - // - Nullable bool - } - - // TypeEnum - TypeEnum struct { - Values []string - Nullable bool - } - - // TypeGeometry - TypeGeometry struct { - // - Nullable bool - } - - // TypeJSON handles coding of arbitrary data into JSON structure - // NOT TO BE CONFUSED with encodedField - // - // Encoding/decoding might be different depending on - // 1) underlying store (and dialect) - // 2) value codec (raw, json ...) - TypeJSON struct { - // - Nullable bool - } - - // TypeBlob store/return data as - TypeBlob struct { - // - Nullable bool - } - - TypeUUID struct { - // - Nullable bool - } -) - -const ( - typeID AttributeType = "id" - typeRef AttributeType = "ref" - typeTimestamp AttributeType = "timestamp" - typeTime AttributeType = "time" - typeDate AttributeType = "date" - typeNumber AttributeType = "number" - typeText AttributeType = "text" - typeBoolean AttributeType = "boolean" - typeEnum AttributeType = "enum" - typeGeometry AttributeType = "geometry" - typeJSON AttributeType = "json" - typeBlob AttributeType = "blob" - typeUUID AttributeType = "uuid" -) - -const ( - SysID = "ID" - SysNamespaceID = "namespaceID" - SysModuleID = "moduleID" - SysCreatedAt = "createdAt" - SysCreatedBy = "createdBy" - SysUpdatedAt = "updatedAt" - SysUpdatedBy = "updatedBy" - SysDeletedAt = "deletedAt" - SysDeletedBy = "deletedBy" - SysOwnedBy = "ownedBy" ) // FindByIdent returns the model that matches the ident @@ -217,7 +68,7 @@ func (aa ModelSet) FilterByReferenced(b *Model) (out ModelSet) { for _, aAttribute := range aModel.Attributes { switch casted := aAttribute.Type.(type) { - case TypeRef: + case *TypeRef: if casted.RefModel.Ident == b.Ident { out = append(out, aModel) } @@ -271,77 +122,3 @@ func (m Model) Validate() error { return nil } - -//// AttributeGroups returns attributes grouped by ident -//func (m Model) AttributeGroups() (gg map[string][]*Attribute) { -// gg = make(map[string][]*Attribute) -// -// for _, attr := range m.Attributes { -// // @todo properly check attribute integrity: -// // is embeddable -// // only same types use the same ident -// if gIdent, embeddable := attr.Store.Embeddable(); !embeddable && len(gg[attr.Ident]) > 0 { -// panic("attribute " + attr.Ident + " is not embeddable") -// } else if len(gIdent) > 0 { -// gg[gIdent] = append(gg[gIdent], attr) -// } else { -// gg[attr.Ident] = append(gg[attr.Ident], attr) -// } -// } -// -// return -//} - -// Receivers to conform to the interface - -func (t TypeID) Type() AttributeType { - return typeID -} - -func (t TypeRef) Type() AttributeType { - return typeRef -} - -func (t TypeTimestamp) Type() AttributeType { - return typeTimestamp -} - -func (t TypeTime) Type() AttributeType { - return typeTime -} - -func (t TypeDate) Type() AttributeType { - return typeDate -} - -func (t TypeNumber) Type() AttributeType { - return typeNumber -} - -func (t TypeText) Type() AttributeType { - return typeText -} - -func (t TypeBoolean) Type() AttributeType { - return typeBoolean -} - -func (t TypeEnum) Type() AttributeType { - return typeEnum -} - -func (t TypeGeometry) Type() AttributeType { - return typeGeometry -} - -func (t TypeJSON) Type() AttributeType { - return typeJSON -} - -func (t TypeBlob) Type() AttributeType { - return typeBlob -} - -func (t TypeUUID) Type() AttributeType { - return typeUUID -} diff --git a/pkg/data/types.go b/pkg/data/types.go new file mode 100644 index 000000000..c217e2ae9 --- /dev/null +++ b/pkg/data/types.go @@ -0,0 +1,153 @@ +package data + +type ( + // temp + Type interface { + Type() string + IsNullable() bool + } + + // TypeID handles ID (uint64) coding + // + // Encoding/decoding might be different depending on + // 1) underlying store (and dialect) + // 2) value codec (raw, json ...) + TypeID struct { + // @todo need to figure out how to support when IDs + // generated/provided by store (SERIAL/AUTOINCREMENT) + GeneratedByStore bool + Nullable bool + } + + // TypeRef handles ID (uint64) coding + reference info + // + // Encoding/decoding might be different depending on + // 1) underlying store (and dialect) + // 2) value codec (raw, json ...) + TypeRef struct { + RefModel *Model + RefAttribute *Attribute + Nullable bool + } + + // TypeTimestamp handles timestamp coding + // + // Encoding/decoding might be different depending on + // 1) underlying store (and dialect) + // 2) value codec (raw, json ...) + TypeTimestamp struct { + Timezone bool + Precision uint + Nullable bool + } + + // TypeTime handles time coding + // + // Encoding/decoding might be different depending on + // 1) underlying store (and dialect) + // 2) value codec (raw, json ...) + TypeTime struct { + Timezone bool + Precision uint + Nullable bool + } + + // TypeDate handles date coding + // + // Encoding/decoding might be different depending on + // 1) underlying store (and dialect) + // 2) value codec (raw, json ...) + TypeDate struct { + // + Nullable bool + } + + // TypeNumber handles number coding + // + // Encoding/decoding might be different depending on + // 1) underlying store (and dialect) + // 2) value codec (raw, json ...) + TypeNumber struct { + Precision uint + Scale uint + Nullable bool + } + + // TypeText handles string coding + // + // Encoding/decoding might be different depending on + // 1) underlying store (and dialect) + // 2) value codec (raw, json ...) + TypeText struct { + Length uint + Nullable bool + } + + // TypeBoolean + TypeBoolean struct { + // + Nullable bool + } + + // TypeEnum + TypeEnum struct { + Values []string + Nullable bool + } + + // TypeGeometry + TypeGeometry struct { + // + Nullable bool + } + + // TypeJSON handles coding of arbitrary data into JSON structure + // NOT TO BE CONFUSED with encodedField + // + // Encoding/decoding might be different depending on + // 1) underlying store (and dialect) + // 2) value codec (raw, json ...) + TypeJSON struct { + // + Nullable bool + } + + // TypeBlob store/return data as + TypeBlob struct { + // + Nullable bool + } + + TypeUUID struct { + // + Nullable bool + } +) + +func (t TypeID) IsNullable() bool { return t.Nullable } +func (t TypeRef) IsNullable() bool { return t.Nullable } +func (t TypeTimestamp) IsNullable() bool { return t.Nullable } +func (t TypeTime) IsNullable() bool { return t.Nullable } +func (t TypeDate) IsNullable() bool { return t.Nullable } +func (t TypeNumber) IsNullable() bool { return t.Nullable } +func (t TypeText) IsNullable() bool { return t.Nullable } +func (t TypeBoolean) IsNullable() bool { return t.Nullable } +func (t TypeEnum) IsNullable() bool { return t.Nullable } +func (t TypeGeometry) IsNullable() bool { return t.Nullable } +func (t TypeJSON) IsNullable() bool { return t.Nullable } +func (t TypeBlob) IsNullable() bool { return t.Nullable } +func (t TypeUUID) IsNullable() bool { return t.Nullable } + +func (t TypeID) Type() string { return "ID" } +func (t TypeRef) Type() string { return "REF" } +func (t TypeTimestamp) Type() string { return "TIMESTAMP" } +func (t TypeTime) Type() string { return "TIME" } +func (t TypeDate) Type() string { return "DATE" } +func (t TypeNumber) Type() string { return "NUMBER" } +func (t TypeText) Type() string { return "TEXT" } +func (t TypeBoolean) Type() string { return "BOOLEAN" } +func (t TypeEnum) Type() string { return "ENUM" } +func (t TypeGeometry) Type() string { return "GEOMETRY" } +func (t TypeJSON) Type() string { return "JSON" } +func (t TypeBlob) Type() string { return "BLOB" } +func (t TypeUUID) Type() string { return "UUID" } diff --git a/store/adapters/rdbms/crs/model.go b/store/adapters/rdbms/crs/model.go index 3e9aa9634..d3e70bf5f 100644 --- a/store/adapters/rdbms/crs/model.go +++ b/store/adapters/rdbms/crs/model.go @@ -9,6 +9,7 @@ import ( "github.com/cortezaproject/corteza-server/compose/crs" "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/data" + "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/store/adapters/rdbms/drivers" "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ql" "github.com/doug-martin/goqu/v9" @@ -26,25 +27,6 @@ type ( sqlx.ExecerContext } - attrExpression interface { - exp.Comparable - exp.Inable - exp.Isable - } - - attributeType interface { - Type() data.AttributeType - } - - column struct { - ident string - columnType attributeType - attributes []*data.Attribute - - encode func(drivers.Dialect, []*data.Attribute, crs.ValueGetter) (any, error) - decode func(drivers.Dialect, []*data.Attribute, any, crs.ValueSetter) error - } - queryParser interface { Parse(string) (exp.Expression, error) } @@ -57,22 +39,6 @@ type ( dialect drivers.Dialect table drivers.TableCodec - - // ID column identifier expression - //sysColumnID exp.IdentifierExpression - //sysPrimaryKeyAttr string - - // optional record fields/columns/expressions - //sysExprNamespaceID attrExpression - //sysSoftDeleteAttr string - //sysExprModuleID attrExpression - //sysModuleAttr string - //sysExprDeletedAt attrExpression - //sysNamespaceAttr string - - // all columns we're selecting from when - // we're selecting from all columns - //columns []*column } ) @@ -168,6 +134,10 @@ func (d *model) Search(f types.RecordFilter) (i *iterator, err error) { continue } + if !c.Attribute().PrimaryKey && !c.Attribute().Sortable { + return nil, fmt.Errorf("can not sort by %q; not sortable, not primary key", attrIdent) + } + // Make sure results are always sorted at least by primary key f.AppendOrderBy(attrIdent, f.Sort.LastDescending()) } @@ -255,20 +225,36 @@ func (d *model) searchSql(f types.RecordFilter) *goqu.SelectDataset { //} } - { - // If model supports soft-deletion (= delete-at attribute is present) - // we need to make sure we respect it - //if d.sysExprDeletedAt != nil { - // switch f.Deleted { - // case filter.StateExclusive: - // // only not-null values - // cnd = append(cnd, d.sysExprDeletedAt.IsNotNull()) - // - // case filter.StateExcluded: - // // exclude all non-null values - // cnd = append(cnd, d.sysExprDeletedAt.IsNull()) - // } - //} + if f.Deleted != filter.StateInclusive { + //If model supports soft-deletion (= delete-at attribute is present) + //we need to make sure we respect it + var attrIdent exp.LiteralExpression + for _, attr := range d.model.Attributes { + if !attr.SoftDeleteFlag { + continue + } + + if !attr.Type.IsNullable() { + // @todo this must be checked much earlier + return base.SetError(fmt.Errorf("can not use non-nullable attribute %q soft-deleting", attr.Ident)) + } + + attrIdent, err = d.table.AttributeExpression(attr.Ident) + if err != nil { + return base.SetError(err) + } + + switch f.Deleted { + case filter.StateExclusive: + // only not-null values + cnd = append(cnd, attrIdent.IsNotNull()) + + case filter.StateExcluded: + // exclude all non-null values + cnd = append(cnd, attrIdent.IsNull()) + } + + } } if len(strings.TrimSpace(f.Query)) > 0 { diff --git a/store/adapters/rdbms/drivers/table.go b/store/adapters/rdbms/drivers/table.go index 41649ee40..f0817b25b 100644 --- a/store/adapters/rdbms/drivers/table.go +++ b/store/adapters/rdbms/drivers/table.go @@ -9,6 +9,7 @@ import ( ) type ( + // TableCodec is an RDBMS representation of data.Model structure and its arguments TableCodec interface { Columns() []Column Ident() exp.IdentifierExpression @@ -18,7 +19,7 @@ type ( AttributeExpression(string) (exp.LiteralExpression, error) } - // translates values from and to native db types + // GenericTableCodec is a generic implementation of TableCodec GenericTableCodec struct { // table identifier (name) ident exp.IdentifierExpression