From 640a90c20cd5fb03c46db07b2ce61827e5e12dc7 Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Sat, 12 Mar 2022 10:12:30 +0100 Subject: [PATCH] Improve temporal value filtering --- compose/service/record_datasource.go | 1 + compose/types/module_field.go | 10 +++++- pkg/report/frame.go | 27 +++++++++++---- store/cockroach/sql_typecasters.go | 9 +++++ store/mysql/sql_typecasters.go | 8 +++++ store/postgres/sql_typecasters.go | 8 +++++ store/rdbms/compose_records.go | 8 +++++ store/rdbms/rdbms.go | 2 ++ store/tests/compose_records_test.go | 50 ++++++++++++++++++++++++++++ 9 files changed, 115 insertions(+), 8 deletions(-) diff --git a/compose/service/record_datasource.go b/compose/service/record_datasource.go index 7c26ad432..b7b02f480 100644 --- a/compose/service/record_datasource.go +++ b/compose/service/record_datasource.go @@ -94,6 +94,7 @@ func (svc record) Datasource(ctx context.Context, ld *report.LoadStepDefinition) c = report.MakeColumnOfKind(k) c.Name = f.Name c.Label = f.Label + c.Options = f.Options if c.Label == "" { c.Label = c.Name } diff --git a/compose/types/module_field.go b/compose/types/module_field.go index cfd5ef329..9e929e7a0 100644 --- a/compose/types/module_field.go +++ b/compose/types/module_field.go @@ -392,7 +392,15 @@ func (f ModuleField) IsNumeric() bool { } func (f ModuleField) IsDateTime() bool { - return f.Kind == "DateTime" + return f.Kind == "DateTime" && !f.IsDateOnly() && !f.IsTimeOnly() +} + +func (f ModuleField) IsDateOnly() bool { + return f.Kind == "DateTime" && f.Options.Bool("onlyDate") +} + +func (f ModuleField) IsTimeOnly() bool { + return f.Kind == "DateTime" && f.Options.Bool("onlyTime") } // IsRef tells us if value of this field be a reference to something diff --git a/pkg/report/frame.go b/pkg/report/frame.go index 3d021aabf..58c663407 100644 --- a/pkg/report/frame.go +++ b/pkg/report/frame.go @@ -14,6 +14,10 @@ import ( ) type ( + ColumnOptions interface { + Bool(key string) bool + } + Frame struct { Name string `json:"name"` Source string `json:"source"` @@ -45,12 +49,13 @@ type ( frameCellCaster func(in interface{}) (expr.TypedValue, error) FrameColumnSet []*FrameColumn FrameColumn struct { - Name string `json:"name"` - Label string `json:"label"` - Kind string `json:"kind"` - Primary bool `json:"primary"` - Unique bool `json:"unique"` - System bool `json:"system"` + Name string `json:"name"` + Label string `json:"label"` + Kind string `json:"kind"` + Primary bool `json:"primary"` + Unique bool `json:"unique"` + System bool `json:"system"` + Options ColumnOptions `json:"-"` Caster frameCellCaster `json:"-" yaml:"-"` } @@ -390,7 +395,15 @@ func (c *FrameColumn) IsNumeric() bool { } func (c *FrameColumn) IsDateTime() bool { - return c.Kind == "DateTime" + return c.Kind == "DateTime" && !c.IsTimeOnly() && !c.IsDateOnly() +} + +func (c *FrameColumn) IsTimeOnly() bool { + return c.Options != nil && c.Options.Bool("onlyDate") +} + +func (c *FrameColumn) IsDateOnly() bool { + return c.Options != nil && c.Options.Bool("onlyTime") } func (c *FrameColumn) IsRef() bool { diff --git a/store/cockroach/sql_typecasters.go b/store/cockroach/sql_typecasters.go index b90bc6fbb..79f1e01ac 100644 --- a/store/cockroach/sql_typecasters.go +++ b/store/cockroach/sql_typecasters.go @@ -2,6 +2,7 @@ package cockroach import ( "fmt" + "github.com/cortezaproject/corteza-server/store/rdbms" ) @@ -29,6 +30,14 @@ func fieldToColumnTypeCaster(field rdbms.ModuleFieldTypeDetector, ident string) tcp := "CAST(%s AS TIMESTAMP)" fc := fmt.Sprintf(fcp, ident) return fmt.Sprintf(tcp, fc), fcp, tcp, nil + case field.IsDateOnly(): + tcp := "CAST(%s AS DATE)" + fc := fmt.Sprintf(fcp, ident) + return fmt.Sprintf(tcp, fc), fcp, tcp, nil + case field.IsTimeOnly(): + tcp := "CAST(%s AS TIME)" + fc := fmt.Sprintf(fcp, ident) + return fmt.Sprintf(tcp, fc), fcp, tcp, nil case field.IsRef(): tcp := "%s" fc := fmt.Sprintf(fcpRef, ident) diff --git a/store/mysql/sql_typecasters.go b/store/mysql/sql_typecasters.go index ed8e0692b..ae8e9566f 100644 --- a/store/mysql/sql_typecasters.go +++ b/store/mysql/sql_typecasters.go @@ -30,6 +30,14 @@ func fieldToColumnTypeCaster(field rdbms.ModuleFieldTypeDetector, ident string) tcp := "CAST(%s AS DATETIME)" fc := fmt.Sprintf(fcp, ident) return fmt.Sprintf(tcp, fc), fcp, tcp, nil + case field.IsDateOnly(): + tcp := "CAST(%s AS DATE)" + fc := fmt.Sprintf(fcp, ident) + return fmt.Sprintf(tcp, fc), fcp, tcp, nil + case field.IsTimeOnly(): + tcp := "CAST(%s AS TIME)" + fc := fmt.Sprintf(fcp, ident) + return fmt.Sprintf(tcp, fc), fcp, tcp, nil case field.IsRef(): tcp := "%s" fc := fmt.Sprintf(fcpRef, ident) diff --git a/store/postgres/sql_typecasters.go b/store/postgres/sql_typecasters.go index a251cebcb..c8f7a419f 100644 --- a/store/postgres/sql_typecasters.go +++ b/store/postgres/sql_typecasters.go @@ -30,6 +30,14 @@ func fieldToColumnTypeCaster(field rdbms.ModuleFieldTypeDetector, ident string) tcp := "to_timestamp(NULLIF(%s, ''),'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"') " fc := fmt.Sprintf(fcp, ident) return fmt.Sprintf(tcp, fc), fcp, tcp, nil + case field.IsDateOnly(): + tcp := "CAST(NULLIF(%s, '') AS DATE)" + fc := fmt.Sprintf(fcp, ident) + return fmt.Sprintf(tcp, fc), fcp, tcp, nil + case field.IsTimeOnly(): + tcp := "CAST(NULLIF(%s, '') AS TIME)" + fc := fmt.Sprintf(fcp, ident) + return fmt.Sprintf(tcp, fc), fcp, tcp, nil case field.IsRef(): tcp := "%s" fc := fmt.Sprintf(fcpRef, ident) diff --git a/store/rdbms/compose_records.go b/store/rdbms/compose_records.go index 67ddbb5c0..727101f4b 100644 --- a/store/rdbms/compose_records.go +++ b/store/rdbms/compose_records.go @@ -25,6 +25,8 @@ type ( boolean bool numeric bool dateTime bool + dateOnly bool + timeOnly bool ref bool } ) @@ -778,6 +780,12 @@ func (t mftd) IsNumeric() bool { func (t mftd) IsDateTime() bool { return t.dateTime } +func (t mftd) IsDateOnly() bool { + return t.dateOnly +} +func (t mftd) IsTimeOnly() bool { + return t.timeOnly +} func (t mftd) IsRef() bool { return t.ref } diff --git a/store/rdbms/rdbms.go b/store/rdbms/rdbms.go index 842a36cad..b9a945f80 100644 --- a/store/rdbms/rdbms.go +++ b/store/rdbms/rdbms.go @@ -54,6 +54,8 @@ type ( IsBoolean() bool IsNumeric() bool IsDateTime() bool + IsTimeOnly() bool + IsDateOnly() bool IsRef() bool } diff --git a/store/tests/compose_records_test.go b/store/tests/compose_records_test.go index 78bcd6e31..4e3378dbc 100644 --- a/store/tests/compose_records_test.go +++ b/store/tests/compose_records_test.go @@ -44,6 +44,8 @@ func testComposeRecords(t *testing.T, s store.ComposeRecords) { &types.ModuleField{Kind: "DateTime", Name: "datetime1"}, &types.ModuleField{Kind: "DateTime", Name: "datetime2"}, &types.ModuleField{Kind: "DateTime", Name: "datetime3"}, + &types.ModuleField{Kind: "DateTime", Name: "date1", Options: map[string]interface{}{"onlyDate": true}}, + &types.ModuleField{Kind: "DateTime", Name: "time1", Options: map[string]interface{}{"onlyTime": true}}, &types.ModuleField{Kind: "Email", Name: "email1"}, &types.ModuleField{Kind: "Email", Name: "email2"}, @@ -1635,4 +1637,52 @@ func testComposeRecords(t *testing.T, s store.ComposeRecords) { req.Equal("1st,1;2nd,22;3rd,3", stringifyValues(set, "str1", "num1")) }) + + t.Run("date-time filtering", func(t *testing.T) { + var ( + err error + set types.RecordSet + + req, _ = truncAndCreate(t, + makeNew(&types.RecordValue{Name: "num1", Value: "1001"}, &types.RecordValue{Name: "datetime1", Value: "2020-10-01T00:00:01"}), + makeNew(&types.RecordValue{Name: "num1", Value: "1002"}, &types.RecordValue{Name: "datetime1", Value: "2020-10-02T00:00:02"}), + makeNew(&types.RecordValue{Name: "num1", Value: "1003"}, &types.RecordValue{Name: "datetime1", Value: "2020-10-03T00:00:03"}), + makeNew(&types.RecordValue{Name: "num1", Value: "1004"}, &types.RecordValue{Name: "datetime1", Value: "2020-10-04T00:00:03"}), + + makeNew(&types.RecordValue{Name: "num1", Value: "2001"}, &types.RecordValue{Name: "date1", Value: "2020-10-01"}), + makeNew(&types.RecordValue{Name: "num1", Value: "2002"}, &types.RecordValue{Name: "date1", Value: "2020-10-02"}), + makeNew(&types.RecordValue{Name: "num1", Value: "2003"}, &types.RecordValue{Name: "date1", Value: "2020-10-03"}), + makeNew(&types.RecordValue{Name: "num1", Value: "2004"}, &types.RecordValue{Name: "date1", Value: "2020-10-04"}), + + makeNew(&types.RecordValue{Name: "num1", Value: "3001"}, &types.RecordValue{Name: "time1", Value: "01:00:00"}), + makeNew(&types.RecordValue{Name: "num1", Value: "3002"}, &types.RecordValue{Name: "time1", Value: "02:00:00"}), + makeNew(&types.RecordValue{Name: "num1", Value: "3003"}, &types.RecordValue{Name: "time1", Value: "03:00:00"}), + makeNew(&types.RecordValue{Name: "num1", Value: "3004"}, &types.RecordValue{Name: "time1", Value: "04:00:00"}), + ) + + cases = []struct { + query string + result string + }{ + {"datetime1 = '2020-10-02T00:00:02'", "1002"}, + {"date1 = '2020-10-02'", "2002"}, + {"time1 = '02:00:00'", "3002"}, + {"datetime1 <= '2020-10-02T00:00:02'", "1001;1002"}, + {"date1 <= '2020-10-02'", "2001;2002"}, + {"time1 <= '02:00:00'", "3001;3002"}, + {"datetime1 > '2020-10-02T00:00:02'", "1003;1004"}, + {"date1 > '2020-10-02'", "2003;2004"}, + {"time1 > '02:00:00'", "3003;3004"}, + } + ) + + for _, c := range cases { + t.Run(c.query, func(t *testing.T) { + req = require.New(t) + set, _, err = s.SearchComposeRecords(ctx, mod, types.RecordFilter{Query: c.query}) + req.NoError(err) + req.Equal(c.result, stringifyValues(set, "num1")) + }) + } + }) }