Null values on datetime fields

server/automation/types/expr_test.go

@@ -2,14 +2,17 @@ package types
 
 import (
 	"context"
+	"testing"
+	"time"
+
 	. "github.com/cortezaproject/corteza/server/pkg/expr"
 	"github.com/stretchr/testify/require"
-	"testing"
 )
 
 func TestExprSet_Eval(t *testing.T) {
 	var (
 		ctx = context.Background()
+		tt  = time.Date(1999, 9, 9, 9, 9, 9, 9, time.UTC)
 
 		cc = []struct {
 			name   string
@@ -25,8 +28,15 @@ func TestExprSet_Eval(t *testing.T) {
 			},
 			{
 				name:   "constant assignment",
-				set:    ExprSet{&Expr{Target: "foo", Expr: `"bar"`}},
-				output: map[string]interface{}{"foo": Must(NewString("bar"))},
+				set:    ExprSet{&Expr{Target: "foo", Expr: `format("%s", bar)`, typ: &String{}}},
+				input:  map[string]interface{}{"bar": Must(NewDateTime(tt))},
+				output: map[string]interface{}{"foo": Must(NewString("1999-09-09 09:09:09.000000009 +0000 UTC"))},
+			},
+			{
+				name:   "constant assignment nil datetime",
+				set:    ExprSet{&Expr{Target: "foo", Expr: `format("%s", bar)`, typ: &String{}}},
+				input:  map[string]interface{}{"bar": Must(NewDateTime(nil))},
+				output: map[string]interface{}{"foo": Must(NewString("<nil>"))},
 			},
 			{
 				name:   "vars with path",

server/compose/automation/expr_types.go

@@ -392,6 +392,10 @@ func assignToComposeRecordValues(res *types.Record, p expr.Pather, val interface
 	case time.Time:
 		rv.Value = utval.Format(time.RFC3339)
 	case *time.Time:
+		if utval == nil {
+			rv.Value = ""
+			break
+		}
 		rv.Value = utval.Format(time.RFC3339)
 	case []string:
 		aux := make([]interface{}, len(utval))

server/compose/automation/expr_types_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"testing"
+	"time"
 
 	aTypes "github.com/cortezaproject/corteza/server/automation/types"
 	"github.com/cortezaproject/corteza/server/compose/types"
@@ -378,9 +379,12 @@ func TestCastToComposeRecordValues(t *testing.T) {
 			// &types.RecordValue{Name: "bools", Value: "false", Place: 1}
 		}
 
+		tt = time.Date(1999, 9, 9, 9, 9, 9, 9, time.UTC)
+
 		nilSlice      []int
 		nilUntypedMap map[string]interface{}
-		cases         = []struct {
+
+		cases = []struct {
 			name string
 			in   interface{}
 			out  types.RecordValueSet
@@ -421,6 +425,12 @@ func TestCastToComposeRecordValues(t *testing.T) {
 					&types.RecordValue{Name: "string", Value: "val"},
 				},
 			},
+			{
+				in: &types.RecordValue{Name: "datetime", Value: tt.String()},
+				out: types.RecordValueSet{
+					&types.RecordValue{Name: "datetime", Value: tt.String()},
+				},
+			},
 			{
 				in:  commonRVS,
 				out: commonRVS,

server/compose/types/record_value.go

@@ -4,11 +4,12 @@ import (
 	"database/sql/driver"
 	"encoding/json"
 	"fmt"
-	"github.com/cortezaproject/corteza/server/pkg/sql"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/cortezaproject/corteza/server/pkg/sql"
+
 	"github.com/cortezaproject/corteza/server/pkg/expr"
 	"github.com/cortezaproject/corteza/server/pkg/filter"
 	"github.com/spf13/cast"
@@ -73,6 +74,9 @@ func (v RecordValue) Cast(f *ModuleField) (interface{}, error) {
 		return v.Ref, nil
 
 	case f.IsDateTime():
+		if v.Value == "" {
+			return nil, nil
+		}
 		return cast.ToTimeE(v.Value)
 
 	case f.IsBoolean():
@@ -256,7 +260,6 @@ func (set RecordValueSet) Merge(mfs ModuleFieldSet, new RecordValueSet, canAcces
 //
 // This satisfies current requirements where record values are always
 // manipulated as a whole (not partial)
-//
 func (set RecordValueSet) merge(new RecordValueSet) (out RecordValueSet) {
 	if len(set) == 0 {
 		// Empty set, copy all new values and return them

server/pkg/expr/expr_types.go

@@ -503,11 +503,14 @@ func CastToDuration(val interface{}) (out time.Duration, err error) {
 
 func CastToDateTime(val interface{}) (out *time.Time, err error) {
 	val = UntypedValue(val)
+
 	switch casted := val.(type) {
 	case *time.Time:
 		return casted, nil
 	case time.Time:
 		return &casted, nil
+	case nil:
+		return nil, nil
 	default:
 		var c time.Time
 		if c, err = cast.ToTimeE(casted); err != nil {
@@ -959,6 +962,10 @@ func (v *Any) Clone() (out TypedValue, err error) {
 	return aux, err
 }
 
+func (t *Array) IsEmpty() bool {
+	return len(t.GetValue()) == 0
+}
+
 func (v *Array) Clone() (out TypedValue, err error) {
 	if len(v.value) > cloneParallelItemThreshold {
 		return v.cloneParallel(cloneParallelItemThreshold)
@@ -1024,6 +1031,10 @@ func (v *Boolean) Clone() (out TypedValue, err error) {
 	return aux, err
 }
 
+func (t *Bytes) IsEmpty() bool {
+	return len(t.GetValue()) == 0
+}
+
 func (v *Bytes) Clone() (out TypedValue, err error) {
 	cpy := make([]byte, len(v.value))
 	copy(cpy, v.value)
@@ -1033,10 +1044,11 @@ func (v *Bytes) Clone() (out TypedValue, err error) {
 }
 
 func (v *DateTime) Clone() (out TypedValue, err error) {
-	t := *v.GetValue()
+	if v.value == nil {
+		return NewDateTime(nil)
+	}
 
-	aux, err := NewDateTime(&t)
-	return aux, err
+	return NewDateTime(*v.value)
 }
 
 func (v *Duration) Clone() (out TypedValue, err error) {
@@ -1128,3 +1140,7 @@ func (v *UnsignedInteger) Clone() (out TypedValue, err error) {
 func (v Unresolved) Clone() (out TypedValue, err error) {
 	return nil, fmt.Errorf("cannot unref unresolved type")
 }
+
+func (v DateTime) IsEmpty() bool {
+	return v.GetValue() == nil
+}

server/pkg/expr/func_generic.go

@@ -47,6 +47,15 @@ func length(i interface{}) int {
 }
 
 func isNil(i interface{}) bool {
+	_, isTyped := i.(TypedValue)
+
+	if isTyped {
+		switch i.(type) {
+		case *Duration, *DateTime, *Bytes, *Array, *Integer:
+			i = i.(TypedValue).Get()
+		}
+	}
+
 	return gvalfnc.IsNil(i)
 }
 
@@ -115,8 +124,6 @@ func isMap(v interface{}) bool {
 
 // toArray removes expr types (if wrapped) and checks if the variable is slice
 // internal only
-//
-//
 func toSlice(vv interface{}) (interface{}, error) {
 	vv = UntypedValue(vv)
 

server/pkg/expr/func_generic_test.go

@@ -2,11 +2,12 @@ package expr
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 )
 
-func Test_empty(t *testing.T) {
+func Test_isEmpty(t *testing.T) {
 	var (
 		req              = require.New(t)
 		unsetSliceString []string
@@ -16,75 +17,276 @@ func Test_empty(t *testing.T) {
 		unsetString      string
 		unsetInt64       int64
 
+		unsetDateTime *time.Time
+		unsetDuration time.Duration
+
 		tcc = []struct {
 			value  interface{}
 			expect interface{}
+			sc     string
 		}{
 			{
 				value:  []string{},
 				expect: true,
+				sc:     "empty string slice",
 			},
 			{
 				value:  map[string]string{},
 				expect: true,
+				sc:     "empty map of strings",
 			},
 			{
 				value:  unsetSliceString,
 				expect: true,
+				sc:     "undefined string slice",
 			},
 			{
 				value:  []int{},
 				expect: true,
+				sc:     "empty int slice",
 			},
 			{
 				value:  []int{1},
 				expect: false,
+				sc:     "1-elem int slice",
 			},
 			{
 				value:  unsetSliceInt,
 				expect: true,
+				sc:     "undefined int slice",
 			},
 			{
 				value:  unsetSliceBool,
 				expect: true,
+				sc:     "undefined slice bool",
 			},
 			{
 				value:  int(1),
 				expect: false,
+				sc:     "defined int",
 			},
 			{
 				value:  int(0),
 				expect: true,
+				sc:     "defined int 0",
 			},
 			{
 				value:  "",
 				expect: true,
+				sc:     "emty string",
 			},
 			{
 				value:  unsetString,
 				expect: true,
+				sc:     "undefined string",
 			},
 			{
 				value:  unsetSliceFloat,
 				expect: true,
+				sc:     "undefined slice float",
 			},
 			{
 				value:  unsetInt64,
 				expect: true,
+				sc:     "undefined slice int64",
 			},
 			{
 				value:  []float32{11.1},
 				expect: false,
+				sc:     "non-empty slice float32",
 			},
 			{
 				value:  []float32{},
 				expect: true,
+				sc:     "empty slice float32",
+			},
+			{
+				value:  unsetDateTime,
+				expect: true,
+				sc:     "undefined datetime",
+			},
+			{
+				value:  Must(NewInteger(nil)),
+				expect: false,
+				sc:     "undefined Integer expr",
+			},
+			{
+				value:  Must(NewDateTime(nil)),
+				expect: true,
+				sc:     "undefined DateTime expr",
+			},
+			{
+				value:  Must(NewDateTime(unsetDateTime)),
+				expect: true,
+				sc:     "undefined DateTime expr",
+			},
+			{
+				value:  Must(NewArray([]string{"A"})),
+				expect: false,
+				sc:     "non empty Array expr",
+			},
+			{
+				value:  Must(NewBoolean(nil)),
+				expect: false,
+				sc:     "undefined Boolean expr",
+			},
+			{
+				value:  Must(NewBytes(nil)),
+				expect: true,
+				sc:     "undefined Bytes expr",
+			},
+			{
+				value:  Must(NewDuration(unsetDuration)),
+				expect: false,
+				sc:     "undefined Duration expr",
 			},
 		}
 	)
 
 	for _, tst := range tcc {
-		req.Equal(tst.expect, isEmpty(tst.value))
+		req.Equal(tst.expect, isEmpty(tst.value), "Failed isEmpty test: %s", tst.sc)
+	}
+}
+func Test_isNil(t *testing.T) {
+	var (
+		req              = require.New(t)
+		unsetSliceString []string
+		unsetSliceInt    []int8
+		unsetSliceBool   []bool
+		unsetSliceFloat  []float32
+		unsetString      string
+		unsetInt64       int64
+
+		unsetDateTime *time.Time
+		unsetDuration time.Duration
+
+		tcc = []struct {
+			value  interface{}
+			expect interface{}
+			sc     string
+		}{
+			{
+				value:  []string{},
+				expect: false,
+				sc:     "empty string slice",
+			},
+			{
+				value:  map[string]string{},
+				expect: false,
+				sc:     "empty map of strings",
+			},
+			{
+				// @todo
+				value:  unsetSliceString,
+				expect: false,
+				sc:     "undefined string slice",
+			},
+			{
+				value:  []int{},
+				expect: false,
+				sc:     "empty int slice",
+			},
+			{
+				value:  []int{1},
+				expect: false,
+				sc:     "1-elem int slice",
+			},
+			{
+				// @todo
+				value:  unsetSliceInt,
+				expect: false,
+				sc:     "undefined int slice",
+			},
+			{
+				value:  unsetSliceBool,
+				expect: false,
+				sc:     "undefined slice bool",
+			},
+			{
+				value:  int(1),
+				expect: false,
+				sc:     "defined int",
+			},
+			{
+				value:  int(0),
+				expect: false,
+				sc:     "defined int 0",
+			},
+			{
+				value:  "",
+				expect: false,
+				sc:     "emty string",
+			},
+			{
+				value:  unsetString,
+				expect: false,
+				sc:     "undefined string",
+			},
+			{
+				value:  unsetSliceFloat,
+				expect: false,
+				sc:     "undefined slice float",
+			},
+			{
+				value:  unsetInt64,
+				expect: false,
+				sc:     "undefined slice int64",
+			},
+			{
+				value:  []float32{11.1},
+				expect: false,
+				sc:     "non-empty slice float32",
+			},
+			{
+				value:  []float32{},
+				expect: false,
+				sc:     "empty slice float32",
+			},
+			{
+				value:  unsetDateTime,
+				expect: true,
+				sc:     "undefined datetime",
+			},
+			{
+				value:  Must(NewInteger(nil)),
+				expect: false,
+				sc:     "undefined Integer expr",
+			},
+			{
+				value:  Must(NewDateTime(nil)),
+				expect: true,
+				sc:     "nil DateTime expr",
+			},
+			{
+				value:  Must(NewDateTime(unsetDateTime)),
+				expect: true,
+				sc:     "undefined DateTime expr",
+			},
+			{
+				value:  Must(NewArray([]string{"A"})),
+				expect: false,
+				sc:     "non empty Array expr",
+			},
+			{
+				value:  Must(NewBoolean(nil)),
+				expect: false,
+				sc:     "undefined Boolean expr",
+			},
+			{
+				value:  Must(NewBytes(nil)),
+				expect: false,
+				sc:     "undefined Bytes expr",
+			},
+			{
+				value:  Must(NewDuration(unsetDuration)),
+				expect: false,
+				sc:     "undefined Duration expr",
+			},
+		}
+	)
+
+	for _, tst := range tcc {
+		req.Equal(tst.expect, isNil(tst.value), "Failed isNil test: %s", tst.sc)
 	}
 }
 

server/store/adapters/rdbms/drivers/types.go

@@ -133,7 +133,7 @@ func (t *TypeTimestamp) Decode(raw any) (any, bool, error) {
 }
 
 func (t *TypeTimestamp) Encode(val any) (driver.Value, error) {
-	if reflect2.IsNil(val) {
+	if reflect2.IsNil(val) || val == "" {
 		return nil, nil
 	}