DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Merge branch 'rem-crm-record-json'

Browse Source
This commit is contained in:
Denis Arh
2019-01-15 18:16:38 +01:00
parent ad0d22f3ea 08c72c60fc
commit 13482c4de0
25 changed files with 905 additions and 506 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
api/crm/spec.json
View File

@@ -417,10 +417,10 @@
],
"post": [
{
"type": "sqlxTypes.JSONText",
"name": "fields",
"type": "types.RecordValueSet",
"name": "values",
"required": true,
"title": "Record JSON"
"title": "Record values"
}
]
}
@@ -469,10 +469,10 @@
],
"post": [
{
"type": "sqlxTypes.JSONText",
"name": "fields",
"type": "types.RecordValueSet",
"name": "values",
"required": true,
"title": "Record JSON"
"title": "Record values"
}
]
}

12
api/crm/spec/module.json
View File

@@ -222,10 +222,10 @@
],
"post": [
{
"name": "fields",
"name": "values",
"required": true,
"title": "Record JSON",
"type": "sqlxTypes.JSONText"
"title": "Record values",
"type": "types.RecordValueSet"
}
]
}
@@ -274,10 +274,10 @@
],
"post": [
{
"name": "fields",
"name": "values",
"required": true,
"title": "Record JSON",
"type": "sqlxTypes.JSONText"
"title": "Record values",
"type": "types.RecordValueSet"
}
]
}

4
codegen.sh
View File

@@ -37,7 +37,9 @@ function types {
CGO_ENABLED=0 go build -o ./build/gen-type-set codegen/v2/type-set.go
fi
./build/gen-type-set --types Module,Page,Chart,Trigger -no-pk-types ModuleField --output crm/types/type.gen.go
./build/gen-type-set --types Module,Page,Chart,Trigger,Record \
--no-pk-types ModuleField,RecordValue \
--output crm/types/type.gen.go
./build/gen-type-set --types MessageAttachment --output sam/types/attachment.gen.go
./build/gen-type-set --types Channel --output sam/types/channel.gen.go

2
crm/db/mysql/statik.go
View File

File diff suppressed because one or more lines are too long

1
crm/db/schema/mysql/20190110175924.rem-crm-record-json-field.up.sql Normal file
View File

@@ -0,0 +1 @@
ALTER TABLE `crm_record` DROP COLUMN `json`;

25
crm/db/schema/mysql/20190114072000.cleanup-record-tables-and-multival.up.sql Normal file
View File

@@ -0,0 +1,25 @@
-- No more links, we'll handle this through ref field on crm_record_value tbl
DROP TABLE IF EXISTS `crm_record_links`;
-- Not columns, values
ALTER TABLE `crm_record_column` RENAME TO `crm_record_value`;
-- Simplify names
ALTER TABLE `crm_record_value` CHANGE COLUMN `column_name` `name` VARCHAR(64);
ALTER TABLE `crm_record_value` CHANGE COLUMN `column_value` `value` TEXT;
-- Add reference
ALTER TABLE `crm_record_value` ADD COLUMN `ref` BIGINT UNSIGNED DEFAULT 0 NOT NULL;
ALTER TABLE `crm_record_value` ADD COLUMN `deleted_at` datetime DEFAULT NULL;
ALTER TABLE `crm_record_value` ADD COLUMN `place` INT UNSIGNED DEFAULT 0 NOT NULL;
ALTER TABLE `crm_record_value` DROP PRIMARY KEY, ADD PRIMARY KEY(`record_id`, `name`, `place`);
CREATE INDEX crm_record_value_ref ON crm_record_value (ref);
-- We want this as a real field
ALTER TABLE `crm_module_form` ADD COLUMN `is_multi` TINYINT(1) NOT NULL;
-- This will be handled through meta(json) fieldd
ALTER TABLE `crm_module_form` DROP COLUMN `help_text`;
ALTER TABLE `crm_module_form` DROP COLUMN `max_length`;
ALTER TABLE `crm_module_form` DROP COLUMN `default_Value`;

2
crm/db/schema/mysql/new.sh
View File

@@ -1,2 +1,2 @@
#!/bin/bash
touch $(date +%Y%m%d%H%M%S).up.sql
touch $(date +%Y%m%d%H%M%S).up.sql

9
crm/repository/ql/ast_nodes.go
View File

@@ -107,3 +107,12 @@ func (nn Columns) String() (out string) {
return
}
func (nn Columns) Strings() (out []string) {
out = make([]string, len(nn))
for i, n := range nn {
out[i] = n.String()
}
return
}

2
crm/repository/ql/ast_parser.go
View File

@@ -25,7 +25,6 @@ type (
// NewParser returns a new instance of Parser.
func NewParser() *Parser {
p := &Parser{
tokbuf: make([]Token, 3),
OnIdent: func(ident Ident) (Ident, error) { return ident, nil },
OnFunction: func(ident Function) (Function, error) { return ident, nil },
}
@@ -51,6 +50,7 @@ func (p *Parser) peekToken(s int) Token {
func (p *Parser) initLexer(s string) {
p.lexer = NewLexer(strings.NewReader(s))
p.tokbuf = make([]Token, 3)
for c := 1; c < cap(p.tokbuf); c++ {
// Fill the buffer

22
crm/repository/ql/ast_parser_test.go
View File

@@ -242,10 +242,27 @@ func TestAstParser_ColumnParser(t *testing.T) {
},
},
},
{
in: `DATE_FORMAT(some_date, '%Y-%m-01')`,
cols: Columns{
Column{
Expr: ASTNodes{
Function{
Name: "DATE_FORMAT",
Arguments: ASTSet{
Ident{Value: "some_date"},
String{Value: "%Y-%m-01"},
},
},
},
},
},
},
}
p := NewParser()
for i, test := range tests {
if cols, err := NewParser().ParseColumns(test.in); err != test.err {
if cols, err := p.ParseColumns(test.in); err != test.err {
t.Fatalf("%d. %s: error mismatch:\n expected: %v\n got: %v\n\n", i, test.in, test.err, err)
} else if test.err == nil && !reflect.DeepEqual(test.cols, cols) {
t.Errorf("%d. %s\n\ncols does not match:\n\nexpected: %#v\n got: %#v\n\n", i, test.in, test.cols, cols)
@@ -265,9 +282,8 @@ func TestAstParser_IdentModifier(t *testing.T) {
},
}
p := NewParser()
for i, test := range tests {
p := NewParser()
p.OnIdent = func(ident Ident) (Ident, error) {
ident.Value = fmt.Sprintf("__wrap_%s_wrap__", ident.Value)
return ident, nil

280
crm/repository/record.go
View File

@@ -2,13 +2,15 @@ package repository
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/jmoiron/sqlx"
"github.com/lann/builder"
"github.com/pkg/errors"
"github.com/titpetric/factory"
"gopkg.in/Masterminds/squirrel.v1"
sq "gopkg.in/Masterminds/squirrel.v1"
"github.com/crusttech/crust/crm/repository/ql"
@@ -21,27 +23,29 @@ type (
FindByID(id uint64) (*types.Record, error)
Report(moduleID uint64, metrics, dimensions, filter string) (results interface{}, err error)
Report(module *types.Module, metrics, dimensions, filter string) (results interface{}, err error)
Find(module *types.Module, filter string, sort string, page int, perPage int) (*FindResponse, error)
Create(mod *types.Record) (*types.Record, error)
Update(mod *types.Record) (*types.Record, error)
Create(record *types.Record) (*types.Record, error)
Update(record *types.Record) (*types.Record, error)
DeleteByID(id uint64) error
Fields(module *types.Module, record *types.Record) ([]*types.RecordColumn, error)
UpdateValues(recordID uint64, rvs types.RecordValueSet) (err error)
LoadValues(IDs ...uint64) (rvs types.RecordValueSet, err error)
}
FindResponseMeta struct {
Filter string `json:"filter,omitempty"`
Page int `json:"page"`
PerPage int `json:"perPage"`
Count int `json:"count"`
Sort string `json:"sort"`
Filter string `json:"filter,omitempty"`
Sort string `json:"sort,omitempty"`
Page int `json:"page"`
PerPage int `json:"perPage"`
Count int `json:"count"`
}
FindResponse struct {
Meta FindResponseMeta `json:"meta"`
Records []*types.Record `json:"records"`
Records types.RecordSet `json:"records"`
}
record struct {
@@ -50,7 +54,7 @@ type (
)
const (
jsonWrap = `JSON_UNQUOTE(JSON_EXTRACT(json, REPLACE(JSON_UNQUOTE(JSON_SEARCH(json, 'one', ?)), '.name', '.value')))`
sortWrap = `sort`
)
func Record(ctx context.Context, db *factory.DB) RecordRepository {
@@ -73,27 +77,15 @@ func (r *record) FindByID(id uint64) (*types.Record, error) {
return mod, nil
}
func (r *record) Report(moduleID uint64, metrics, dimensions, filter string) (results interface{}, err error) {
crb := NewRecordReportBuilder(moduleID)
if err = crb.SetMetrics(metrics); err != nil {
return
}
if err = crb.SetDimensions(dimensions); err != nil {
return
}
if err = crb.SetFilter(filter); err != nil {
return
}
func (r *record) Report(module *types.Module, metrics, dimensions, filter string) (results interface{}, err error) {
crb := NewRecordReportBuilder(module)
var result = make([]map[string]interface{}, 0)
if query, args, err := crb.Build(); err != nil {
return nil, errors.Wrap(err, "Can not generate report query")
if query, args, err := crb.Build(metrics, dimensions, filter); err != nil {
return nil, errors.Wrap(err, "can not generate report query")
} else if rows, err := r.db().Query(query, args...); err != nil {
return nil, errors.Wrapf(err, "Can not execute report query (%s)", query)
return nil, errors.Wrapf(err, "can not execute report query (%s)", query)
} else {
for rows.Next() {
result = append(result, crb.Cast(rows))
@@ -127,26 +119,14 @@ func (r *record) Find(module *types.Module, filter string, sort string, page int
Records: make([]*types.Record, 0),
}
// Create query for fetching and counting records.
query := squirrel.
Select().
From("crm_record").
Where("(module_id = ? AND deleted_at IS NULL AND json IS NOT NULL)", module.ID)
// Parse filters.
p := ql.NewParser()
p.OnIdent = ql.MakeIdentWrapHandler(jsonWrap, "created_at", "updated_at", "id", "user_id")
where, err := p.ParseExpression(filter)
var query, err = r.buildQuery(module, filter, sort)
if err != nil {
return nil, err
}
// Append filtering to query.
query = query.Where(squirrel.And{where})
// Create count SQL sentences.
count := query.Column(squirrel.Alias(squirrel.Expr("COUNT(*)"), "count"))
// Assemble SQL for counting (includes only where)
count := query.Column("COUNT(*)")
count = builder.Delete(count, "OrderBys").(sq.SelectBuilder)
sqlSelect, argsSelect, err := count.ToSql()
if err != nil {
return nil, err
@@ -162,31 +142,12 @@ func (r *record) Find(module *types.Module, filter string, sort string, page int
return response, nil
}
// Create query for fetching records.
// Assemble SQL for fetching record (where + sorting + paging)...
query = query.
Column("*").
Column("crm_record.*").
Limit(uint64(perPage)).
Offset(uint64(page))
// Append Sorting.
p = ql.NewParser()
p.OnIdent = ql.MakeIdentOrderWrapHandler(jsonWrap, "id", "module_id", "user_id", "created_at", "updated_at")
orderColumns, err := p.ParseColumns(sort)
if err != nil {
return nil, err
}
var argsOrder = make([]interface{}, 0)
for _, column := range orderColumns {
sql, args, err := column.ToSql()
if err != nil {
return nil, err
}
argsOrder = append(argsOrder, args...)
query = query.OrderBy(sql)
}
// Create actual fetch SQL sentences.
sqlSelect, argsSelect, err = query.ToSql()
if err != nil {
@@ -194,7 +155,6 @@ func (r *record) Find(module *types.Module, filter string, sort string, page int
}
// Append order args to select args and execute actual query.
argsSelect = append(argsSelect, argsOrder...)
if err := r.db().Select(&response.Records, sqlSelect, argsSelect...); err != nil {
return nil, err
}
@@ -202,68 +162,117 @@ func (r *record) Find(module *types.Module, filter string, sort string, page int
return response, nil
}
func (r *record) Create(mod *types.Record) (*types.Record, error) {
mod.ID = factory.Sonyflake.NextID()
mod.CreatedAt = time.Now()
mod.UserID = Identity(r.Context())
func (r *record) buildQuery(module *types.Module, filter string, sort string) (query sq.SelectBuilder, err error) {
// Create query for fetching and counting records.
query = sq.Select().
From("crm_record").
Where(sq.Eq{"module_id": module.ID}).
Where(sq.Eq{"deleted_at": nil})
fields := make([]types.RecordColumn, 0)
if err := json.Unmarshal(mod.Fields, &fields); err != nil {
return nil, errors.Wrap(err, "No content")
// Do not translate/wrap these
var realColumns = []string{
"id",
"module_id",
"user_id",
"created_at",
"updated_at",
}
r.db().Exec("delete from crm_record_links where record_id=?", mod.ID)
for _, v := range fields {
v.RecordID = mod.ID
if err := r.db().Replace("crm_record_column", v); err != nil {
return nil, errors.Wrap(err, "Error adding columns")
}
for _, related := range v.Related {
row := types.Related{
RecordID: v.RecordID,
Name: v.Name,
RelatedRecordID: related,
const colWrap = `(SELECT value FROM crm_record_value WHERE name = ? AND record_id = crm_record.id AND deleted_at IS NULL)`
// Parse filters.
if filter != "" {
var (
// Filter parser
fp = ql.NewParser()
// Filter node
fn ql.ASTNode
)
// Make a nice wrapper that will translate module fields to subqueries
fp.OnIdent = func(i ql.Ident) (ql.Ident, error) {
for _, s := range realColumns {
if s == i.Value {
return i, nil
}
}
if err := r.db().Replace("crm_record_links", row); err != nil {
return nil, errors.Wrap(err, "Error adding column links")
if !module.Fields.HasName(i.Value) {
return i, errors.Errorf("unknown field %q", i.Value)
}
// @todo switch value for ref when doing Record/User lookup
i.Args = []interface{}{i.Value}
i.Value = colWrap
return i, nil
}
if fn, err = fp.ParseExpression(filter); err != nil {
return
}
query = query.Where(fn)
}
if err := r.db().Insert("crm_record", mod); err != nil {
return nil, err
if sort != "" {
var (
// Sort parser
sp = ql.NewParser()
// Sort columns
sc ql.Columns
)
sp.OnIdent = func(i ql.Ident) (ql.Ident, error) {
for _, s := range realColumns {
if s == i.Value {
i.Value += " "
return i, nil
}
}
if !module.Fields.HasName(i.Value) {
return i, errors.Errorf("unknown field %q", i.Value)
}
i.Value = strings.Replace(colWrap, "?", fmt.Sprintf("'%s'", i.Value), 1) + " "
return i, nil
}
if sc, err = sp.ParseColumns(sort); err != nil {
return
}
query = query.OrderBy(sc.Strings()...)
}
return mod, nil
return
}
func (r *record) Update(mod *types.Record) (*types.Record, error) {
func (r *record) Create(record *types.Record) (*types.Record, error) {
record.ID = factory.Sonyflake.NextID()
record.CreatedAt = time.Now()
record.UserID = Identity(r.Context())
if err := r.db().Replace("crm_record", record); err != nil {
return nil, errors.Wrap(err, "could not update record")
}
return record, nil
}
func (r *record) Update(record *types.Record) (*types.Record, error) {
now := time.Now()
mod.UpdatedAt = &now
record.UpdatedAt = &now
fields := make([]types.RecordColumn, 0)
if err := json.Unmarshal(mod.Fields, &fields); err != nil {
return nil, errors.Wrap(err, "Error when saving record, no content")
if err := r.db().Replace("crm_record", record); err != nil {
return nil, errors.Wrap(err, "could not update record")
}
r.db().Exec("delete from crm_record_links where record_id=?", mod.ID)
for _, v := range fields {
v.RecordID = mod.ID
if err := r.db().Replace("crm_record_column", v); err != nil {
return nil, errors.Wrap(err, "Error adding columns to database")
}
for _, related := range v.Related {
row := types.Related{
RecordID: v.RecordID,
Name: v.Name,
RelatedRecordID: related,
}
if err := r.db().Replace("crm_record_links", row); err != nil {
return nil, errors.Wrap(err, "Error adding column links")
}
}
}
return mod, r.db().Replace("crm_record", mod)
return record, nil
}
func (r *record) DeleteByID(id uint64) error {
@@ -271,25 +280,32 @@ func (r *record) DeleteByID(id uint64) error {
return err
}
func (r *record) Fields(module *types.Module, record *types.Record) ([]*types.RecordColumn, error) {
result := make([]*types.RecordColumn, 0)
if module.ID != record.ModuleID {
return result, errors.New("Record does not belong to the module")
func (r *record) UpdateValues(recordID uint64, rvs types.RecordValueSet) (err error) {
// Remove all records and prepare to be updated
// @todo be more selective and delete only removed values
if _, err = r.db().Exec("DELETE FROM crm_record_value WHERE record_id = ?", recordID); err != nil {
return errors.Wrap(err, "could not remove record values")
}
fieldNames := module.Fields.Names()
err = rvs.Walk(func(value *types.RecordValue) error {
value.RecordID = recordID
return r.db().Replace("crm_record_value", value)
})
if len(fieldNames) == 0 {
return result, errors.New("Module has no fields")
}
return errors.Wrap(err, "could not replace record values")
order := "FIELD(column_name" + strings.Repeat(",?", len(fieldNames)) + ")"
args := []interface{}{
record.ID,
}
for _, v := range fieldNames {
args = append(args, v)
}
return result, r.db().Select(&result, "select * FROM crm_record_column where record_id=? order by "+order, args...)
}
func (r *record) LoadValues(IDs ...uint64) (rvs types.RecordValueSet, err error) {
if len(IDs) == 0 {
return
}
var sql = "SELECT * FROM crm_record_value WHERE record_id IN (?) AND deleted_at IS NULL ORDER BY record_id, place"
if sql, args, err := sqlx.In(sql, IDs); err != nil {
return nil, err
} else {
return rvs, r.db().Select(&rvs, sql, args...)
}
}

164
crm/repository/record_report_builder.go
View File

@@ -6,21 +6,22 @@ import (
"strings"
"github.com/jmoiron/sqlx"
"github.com/pkg/errors"
"gopkg.in/Masterminds/squirrel.v1"
"github.com/crusttech/crust/crm/repository/ql"
"github.com/crusttech/crust/crm/types"
)
type (
recordReportBuilder struct {
moduleID uint64
metrics ql.Columns
dimensions ql.Columns
filter ql.ASTNode
module *types.Module
// This is set by metric/column building to assist Cast()
numerics []string
report squirrel.SelectBuilder
parser *ql.Parser
}
)
@@ -37,23 +38,6 @@ func stdAggregationHandler(f ql.Function) (ql.Function, error) {
}
}
// Identifiers should be names of the fields (physical table columns OR json fields, defined in module)
func stdGroupByFuncHandler(f ql.Function) (ql.Function, error) {
switch strings.ToUpper(f.Name) {
case "DATE_FORMAT":
if len(f.Arguments) == 2 {
return f, nil
} else {
return f, fmt.Errorf("incorrect parameter count for group-by function '%s'", f.Name)
}
case "CONCAT", "QUARTER", "YEAR", "DATE", "NOW":
return f, nil
default:
return f, fmt.Errorf("unsupported group-by function %q", f.Name)
}
}
// Identifiers should be names of the fields (physical table columns OR json fields, defined in module)
func stdFilterFuncHandler(f ql.Function) (ql.Function, error) {
switch strings.ToUpper(f.Name) {
@@ -65,49 +49,77 @@ func stdFilterFuncHandler(f ql.Function) (ql.Function, error) {
}
}
func NewRecordReportBuilder(moduleID uint64) *recordReportBuilder {
return &recordReportBuilder{moduleID: moduleID}
}
func (b *recordReportBuilder) SetMetrics(metrics string) (err error) {
p := ql.NewParser()
p.OnIdent = ql.MakeIdentWrapHandler(jsonWrap, "created_at", "updated_at")
p.OnFunction = stdAggregationHandler
b.metrics, err = p.ParseColumns(metrics)
return
}
func (b *recordReportBuilder) SetDimensions(dimensions string) (err error) {
p := ql.NewParser()
p.OnIdent = ql.MakeIdentWrapHandler(jsonWrap, "created_at", "updated_at")
p.OnFunction = stdGroupByFuncHandler
b.dimensions, err = p.ParseColumns(dimensions)
return
}
func (b *recordReportBuilder) SetFilter(filters string) (err error) {
p := ql.NewParser()
p.OnIdent = ql.MakeIdentWrapHandler(jsonWrap, "created_at", "updated_at", "id", "user_id")
p.OnFunction = stdFilterFuncHandler
b.filter, err = p.ParseExpression(filters)
return
}
func (b *recordReportBuilder) Build() (sql string, args []interface{}, err error) {
report := squirrel.
func NewRecordReportBuilder(module *types.Module) *recordReportBuilder {
var report = squirrel.
Select().
Column(squirrel.Alias(squirrel.Expr("COUNT(*)"), "count")).
From("crm_record").
Where("module_id = ?", b.moduleID)
Where("module_id = ?", module.ID)
return &recordReportBuilder{
parser: ql.NewParser(),
module: module,
report: report,
}
}
func (b *recordReportBuilder) isRealCol(name string) bool {
switch name {
case "id",
"module_id",
"user_id",
"created_at",
"updated_at":
return true
}
return false
}
func (b *recordReportBuilder) Build(metrics, dimensions, filters string) (sql string, args []interface{}, err error) {
var joinedFields = []string{}
var alreadyJoined = func(f string) bool {
for _, a := range joinedFields {
if a == f {
return true
}
}
joinedFields = append(joinedFields, f)
return false
}
b.parser.OnIdent = func(i ql.Ident) (ql.Ident, error) {
if b.isRealCol(i.Value) {
return i, nil
}
if !b.module.Fields.HasName(i.Value) {
return i, errors.Errorf("unknown field %q", i.Value)
}
if !alreadyJoined(i.Value) {
b.report = b.report.LeftJoin(fmt.Sprintf(
"crm_record_value AS rv_%s ON (rv_%s.record_id = crm_record.id AND rv_%s.name = ? AND rv_%s.deleted_at IS NULL)",
i.Value, i.Value, i.Value, i.Value,
), i.Value)
}
// @todo switch value for ref when doing Record/User lookup
i.Value = fmt.Sprintf("rv_%s.value", i.Value)
return i, nil
}
var columns ql.Columns
b.parser.OnFunction = stdAggregationHandler
if columns, err = b.parser.ParseColumns(metrics); err != nil {
err = errors.Wrapf(err, "could not parse metrics %q", metrics)
return
}
// Add all metrics to columns
for i, m := range b.metrics {
for i, m := range columns {
if m.Alias == "" {
// Generate alias
m.Alias = fmt.Sprintf("metric_%d", i)
@@ -115,25 +127,41 @@ func (b *recordReportBuilder) Build() (sql string, args []interface{}, err error
// Wrap to cast func to ensure numeric output
col := squirrel.Alias(SqlConcatExpr("CAST(", m.Expr, " AS DECIMAL(14,2))"), m.Alias)
report = report.Column(col)
b.report = b.report.Column(col)
b.numerics = append(b.numerics, m.Alias)
}
// Add all dimensions to columns
for i, d := range b.dimensions {
b.parser.OnFunction = stdFilterFuncHandler
if columns, err = b.parser.ParseColumns(dimensions); err != nil {
err = errors.Wrapf(err, "could not parse dimensions %q", dimensions)
return
}
// Add dimensions
for i, d := range columns {
if d.Alias == "" {
d.Alias = fmt.Sprintf("dimension_%d", i)
}
report = report.Column(d)
report = report.GroupBy(d.Alias)
report = report.OrderBy(d.Alias)
b.report = b.report.
Column(d).
GroupBy(d.Alias).
OrderBy(d.Alias)
}
report = report.Where(b.filter)
// Use a different handler for filter functions for this
b.parser.OnFunction = stdFilterFuncHandler
return report.ToSql()
var filter ql.ASTNode
if filter, err = b.parser.ParseExpression(filters); err != nil {
err = errors.Wrapf(err, "could not parse filters %q", filters)
return
}
b.report = b.report.Where(filter)
return b.report.ToSql()
}
func (b recordReportBuilder) Cast(row sqlx.ColScanner) map[string]interface{} {

42
crm/repository/record_report_builder_test.go
View File

@@ -2,26 +2,32 @@ package repository
import (
"testing"
"github.com/crusttech/crust/crm/types"
"github.com/crusttech/crust/internal/test"
)
func TestRecordReportBuilder_parseExpression(t *testing.T) {
b := recordReportBuilder{jsonField: "JSONFIELD"}
func TestRecordReportBuilder2(t *testing.T) {
builder := NewRecordReportBuilder(&types.Module{
ID: 1000,
Fields: types.ModuleFieldSet{
&types.ModuleField{Name: "single1"},
&types.ModuleField{Name: "multi1", Multi: true},
&types.ModuleField{Name: "ref1", Kind: "Record"},
&types.ModuleField{Name: "multiRef1", Kind: "Record", Multi: true},
}},
)
tc := []struct {
exp string
sql string
arg []interface{}
err error
}{
{exp: "count(foo)", sql: "COUNT(JSONFIELD)", arg: []interface{}{"foo"}},
{exp: "sum(count(foo))", sql: "SUM(COUNT(JSONFIELD))", arg: []interface{}{"foo"}},
{exp: "sum( count( foo)) ", sql: "SUM(COUNT(JSONFIELD))", arg: []interface{}{"foo"}},
}
expected := "SELECT (COUNT(*)) AS count, (CAST(max(rv_single1.value) AS DECIMAL(14,2))) AS metric_0, " +
"(QUARTER(rv_ref1.value)) AS dimension_0 " +
"FROM crm_record " +
"LEFT JOIN crm_record_value AS rv_single1 ON (rv_single1.record_id = crm_record.id AND rv_single1.name = ? AND rv_single1.deleted_at IS NULL) " +
"LEFT JOIN crm_record_value AS rv_ref1 ON (rv_ref1.record_id = crm_record.id AND rv_ref1.name = ? AND rv_ref1.deleted_at IS NULL) " +
"WHERE module_id = ? AND rv_ref1.value = 2 " +
"GROUP BY dimension_0 " +
"ORDER BY dimension_0"
for _, c := range tc {
sql, arg, err := b.parseExpression(c.exp).ToSql()
assert(t, sql == c.sql, "Expecting expression SQL to match (%v == %v)", sql, c.sql)
assert(t, len(arg) == len(c.arg), "Expecting arguments count to match (%v == %v)", arg, c.arg)
assert(t, err == c.err, "Expecting errors to match (%v == %v)", err, c.err)
}
sql, _, err := builder.Build("max(single1)", "QUARTER(ref1)", "ref1 = 2")
test.ErrNil(t, err, "report builder returned an error: %v")
test.Assert(t, expected == sql, "did not get expected sql for report, got: %s", sql)
}

60
crm/repository/record_test.go Normal file
View File

@@ -0,0 +1,60 @@
package repository
import (
"strings"
"testing"
"github.com/crusttech/crust/crm/types"
"github.com/crusttech/crust/internal/test"
)
func TestRecordFinder(t *testing.T) {
r := record{}
m := &types.Module{
ID: 123,
Fields: types.ModuleFieldSet{
&types.ModuleField{Name: "foo"},
&types.ModuleField{Name: "bar"},
},
}
ttc := []struct {
filter string
sort string
match []string
args []interface{}
}{
{
match: []string{"SELECT * FROM crm_record WHERE module_id = ? AND deleted_at IS NULL"},
args: []interface{}{123}},
{
filter: "id = 5 AND foo = 7",
match: []string{
" AND id = 5",
" AND (SELECT value FROM crm_record_value WHERE name = ? AND record_id = crm_record.id AND deleted_at IS NULL) = 7"},
args: []interface{}{123}},
{
sort: "id ASC, foo DESC",
match: []string{
" id ASC, (SELECT value FROM crm_record_value WHERE name = 'foo' AND record_id = crm_record.id AND deleted_at IS NULL) DESC"},
args: []interface{}{123}},
}
for _, tc := range ttc {
sb, err := r.buildQuery(m, tc.filter, tc.sort)
test.Assert(t, err == nil, "buildQuery(%q, %q) returned an error: %v", tc.filter, tc.sort, err)
sb = sb.Column("*")
sql, args, err := sb.ToSql()
for _, m := range tc.match {
test.Assert(t, strings.Contains(sql, m),
"assertion failed; query %q \n "+
" did not contain %q", sql, m)
}
_ = args
// test.Assert(t, reflect.DeepEqual(args, tc.args),
// "assertion failed; args %v \n "+
// " do not match expected %v", args, tc.args)
}
}

28
crm/rest/module.go
View File

@@ -12,15 +12,15 @@ import (
type (
Module struct {
module service.ModuleService
content service.RecordService
module service.ModuleService
record service.RecordService
}
)
func (Module) New() *Module {
return &Module{
module: service.DefaultModule,
content: service.DefaultRecord,
module: service.DefaultModule,
record: service.DefaultRecord,
}
}
@@ -56,34 +56,28 @@ func (s *Module) Edit(ctx context.Context, r *request.ModuleEdit) (interface{},
}
func (s *Module) RecordReport(ctx context.Context, r *request.ModuleRecordReport) (interface{}, error) {
return s.content.With(ctx).Report(r.ModuleID, r.Metrics, r.Dimensions, r.Filter)
return s.record.With(ctx).Report(r.ModuleID, r.Metrics, r.Dimensions, r.Filter)
}
func (s *Module) RecordList(ctx context.Context, r *request.ModuleRecordList) (interface{}, error) {
return s.content.With(ctx).Find(r.ModuleID, r.Filter, r.Sort, r.Page, r.PerPage)
return s.record.With(ctx).Find(r.ModuleID, r.Filter, r.Sort, r.Page, r.PerPage)
}
func (s *Module) RecordRead(ctx context.Context, r *request.ModuleRecordRead) (interface{}, error) {
return s.content.With(ctx).FindByID(r.ModuleID, r.RecordID)
return s.record.With(ctx).FindByID(r.RecordID)
}
func (s *Module) RecordCreate(ctx context.Context, r *request.ModuleRecordCreate) (interface{}, error) {
item := &types.Record{
ModuleID: r.ModuleID,
Fields: r.Fields,
}
return s.content.With(ctx).Create(item)
return s.record.With(ctx).Create(&types.Record{ModuleID: r.ModuleID, Values: r.Values})
}
func (s *Module) RecordEdit(ctx context.Context, r *request.ModuleRecordEdit) (interface{}, error) {
item := &types.Record{
return s.record.With(ctx).Update(&types.Record{
ID: r.RecordID,
ModuleID: r.ModuleID,
Fields: r.Fields,
}
return s.content.With(ctx).Update(item)
Values: r.Values})
}
func (s *Module) RecordDelete(ctx context.Context, r *request.ModuleRecordDelete) (interface{}, error) {
return resputil.OK(), s.content.With(ctx).DeleteByID(r.RecordID)
return resputil.OK(), s.record.With(ctx).DeleteByID(r.RecordID)
}

16
crm/rest/request/module.go
View File

@@ -398,7 +398,7 @@ var _ RequestFiller = NewModuleRecordList()
// Module record/create request parameters
type ModuleRecordCreate struct {
ModuleID uint64 `json:",string"`
Fields sqlxTypes.JSONText
Values types.RecordValueSet
}
func NewModuleRecordCreate() *ModuleRecordCreate {
@@ -433,12 +433,6 @@ func (m *ModuleRecordCreate) Fill(r *http.Request) (err error) {
}
m.ModuleID = parseUInt64(chi.URLParam(r, "moduleID"))
if val, ok := post["fields"]; ok {
if m.Fields, err = parseJSONTextWithErr(val); err != nil {
return err
}
}
return err
}
@@ -494,7 +488,7 @@ var _ RequestFiller = NewModuleRecordRead()
type ModuleRecordEdit struct {
ModuleID uint64 `json:",string"`
RecordID uint64 `json:",string"`
Fields sqlxTypes.JSONText
Values types.RecordValueSet
}
func NewModuleRecordEdit() *ModuleRecordEdit {
@@ -530,12 +524,6 @@ func (m *ModuleRecordEdit) Fill(r *http.Request) (err error) {
m.ModuleID = parseUInt64(chi.URLParam(r, "moduleID"))
m.RecordID = parseUInt64(chi.URLParam(r, "recordID"))
if val, ok := post["fields"]; ok {
if m.Fields, err = parseJSONTextWithErr(val); err != nil {
return err
}
}
return err
}

2
crm/service/main_test.go
View File

@@ -50,7 +50,7 @@ func TestMain(m *testing.M) {
// clean up tables
{
for _, name := range []string{"crm_chart", "crm_trigger", "crm_module", "crm_module_form", "crm_record", "crm_record_column", "crm_page", "sys_user"} {
for _, name := range []string{"crm_chart", "crm_trigger", "crm_module", "crm_module_form", "crm_record", "crm_record_value", "crm_page", "sys_user"} {
_, err := db.Exec("truncate " + name)
if err != nil {
panic("Error when clearing " + name + ": " + err.Error())

260
crm/service/record.go
View File

@@ -2,6 +2,7 @@ package service
import (
"context"
"strconv"
"github.com/pkg/errors"
"github.com/titpetric/factory"
@@ -18,7 +19,6 @@ type (
ctx context.Context
repository repository.RecordRepository
pageRepo repository.PageRepository
moduleRepo repository.ModuleRepository
userSvc systemService.UserService
@@ -27,7 +27,7 @@ type (
RecordService interface {
With(ctx context.Context) RecordService
FindByID(moduleID uint64, recordID uint64) (*types.Record, error)
FindByID(recordID uint64) (*types.Record, error)
Report(moduleID uint64, metrics, dimensions, filter string) (interface{}, error)
Find(moduleID uint64, filter string, sort string, page int, perPage int) (*repository.FindResponse, error)
@@ -36,7 +36,7 @@ type (
Update(record *types.Record) (*types.Record, error)
DeleteByID(recordID uint64) error
Fields(module *types.Module, record *types.Record) ([]*types.RecordColumn, error)
// Fields(module *types.Module, record *types.Record) ([]*types.RecordValue, error)
}
)
@@ -46,84 +46,216 @@ func Record() RecordService {
}).With(context.Background())
}
func (s *record) With(ctx context.Context) RecordService {
func (svc *record) With(ctx context.Context) RecordService {
db := repository.DB(ctx)
return &record{
db: db,
ctx: ctx,
repository: repository.Record(ctx, db),
pageRepo: repository.Page(ctx, db),
moduleRepo: repository.Module(ctx, db),
userSvc: s.userSvc.With(ctx),
userSvc: svc.userSvc.With(ctx),
}
}
func (s *record) FindByID(moduleID uint64, id uint64) (response *types.Record, err error) {
var module *types.Module
if module, err = s.moduleRepo.FindByID(moduleID); err != nil {
return nil, err
}
if response, err = s.repository.FindByID(id); err != nil {
return nil, err
}
return response, s.preload(module, response, "page", "user", "fields")
}
func (s *record) Report(moduleID uint64, metrics, dimensions, filter string) (interface{}, error) {
return s.repository.Report(moduleID, metrics, dimensions, filter)
}
func (s *record) Find(moduleID uint64, filter string, sort string, page int, perPage int) (response *repository.FindResponse, err error) {
var module *types.Module
if module, err = s.moduleRepo.FindByID(moduleID); err != nil {
return nil, err
} else if response, err = s.repository.Find(module, filter, sort, page, perPage); err != nil {
return nil, err
} else if err := s.preloadAll(module, response.Records, "user", "fields"); err != nil {
return nil, err
}
return response, nil
}
func (s *record) Create(mod *types.Record) (*types.Record, error) {
response, err := s.repository.Create(mod)
if err != nil {
return nil, err
}
return response, s.preload(nil, response, "user", "fields")
}
func (s *record) Update(record *types.Record) (c *types.Record, err error) {
validate := func() error {
if record.ID == 0 {
return errors.New("Error updating record: invalid ID")
} else if c, err = s.repository.FindByID(record.ID); err != nil {
return errors.Wrap(err, "Error while loading record for update")
} else {
record.CreatedAt = c.CreatedAt
func (svc *record) FindByID(recordID uint64) (r *types.Record, err error) {
err = svc.db.Transaction(func() (err error) {
if r, err = svc.repository.FindByID(recordID); err != nil {
return
}
return nil
}
if err = svc.preloadValues(r); err != nil {
return
}
if err = validate(); err != nil {
return nil, err
}
if err = svc.preloadUsers(r); err != nil {
return
}
return c, s.db.Transaction(func() (err error) {
c, err = s.repository.Update(record)
return
})
return r, errors.Wrap(err, "unable to find record")
}
func (svc *record) Report(moduleID uint64, metrics, dimensions, filter string) (out interface{}, err error) {
var module *types.Module
err = svc.db.Transaction(func() (err error) {
if module, err = svc.moduleRepo.FindByID(moduleID); err != nil {
return
}
out, err = svc.repository.Report(module, metrics, dimensions, filter)
return
})
return out, errors.Wrap(err, "unable to build a report")
}
func (svc *record) Find(moduleID uint64, filter, sort string, page, perPage int) (rsp *repository.FindResponse, err error) {
var module *types.Module
err = svc.db.Transaction(func() (err error) {
if module, err = svc.moduleRepo.FindByID(moduleID); err != nil {
return
}
if rsp, err = svc.repository.Find(module, filter, sort, page, perPage); err != nil {
return
}
if err = svc.preloadValues(rsp.Records...); err != nil {
return
}
if err = svc.preloadUsers(rsp.Records...); err != nil {
return
}
return
})
return rsp, errors.Wrap(err, "unable to find records")
}
func (svc *record) Create(new *types.Record) (record *types.Record, err error) {
var module *types.Module
err = svc.db.Transaction(func() (err error) {
if module, err = svc.moduleRepo.FindByID(new.ModuleID); err != nil {
return
}
if err = svc.sanitizeValues(module, new.Values); err != nil {
return
}
if record, err = svc.repository.Create(new); err != nil {
return
}
if err = svc.repository.UpdateValues(record.ID, new.Values); err != nil {
return
}
if err = svc.preloadValues(record); err != nil {
return
}
if err = svc.preloadUsers(record); err != nil {
return
}
return
})
return record, errors.Wrap(err, "unable to create record")
}
func (svc *record) Update(updated *types.Record) (record *types.Record, err error) {
var module *types.Module
err = svc.db.Transaction(func() (err error) {
if updated.ID == 0 {
return errors.New("invalid record ID")
}
if record, err = svc.repository.FindByID(updated.ID); err != nil {
return errors.Wrap(err, "nonexistent record")
}
updated.CreatedAt = record.CreatedAt
updated.UserID = record.UserID
if module, err = svc.moduleRepo.FindByID(updated.ModuleID); err != nil {
return
}
if err = svc.sanitizeValues(module, updated.Values); err != nil {
return
}
if record, err = svc.repository.Update(updated); err != nil {
return
}
if err = svc.repository.UpdateValues(record.ID, updated.Values); err != nil {
return
}
if err = svc.preloadUsers(record); err != nil {
return
}
return
})
return record, errors.Wrap(err, "unable to update record")
}
// func (s *record) Fields(module *types.Module, record *types.Record) ([]*types.RecordValue, error) {
// return s.repository.Fields(module, record)
// }
func (svc *record) DeleteByID(id uint64) error {
return svc.repository.DeleteByID(id)
}
// Validates and filters record values
func (svc *record) sanitizeValues(module *types.Module, values types.RecordValueSet) (err error) {
// Make sure there are no multi values in a non-multi value fields
err = module.Fields.Walk(func(field *types.ModuleField) error {
if !field.Multi && len(values.FilterByName(field.Name)) > 1 {
return errors.Errorf("more than one value for a single-value field %q", field.Name)
}
return nil
})
if err != nil {
return
}
var places = map[string]uint{}
// var has bool
return values.Walk(func(value *types.RecordValue) (err error) {
var field = module.Fields.FindByName(value.Name)
if field == nil {
return errors.Errorf("no such field %q", value.Name)
}
if field.IsRef() {
if value.Ref, err = strconv.ParseUint(value.Value, 10, 64); err != nil {
return err
}
}
value.Place = places[field.Name]
places[field.Name]++
return nil
})
}
func (s *record) Fields(module *types.Module, record *types.Record) ([]*types.RecordColumn, error) {
return s.repository.Fields(module, record)
func (svc *record) preloadValues(rr ...*types.Record) error {
if rvs, err := svc.repository.LoadValues(types.RecordSet(rr).IDs()...); err != nil {
return err
} else {
return types.RecordSet(rr).Walk(func(r *types.Record) error {
r.Values = rvs.FilterByRecordID(r.ID)
return nil
})
}
}
func (s *record) DeleteByID(id uint64) error {
return s.repository.DeleteByID(id)
func (svc *record) preloadUsers(rr ...*types.Record) error {
if uu, err := svc.userSvc.FindByIDs(types.RecordSet(rr).UserIDs()...); err != nil {
return err
} else {
return types.RecordSet(rr).Walk(func(r *types.Record) error {
r.User = uu.FindByID(r.UserID)
return nil
})
}
}

184
crm/service/record_test.go
View File

@@ -4,12 +4,9 @@ import (
"context"
"testing"
"encoding/json"
"github.com/pkg/errors"
"github.com/crusttech/crust/crm/types"
"github.com/crusttech/crust/internal/auth"
"github.com/crusttech/crust/internal/test"
systemRepository "github.com/crusttech/crust/system/repository"
systemTypes "github.com/crusttech/crust/system/types"
)
@@ -38,19 +35,20 @@ func TestRecord(t *testing.T) {
Fields: types.ModuleFieldSet{
&types.ModuleField{
Name: "name",
Kind: "input",
},
&types.ModuleField{
Name: "email",
Kind: "email",
},
&types.ModuleField{
Name: "options",
Kind: "select_multi",
Name: "options",
Multi: true,
},
&types.ModuleField{
Name: "description",
Kind: "text",
},
&types.ModuleField{
Name: "another_record",
Kind: "Record",
},
},
}
@@ -61,59 +59,43 @@ func TestRecord(t *testing.T) {
assert(t, err == nil, "Error when creating module: %+v", err)
assert(t, module.ID > 0, "Expected auto generated ID")
columns := []types.RecordColumn{
types.RecordColumn{
Name: "name",
Value: "Tit Petric",
},
types.RecordColumn{
Name: "email",
Value: "tit.petric@example.com",
},
types.RecordColumn{
Name: "options",
Related: []string{"1", "2", "3"},
},
types.RecordColumn{
Name: "description",
Value: "jack of all trades",
},
}
record1 := &types.Record{
ModuleID: module.ID,
}
(&record1.Fields).Scan(func() []byte {
b, _ := json.Marshal(columns)
return b
}())
columns2 := []types.RecordColumn{
types.RecordColumn{
Name: "name",
Value: "Marko Novak",
},
types.RecordColumn{
Name: "email",
Value: "marko.n@example.com",
},
types.RecordColumn{
Name: "options",
Related: []string{"1", "2", "3"},
},
types.RecordColumn{
Name: "description",
Value: "persona non grata",
},
}
record2 := &types.Record{
ModuleID: module.ID,
Values: types.RecordValueSet{
&types.RecordValue{
Name: "name",
Value: "John Doe",
},
&types.RecordValue{
Name: "email",
Value: "john.doe@example.com",
},
&types.RecordValue{
Name: "options",
Value: "1",
},
&types.RecordValue{
Name: "options",
Value: "2",
},
&types.RecordValue{
Name: "options",
Value: "3",
},
&types.RecordValue{
Name: "description",
Value: "just an example",
},
&types.RecordValue{
Name: "another_record",
Value: "918273645",
},
},
}
(&record2.Fields).Scan(func() []byte {
b, _ := json.Marshal(columns2)
return b
}())
// now work with records
{
@@ -127,42 +109,18 @@ func TestRecord(t *testing.T) {
m1, err := repository.Create(record1)
assert(t, err == nil, "Error when creating record: %+v", err)
assert(t, m1.ID > 0, "Expected auto generated ID")
assert(t, m1.User != nil, "Expected non-nil user when creating record")
assert(t, m1.User.Username == "TestUser", "Expected 'TestUser' as username, got '%s'", m1.User.Username)
// create record
m2, err := repository.Create(record2)
assert(t, err == nil, "Error when creating record: %+v", err)
assert(t, m2.ID > 0, "Expected auto generated ID")
assert(t, m2.User != nil, "Expected non-nil user when creating record")
assert(t, m2.User.Username == "TestUser", "Expected 'TestUser' as username, got '%s'", m2.User.Username)
// fetch created record
{
ms, err := repository.FindByID(module.ID, m1.ID)
ms, err := repository.FindByID(m1.ID)
assert(t, err == nil, "Error when retrieving record by id: %+v", err)
assert(t, ms.ID == m1.ID, "Expected ID from database to match, %d != %d", m1.ID, ms.ID)
assert(t, ms.ModuleID == m1.ModuleID, "Expected Module ID from database to match, %d != %d", m1.ModuleID, ms.ModuleID)
{
fields, err := repository.Fields(module, ms)
// fields := make([]testRecordRow, 0)
// err = json.Unmarshal(ms.Fields, &fields)
assert(t, err == nil, "%+v", errors.Wrap(err, "Didn't expect error when unmarshalling"))
assert(t, len(fields) == len(columns), "Expected different field count: %d != %d", 2, len(fields))
for k, v := range columns {
assert(t, fields[k].Name == v.Name, "Expected fields[%d].Name = %s, got %s", k, fields[k].Name, v.Name)
}
}
{
fields := make([]types.RecordColumn, 0)
err := json.Unmarshal(ms.Fields, &fields)
assert(t, err == nil, "%+v", errors.Wrap(err, "Didn't expect error when unmarshalling"))
assert(t, len(fields) == len(columns), "Expected different field count: %d != %d", 2, len(fields))
for k, v := range columns {
assert(t, fields[k].Name == v.Name, "Expected fields[%d].Name = %s, got %s", k, fields[k].Name, v.Name)
}
}
}
// update created record
@@ -173,7 +131,7 @@ func TestRecord(t *testing.T) {
// re-fetch record
{
ms, err := repository.FindByID(module.ID, m1.ID)
ms, err := repository.FindByID(m1.ID)
assert(t, err == nil, "Error when retrieving record by id: %+v", err)
assert(t, ms.ID == m1.ID, "Expected ID from database to match, %d != %d", m1.ID, ms.ID)
assert(t, ms.ModuleID == m1.ModuleID, "Expected ID from database to match, %d != %d", m1.ModuleID, ms.ModuleID)
@@ -198,7 +156,9 @@ func TestRecord(t *testing.T) {
assert(t, mr.Meta.Count == 2, "Expected Meta.Count == 2, got %d", mr.Meta.Count)
assert(t, mr.Meta.Sort == "name asc, email desc", "Expected Meta.Sort == 'name asc, email desc' '%s'", mr.Meta.Sort)
assert(t, mr.Records[0].ModuleID == m1.ModuleID, "Expected record module to match, %d != %d", m1.ModuleID, mr.Records[0].ModuleID)
assert(t, mr.Records[0].ID > mr.Records[1].ID, "Expected order to be ascending")
// @todo sort is not stable
// assert(t, mr.Records[0].ID > mr.Records[1].ID, "Expected order to be ascending")
}
// fetch all records
@@ -209,24 +169,29 @@ func TestRecord(t *testing.T) {
assert(t, mr.Meta.Count == 2, "Expected Meta.Count == 2, got %d", mr.Meta.Count)
assert(t, mr.Meta.Sort == "created_at desc", "Expected Meta.Sort == created_at desc, got '%s'", mr.Meta.Sort)
assert(t, mr.Records[0].ModuleID == m1.ModuleID, "Expected record module to match, %d != %d", m1.ModuleID, mr.Records[0].ModuleID)
assert(t, mr.Records[0].ID > mr.Records[1].ID, "Expected order to be ascending")
// @todo sort is not stable
// assert(t, mr.Records[0].ID > mr.Records[1].ID, "Expected order to be ascending")
}
// fetch all records by query
{
mr, err := repository.Find(module.ID, "name='Tit Petric'", "id desc", 0, 20)
filter := "name='John Doe' AND email='john.doe@example.com'"
sort := "id desc"
mr, err := repository.Find(module.ID, filter, sort, 0, 20)
assert(t, err == nil, "Error when retrieving records: %+v", err)
assert(t, len(mr.Records) == 1, "Expected one record, got %d", len(mr.Records))
assert(t, mr.Meta.Count == 1, "Expected Meta.Count == 1, got %d", mr.Meta.Count)
assert(t, mr.Meta.Page == 0, "Expected Meta.Page == 0, got %d", mr.Meta.Page)
assert(t, mr.Meta.PerPage == 20, "Expected Meta.PerPage == 20, got %d", mr.Meta.PerPage)
assert(t, mr.Meta.Filter == "name='Tit Petric'", "Expected Meta.Filter == name='Tit Petric', got '%s'", mr.Meta.Filter)
assert(t, mr.Meta.Sort == "id desc", "Expected Meta.Sort == id desc, got '%s'", mr.Meta.Sort)
assert(t, mr.Meta.Filter == filter, "Expected Meta.Filter == %q, got %q", filter, mr.Meta.Filter)
assert(t, mr.Meta.Sort == sort, "Expected Meta.Sort == %q, got %q", sort, mr.Meta.Sort)
}
// fetch all records by query
{
mr, err := repository.Find(module.ID, "niall", "id asc", 0, 20)
mr, err := repository.Find(module.ID, "name='niall'", "id asc", 0, 20)
assert(t, err == nil, "Error when retrieving records: %+v", err)
assert(t, len(mr.Records) == 0, "Expected no records, got %d", len(mr.Records))
}
@@ -248,3 +213,48 @@ func TestRecord(t *testing.T) {
}
}
}
func TestValueSanitizer(t *testing.T) {
var (
svc = record{}
module = &types.Module{
Fields: types.ModuleFieldSet{
&types.ModuleField{Name: "single1"},
&types.ModuleField{Name: "multi1", Multi: true},
&types.ModuleField{Name: "ref1", Kind: "Record"},
&types.ModuleField{Name: "multiRef1", Kind: "Record", Multi: true},
},
}
rvs types.RecordValueSet
)
rvs = types.RecordValueSet{{Name: "single1", Value: "single"}}
test.ErrNil(t, svc.sanitizeValues(module, rvs), "unexpected error for sanitizeValues() call: %v")
test.Assert(t, len(rvs) == 1, "expecting 1 record value after sanitization, got %d", len(rvs))
rvs = types.RecordValueSet{{Name: "unknown", Value: "single"}}
test.Assert(t, svc.sanitizeValues(module, rvs) != nil, "expecting sanitizeValues() to return an error, got nil")
rvs = types.RecordValueSet{{Name: "single1", Value: "single"}, {Name: "single1", Value: "single2"}}
test.Assert(t, svc.sanitizeValues(module, rvs) != nil, "expecting sanitizeValues() to return an error, got nil")
rvs = types.RecordValueSet{{Name: "multi1", Value: "multi1"}, {Name: "multi1", Value: "multi1"}}
test.ErrNil(t, svc.sanitizeValues(module, rvs), "unexpected error for sanitizeValues() call: %v")
test.Assert(t, len(rvs) == 2, "expecting 2 record values after sanitization, got %d", len(rvs))
test.Assert(t, rvs[0].Place == 0, "expecting first value to have place value 0, got %d", rvs[0].Place)
test.Assert(t, rvs[1].Place == 1, "expecting second value to have place value 1, got %d", rvs[1].Place)
rvs = types.RecordValueSet{{Name: "ref1", Value: "multi1"}}
test.Assert(t, svc.sanitizeValues(module, rvs) != nil, "expecting sanitizeValues() to return an error, got nil")
rvs = types.RecordValueSet{{Name: "ref1", Value: "12345"}}
test.ErrNil(t, svc.sanitizeValues(module, rvs), "unexpected error for sanitizeValues() call: %v")
test.Assert(t, len(rvs) == 1, "expecting 1 record values after sanitization, got %d", len(rvs))
test.Assert(t, rvs[0].Ref == 12345, "expecting parsed ref value to match, got %d", rvs[0].Ref)
rvs = types.RecordValueSet{{Name: "multiRef1", Value: "12345"}, {Name: "multiRef1", Value: "67890"}}
test.ErrNil(t, svc.sanitizeValues(module, rvs), "unexpected error for sanitizeValues() call: %v")
test.Assert(t, len(rvs) == 2, "expecting 2 record values after sanitization, got %d", len(rvs))
test.Assert(t, rvs[0].Ref == 12345, "expecting parsed ref value to match, got %d", rvs[0].Ref)
test.Assert(t, rvs[1].Ref == 67890, "expecting parsed ref value to match, got %d", rvs[1].Ref)
}

63
crm/service/record_util.go
View File

@@ -1,63 +0,0 @@
package service
import (
"encoding/json"
"github.com/crusttech/crust/crm/types"
)
func (s *record) preloadAll(module *types.Module, records []*types.Record, fields ...string) (err error) {
if len(records) == 0 {
return nil
}
if module == nil {
if module, err = s.moduleRepo.FindByID(records[0].ID); err != nil {
// Assuming all records are from the same module
return
}
}
for _, record := range records {
if err = s.preload(module, record, fields...); err != nil {
return
}
}
return
}
func (s *record) preload(module *types.Module, record *types.Record, fields ...string) (err error) {
if module == nil {
if module, err = s.moduleRepo.FindByID(record.ModuleID); err != nil {
return err
}
}
for _, field := range fields {
switch field {
case "fields":
fields, err := s.Fields(module, record)
if err != nil {
return err
}
json, err := json.Marshal(fields)
if err != nil {
return err
}
if err := (&record.Fields).Scan(json); err != nil {
return err
}
case "page":
if record.Page, err = s.pageRepo.FindByModuleID(record.ModuleID); err != nil {
return
}
case "user":
if record.UserID > 0 {
if record.User, err = s.userSvc.FindByID(record.UserID); err != nil {
return
}
}
}
}
return
}

96
crm/types/type.gen.go
View File

@@ -24,10 +24,20 @@ type (
// This type is auto-generated.
TriggerSet []*Trigger
// RecordSet slice of Record
//
// This type is auto-generated.
RecordSet []*Record
// ModuleFieldSet slice of ModuleField
//
// This type is auto-generated.
ModuleFieldSet []*ModuleField
// RecordValueSet slice of RecordValue
//
// This type is auto-generated.
RecordValueSet []*RecordValue
)
// Walk iterates through every slice item and calls w(Module) err
@@ -254,6 +264,62 @@ func (set TriggerSet) IDs() (IDs []uint64) {
return
}
// Walk iterates through every slice item and calls w(Record) err
//
// This function is auto-generated.
func (set RecordSet) Walk(w func(*Record) error) (err error) {
for i := range set {
if err = w(set[i]); err != nil {
return
}
}
return
}
// Filter iterates through every slice item, calls f(Record) (bool, err) and return filtered slice
//
// This function is auto-generated.
func (set RecordSet) Filter(f func(*Record) (bool, error)) (out RecordSet, err error) {
var ok bool
out = RecordSet{}
for i := range set {
if ok, err = f(set[i]); err != nil {
return
} else if ok {
out = append(out, set[i])
}
}
return
}
// FindByID finds items from slice by its ID property
//
// This function is auto-generated.
func (set RecordSet) FindByID(ID uint64) *Record {
for i := range set {
if set[i].ID == ID {
return set[i]
}
}
return nil
}
// IDs returns a slice of uint64s from all items in the set
//
// This function is auto-generated.
func (set RecordSet) IDs() (IDs []uint64) {
IDs = make([]uint64, len(set))
for i := range set {
IDs[i] = set[i].ID
}
return
}
// Walk iterates through every slice item and calls w(ModuleField) err
//
// This function is auto-generated.
@@ -283,3 +349,33 @@ func (set ModuleFieldSet) Filter(f func(*ModuleField) (bool, error)) (out Module
return
}
// Walk iterates through every slice item and calls w(RecordValue) err
//
// This function is auto-generated.
func (set RecordValueSet) Walk(w func(*RecordValue) error) (err error) {
for i := range set {
if err = w(set[i]); err != nil {
return
}
}
return
}
// Filter iterates through every slice item, calls f(RecordValue) (bool, err) and return filtered slice
//
// This function is auto-generated.
func (set RecordValueSet) Filter(f func(*RecordValue) (bool, error)) (out RecordValueSet, err error) {
var ok bool
out = RecordValueSet{}
for i := range set {
if ok, err = f(set[i]); err != nil {
return
} else if ok {
out = append(out, set[i])
}
}
return
}

99
crm/types/types.go
View File

@@ -22,27 +22,21 @@ type (
Page *Page `json:"page,omitempty"`
Fields types.JSONText `json:"fields,omitempty" db:"json"`
Values RecordValueSet `json:"values,omitempty" db:"-"`
CreatedAt time.Time `db:"created_at" json:"createdAt,omitempty"`
UpdatedAt *time.Time `db:"updated_at" json:"updatedAt,omitempty"`
DeletedAt *time.Time `db:"deleted_at" json:"deletedAt,omitempty"`
}
// RecordColumn is a stored row in the `record_column` table
RecordColumn struct {
RecordID uint64 `json:"-" db:"record_id"`
Name string `json:"name" db:"column_name"`
Value string `json:"value" db:"column_value"`
Related []string `json:"related" db:"-"`
}
Related struct {
RecordID uint64 `json:"-" db:"record_id"`
Name string `json:"-" db:"column_name"`
// RelatedRecordID isn't necessarily a record ID (multiple-select anything goes options)
RelatedRecordID string `json:"-" db:"rel_record_id"`
// RecordValue is a stored row in the `record_value` table
RecordValue struct {
RecordID uint64 `db:"record_id" json:"-"`
Name string `db:"name" json:"name"`
Value string `db:"value" json:"value,omitempty"`
Ref uint64 `db:"ref" json:"-"`
Place uint `db:"place" json:"-"`
DeletedAt *time.Time `db:"deleted_at" json:"deletedAt,omitempty"`
}
// Modules - CRM module definitions
@@ -63,18 +57,16 @@ type (
ModuleID uint64 `json:"moduleID,string" db:"module_id"`
Place int `json:"-" db:"place"`
Kind string `json:"kind" db:"kind"`
Name string `json:"name" db:"name"`
Label string `json:"label" db:"label"`
HelpText string `json:"helpText,omitempty" db:"help_text"`
Default string `json:"defaultValue,omitempty" db:"default_value"`
MaxLength int `json:"maxLength" db:"max_length"`
Kind string `json:"kind" db:"kind"`
Name string `json:"name" db:"name"`
Label string `json:"label" db:"label"`
Options types.JSONText `json:"options" db:"json"`
Private bool `json:"isPrivate" db:"is_private"`
Required bool `json:"isRequired" db:"is_required"`
Visible bool `json:"isVisible" db:"is_visible"`
Multi bool `json:"isMulti" db:"is_multi"`
}
// Page - page structure
@@ -130,6 +122,26 @@ func (set ModuleFieldSet) Names() (names []string) {
return
}
func (set ModuleFieldSet) HasName(name string) bool {
for i := range set {
if name == set[i].Name {
return true
}
}
return false
}
func (set ModuleFieldSet) FindByName(name string) *ModuleField {
for i := range set {
if name == set[i].Name {
return set[i]
}
}
return nil
}
func (set ModuleFieldSet) FilterByModule(moduleID uint64) (ff ModuleFieldSet) {
for i := range set {
if set[i].ModuleID == moduleID {
@@ -139,3 +151,48 @@ func (set ModuleFieldSet) FilterByModule(moduleID uint64) (ff ModuleFieldSet) {
return
}
// IsRef tells us if value of this field be a reference to something (another record, user)?
func (f ModuleField) IsRef() bool {
return f.Kind == "Record" || f.Kind == "User"
}
// UserIDs returns a slice of user IDs from all items in the set
//
// This function is auto-generated.
func (set RecordSet) UserIDs() (IDs []uint64) {
IDs = make([]uint64, 0)
loop:
for i := range set {
for _, id := range IDs {
if id == set[i].UserID {
continue loop
}
}
IDs = append(IDs, set[i].UserID)
}
return
}
func (set RecordValueSet) FilterByName(name string) (vv RecordValueSet) {
for i := range set {
if set[i].Name == name {
vv = append(vv, set[i])
}
}
return
}
func (set RecordValueSet) FilterByRecordID(recordID uint64) (vv RecordValueSet) {
for i := range set {
if set[i].RecordID == recordID {
vv = append(vv, set[i])
}
}
return
}

4
docs/crm/README.md
View File

@@ -202,7 +202,7 @@ CRM module definitions
| Parameter | Type | Method | Description | Default | Required? |
| --------- | ---- | ------ | ----------- | ------- | --------- |
| moduleID | uint64 | PATH | Module ID | N/A | YES |
| fields | sqlxTypes.JSONText | POST | Record JSON | N/A | YES |
| values | types.RecordValueSet | POST | Record values | N/A | YES |
## Read records by ID from module section
@@ -233,7 +233,7 @@ CRM module definitions
| --------- | ---- | ------ | ----------- | ------- | --------- |
| moduleID | uint64 | PATH | Module ID | N/A | YES |
| recordID | uint64 | PATH | Record ID | N/A | YES |
| fields | sqlxTypes.JSONText | POST | Record JSON | N/A | YES |
| values | types.RecordValueSet | POST | Record values | N/A | YES |
## Delete record row from module section

17
system/repository/user.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"time"
"github.com/jmoiron/sqlx"
"github.com/titpetric/factory"
"github.com/crusttech/crust/system/types"
@@ -17,6 +18,7 @@ type (
FindByEmail(email string) (*types.User, error)
FindByUsername(username string) (*types.User, error)
FindByID(id uint64) (*types.User, error)
FindByIDs(id ...uint64) (types.UserSet, error)
FindBySatosaID(id string) (*types.User, error)
Find(filter *types.UserFilter) ([]*types.User, error)
@@ -87,6 +89,21 @@ func (r *user) FindByID(id uint64) (*types.User, error) {
return mod, r.prepare(mod, "teams")
}
func (r *user) FindByIDs(IDs ...uint64) (uu types.UserSet, err error) {
if len(IDs) == 0 {
return
}
sql := fmt.Sprintf(sqlUserSelect, r.users) + " AND id IN (?)"
if sql, args, err := sqlx.In(sql, IDs); err != nil {
return nil, err
} else {
return uu, r.db().Select(&uu, sql, args...)
}
}
func (r *user) Find(filter *types.UserFilter) ([]*types.User, error) {
rval := make([]*types.User, 0)
params := make([]interface{}, 0)

5
system/service/user.go
View File

@@ -31,6 +31,7 @@ type (
FindByUsername(username string) (*types.User, error)
FindByEmail(email string) (*types.User, error)
FindByID(id uint64) (*types.User, error)
FindByIDs(id ...uint64) (types.UserSet, error)
Find(filter *types.UserFilter) (types.UserSet, error)
FindOrCreate(*types.User) (*types.User, error)
@@ -93,6 +94,10 @@ func (svc *user) FindByID(id uint64) (*types.User, error) {
return svc.user.FindByID(id)
}
func (svc *user) FindByIDs(ids ...uint64) (types.UserSet, error) {
return svc.user.FindByIDs(ids...)
}
func (svc *user) FindByEmail(email string) (*types.User, error) {
return svc.user.FindByEmail(email)
}