Merge branch 'record-import'

Gopkg.lock

@@ -100,6 +100,18 @@
   pruneopts = "UT"
   revision = "b57537c92a6b4155c5219ce0d7d2a9c0cc74b1c4"
 
+[[projects]]
+  digest = "1:04904b7a6f24221e7999b554094e347885b4837beeba9933efadc93f4f5421ff"
+  name = "github.com/gabriel-vasile/mimetype"
+  packages = [
+    ".",
+    "internal/json",
+    "internal/matchers",
+  ]
+  pruneopts = "UT"
+  revision = "b9686d36d26efccccd34eff91a19f42a61bcdc95"
+  version = "v0.3.17"
+
 [[projects]]
   digest = "1:f59f34bb582fbc13885cd0338570bc0f21738fc711065527c36928213de62c1b"
   name = "github.com/getsentry/sentry-go"
@@ -690,6 +702,7 @@
     "github.com/dgrijalva/jwt-go",
     "github.com/disintegration/imaging",
     "github.com/edwvee/exiffix",
+    "github.com/gabriel-vasile/mimetype",
     "github.com/getsentry/sentry-go",
     "github.com/getsentry/sentry-go/http",
     "github.com/go-chi/chi",

Gopkg.toml

@@ -29,3 +29,7 @@ required = [
   [[prune.project]]
     name = "github.com/cortezaproject/corteza-protobuf"
     unused-packages = false
+
+[[constraint]]
+  name = "github.com/gabriel-vasile/mimetype"
+  version = "0.3.17"

api/compose/spec.json

@@ -667,6 +667,68 @@
                     ]
                 }
             },
+            {
+                "name": "importInit",
+                "path": "/import",
+                "method": "POST",
+                "title": "Initiate record import session",
+                "parameters": {
+                    "post": [
+                        {
+                            "name": "upload",
+                            "type": "*multipart.FileHeader",
+                            "required": true,
+                            "title": "File import"
+                        }
+                    ]
+                }
+            },
+            {
+                "name": "importRun",
+                "path": "/import/{sessionID}",
+                "method": "PATCH",
+                "title": "Run record import",
+                "parameters": {
+                    "path": [
+                        {
+                            "name": "sessionID",
+                            "type": "uint64",
+                            "required": true,
+                            "title": "Import session"
+                        }
+                    ],
+                    "post": [
+                        {
+                            "name": "fields",
+                            "type": "json.RawMessage",
+                            "required": true,
+                            "title": "Fields defined by import file"
+                        },
+                        {
+                            "name": "onError",
+                            "type": "string",
+                            "required": true,
+                            "title": "What happens if record fails to import"
+                        }
+                    ]
+                }
+            },
+            {
+                "name": "importProgress",
+                "path": "/import/{sessionID}",
+                "method": "GET",
+                "title": "Get import progress",
+                "parameters": {
+                    "path": [
+                        {
+                            "name": "sessionID",
+                            "type": "uint64",
+                            "required": true,
+                            "title": "Import session"
+                        }
+                    ]
+                }
+            },
             {
                 "name": "export",
                 "path": "/export{filename}.{ext}",

api/compose/spec/record.json

@@ -91,6 +91,68 @@
         ]
       }
     },
+    {
+      "Name": "importInit",
+      "Method": "POST",
+      "Title": "Initiate record import session",
+      "Path": "/import",
+      "Parameters": {
+        "post": [
+          {
+            "name": "upload",
+            "required": true,
+            "title": "File import",
+            "type": "*multipart.FileHeader"
+          }
+        ]
+      }
+    },
+    {
+      "Name": "importRun",
+      "Method": "PATCH",
+      "Title": "Run record import",
+      "Path": "/import/{sessionID}",
+      "Parameters": {
+        "path": [
+          {
+            "name": "sessionID",
+            "required": true,
+            "title": "Import session",
+            "type": "uint64"
+          }
+        ],
+        "post": [
+          {
+            "name": "fields",
+            "required": true,
+            "title": "Fields defined by import file",
+            "type": "json.RawMessage"
+          },
+          {
+            "name": "onError",
+            "required": true,
+            "title": "What happens if record fails to import",
+            "type": "string"
+          }
+        ]
+      }
+    },
+    {
+      "Name": "importProgress",
+      "Method": "GET",
+      "Title": "Get import progress",
+      "Path": "/import/{sessionID}",
+      "Parameters": {
+        "path": [
+          {
+            "name": "sessionID",
+            "required": true,
+            "title": "Import session",
+            "type": "uint64"
+          }
+        ]
+      }
+    },
     {
       "Name": "export",
       "Method": "GET",

compose/decoder/decoder.go

@@ -0,0 +1,169 @@
+package decoder
+
+import (
+	"io"
+
+	"github.com/cortezaproject/corteza-server/pkg/count"
+)
+
+type (
+	multiple uint
+
+	FlatReader interface {
+		Read() ([]string, error)
+	}
+
+	StructuredDecoder interface {
+		Decode(interface{}) error
+		More() bool
+	}
+
+	flatReader struct {
+		f      io.ReadSeeker
+		r      FlatReader
+		header []string
+		more   bool
+	}
+
+	structuredDecoder struct {
+		f      io.ReadSeeker
+		header []string
+		d      StructuredDecoder
+		buf    []map[string]interface{}
+	}
+
+	// callbacks
+	sdCallback func(map[string]interface{}) error
+	fdCallback func([]string) error
+)
+
+// flat reader
+func NewFlatReader(r FlatReader, f io.ReadSeeker) *flatReader {
+	return &flatReader{
+		f:    f,
+		r:    r,
+		more: true,
+	}
+}
+
+func (dec *flatReader) EntryCount() (uint64, error) {
+	defer dec.f.Seek(0, 0)
+
+	c, err := count.Lines(dec.f)
+	if err != nil {
+		return 0, err
+	}
+	if c <= 0 {
+		return 0, nil
+	}
+	return c - 1, nil
+}
+
+func (dec *flatReader) get(fnc fdCallback) error {
+	v, err := dec.r.Read()
+	if err == io.EOF {
+		dec.more = false
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	return fnc(v)
+}
+
+func (dec *flatReader) walk(fnc fdCallback) error {
+	for dec.more {
+		if err := dec.get(fnc); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (dec *flatReader) Header() []string {
+	if len(dec.header) > 0 {
+		return dec.header
+	}
+
+	dec.get(func(rtr []string) error {
+		dec.header = rtr
+		return nil
+	})
+
+	return dec.header
+}
+
+// structured decoder
+func NewStructuredDecoder(d StructuredDecoder, f io.ReadSeeker) *structuredDecoder {
+	return &structuredDecoder{
+		f: f,
+		d: d,
+	}
+}
+
+func (dec *structuredDecoder) EntryCount() (uint64, error) {
+	defer dec.f.Seek(0, 0)
+	return count.Lines(dec.f)
+}
+
+func (dec *structuredDecoder) get(fnc sdCallback) error {
+	if !dec.d.More() {
+		return nil
+	}
+
+	var tmp map[string]interface{}
+	err := dec.d.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+
+	return fnc(tmp)
+}
+
+func (dec *structuredDecoder) exhaustBuffer(fnc sdCallback) error {
+	if dec.buf != nil {
+		for _, b := range dec.buf {
+			fnc(b)
+		}
+		dec.buf = nil
+	}
+	return nil
+}
+
+func (dec *structuredDecoder) walk(fnc sdCallback) error {
+	if err := dec.exhaustBuffer(fnc); err != nil {
+		return err
+	}
+
+	for dec.d.More() {
+		if err := dec.get(fnc); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (dec *structuredDecoder) Header() []string {
+	if len(dec.header) > 0 {
+		return dec.header
+	}
+
+	var tmp []string
+	dec.get(func(rtr map[string]interface{}) error {
+		// buffer first row or else it will be lost
+		dec.buf = append(dec.buf, rtr)
+
+		tmp = make([]string, len(rtr))
+		i := 0
+		for k := range rtr {
+			tmp[i] = k
+			i++
+		}
+
+		return nil
+	})
+
+	dec.header = tmp
+	return tmp
+}

compose/decoder/decoder_test.go

@@ -0,0 +1,179 @@
+package decoder
+
+import (
+	"bytes"
+	"encoding/csv"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"testing"
+
+	"github.com/cortezaproject/corteza-server/internal/test"
+)
+
+func makeReadSeeker(c string) io.ReadSeeker {
+	bb := []byte(c)
+	return bytes.NewReader(bb)
+}
+
+const (
+	testCSV   string = "f1,f2,ID\nr1v1,r1v2,1\n"
+	testJSONL string = "{ \"f1\": \"nr1v1\", \"f2\": \"r1v2\", \"ID\": \"1\" }\n"
+)
+
+func TestEntryCount(t *testing.T) {
+	t.Run("Flat reader", func(t *testing.T) {
+		rs := makeReadSeeker(testCSV)
+		fr := NewFlatReader(csv.NewReader(rs), rs)
+
+		c, err := fr.EntryCount()
+		test.Assert(t,
+			c == 1,
+			fmt.Sprintf("Invalid number of entries determines; found %d, expected %d", c, 1),
+		)
+
+		test.Assert(t,
+			err == nil,
+			"Returned with error",
+		)
+	})
+
+	t.Run("Structured decoder", func(t *testing.T) {
+		rs := makeReadSeeker(testJSONL)
+		sd := NewStructuredDecoder(json.NewDecoder(rs), rs)
+
+		c, err := sd.EntryCount()
+		test.Assert(t,
+			c == 1,
+			fmt.Sprintf("Invalid number of entries determines; found %d, expected %d", c, 1),
+		)
+
+		test.Assert(t,
+			err == nil,
+			"Returned with error",
+		)
+	})
+}
+
+func TestGet(t *testing.T) {
+	t.Run("Flat reader", func(t *testing.T) {
+		rs := makeReadSeeker(testCSV)
+		fr := NewFlatReader(csv.NewReader(rs), rs)
+
+		// dump first line
+		fr.get(func(f []string) error { return nil })
+		err := fr.get(func(f []string) error {
+			return errors.New("called")
+		})
+		test.Assert(t,
+			err != nil,
+			"Error should be returned to indicate that get did read",
+		)
+
+		err = fr.get(func(f []string) error {
+			return errors.New("called")
+		})
+		test.Assert(t,
+			err == nil,
+			"Error should NOT be returned to indicate that get didn't read",
+		)
+	})
+
+	t.Run("Structured decoder", func(t *testing.T) {
+		rs := makeReadSeeker(testJSONL)
+		sd := NewStructuredDecoder(json.NewDecoder(rs), rs)
+
+		err := sd.get(func(f map[string]interface{}) error {
+			return errors.New("called")
+		})
+		test.Assert(t,
+			err != nil,
+			"Error should be returned to indicate that get did read",
+		)
+
+		err = sd.get(func(f map[string]interface{}) error {
+			return errors.New("called")
+		})
+		test.Assert(t,
+			err == nil,
+			"Error should NOT be returned to indicate that get didn't read",
+		)
+	})
+}
+
+func TestWalk(t *testing.T) {
+	t.Run("Flat reader", func(t *testing.T) {
+		rs := makeReadSeeker(testCSV)
+		fr := NewFlatReader(csv.NewReader(rs), rs)
+
+		i := 0
+		err := fr.walk(func(f []string) error {
+			i++
+			return nil
+		})
+
+		test.Assert(t,
+			i == 2,
+			"Invalid number of reads",
+		)
+
+		test.Assert(t,
+			err == nil,
+			"Returned with error",
+		)
+	})
+
+	t.Run("Structured decoder", func(t *testing.T) {
+		rs := makeReadSeeker(testJSONL)
+		sd := NewStructuredDecoder(json.NewDecoder(rs), rs)
+
+		i := 0
+		err := sd.walk(func(f map[string]interface{}) error {
+			i++
+			return nil
+		})
+
+		test.Assert(t,
+			i == 1,
+			"Invalid number of reads",
+		)
+
+		test.Assert(t,
+			err == nil,
+			"Returned with error",
+		)
+	})
+}
+
+func TestHeader(t *testing.T) {
+	t.Run("Flat reader", func(t *testing.T) {
+		rs := makeReadSeeker(testCSV)
+		fr := NewFlatReader(csv.NewReader(rs), rs)
+
+		h := fr.Header()
+		test.Assert(t,
+			len(h) == 3,
+			"Invalid number of header fields",
+		)
+
+		expect := [...]string{"f1", "f2", "ID"}
+		for i, h := range h {
+			test.Assert(t,
+				h == expect[i],
+				"Invalid header value",
+			)
+		}
+	})
+
+	t.Run("Structured decoder", func(t *testing.T) {
+		rs := makeReadSeeker(testJSONL)
+		sd := NewStructuredDecoder(json.NewDecoder(rs), rs)
+
+		h := sd.Header()
+		test.Assert(t,
+			len(h) == 3,
+			"Invalid number of header fields",
+		)
+	})
+}

compose/decoder/record.go

@@ -0,0 +1,135 @@
+package decoder
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/cortezaproject/corteza-server/compose/types"
+)
+
+type (
+	RecordCreator func(mod *types.Record) error
+)
+
+func fmtTime(tp string) (time.Time, error) {
+	return time.Parse(time.RFC3339, tp)
+}
+func fmtTimePtr(tp string) (*time.Time, error) {
+	t, err := fmtTime(tp)
+	if err != nil {
+		return nil, err
+	}
+	return &t, nil
+}
+
+func mapify(header, values []string) map[string]string {
+	if len(header) != len(values) {
+		return nil
+	}
+
+	rtr := make(map[string]string)
+	for i, v := range values {
+		rtr[header[i]] = v
+	}
+
+	return rtr
+}
+
+func setSystemField(r *types.Record, name, value string) (is bool, err error) {
+	switch name {
+	case "recordID", "ID":
+		r.ID, err = strconv.ParseUint(value, 10, 64)
+	case "moduleID":
+		r.ModuleID, err = strconv.ParseUint(value, 10, 64)
+	case "namespaceID":
+		r.NamespaceID, err = strconv.ParseUint(value, 10, 64)
+	case "ownedBy":
+		r.OwnedBy, err = strconv.ParseUint(value, 10, 64)
+	case "createdBy":
+		r.CreatedBy, err = strconv.ParseUint(value, 10, 64)
+	case "createdAt":
+		r.CreatedAt, err = fmtTime(value)
+	case "updatedBy":
+		r.UpdatedBy, err = strconv.ParseUint(value, 10, 64)
+	case "updatedAt":
+		r.UpdatedAt, err = fmtTimePtr(value)
+	case "deletedBy":
+		r.DeletedBy, err = strconv.ParseUint(value, 10, 64)
+	case "deletedAt":
+		r.DeletedAt, err = fmtTimePtr(value)
+	default:
+		return false, err
+	}
+	return true, err
+}
+
+func (dec flatReader) Records(fields map[string]string, Create RecordCreator) error {
+	header := dec.Header()
+
+	err := dec.walk(func(row []string) error {
+		mapped := mapify(header, row)
+		r := types.Record{}
+		rvs := types.RecordValueSet{}
+
+		i := 0
+		for imp, rec := range fields {
+			if rec == "" {
+				return errors.New("Can not import record: Record field not defined")
+			}
+
+			val := mapped[imp]
+			if system, err := setSystemField(&r, rec, val); err != nil {
+				return err
+			} else if !system {
+				rv := types.RecordValue{
+					Name:  rec,
+					Value: val,
+					Place: uint(i),
+				}
+				i++
+
+				rvs = append(rvs, &rv)
+			}
+		}
+
+		r.Values = rvs
+		return Create(&r)
+	})
+
+	return err
+}
+
+func (dec structuredDecoder) Records(fields map[string]string, Create RecordCreator) error {
+	err := dec.walk(func(entry map[string]interface{}) error {
+		r := types.Record{}
+		rvs := types.RecordValueSet{}
+
+		i := 0
+		for imp, rec := range fields {
+			if rec == "" {
+				return errors.New("Can not import record: Record field not defined")
+			}
+
+			val := fmt.Sprintf("%v", entry[imp])
+			if system, err := setSystemField(&r, rec, val); err != nil {
+				return err
+			} else if !system {
+				rv := types.RecordValue{
+					Name:  rec,
+					Value: val,
+					Place: uint(i),
+				}
+				i++
+
+				rvs = append(rvs, &rv)
+			}
+		}
+
+		r.Values = rvs
+		return Create(&r)
+	})
+
+	return err
+}

compose/decoder/record_test.go

@@ -0,0 +1,135 @@
+package decoder
+
+import (
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/cortezaproject/corteza-server/compose/types"
+	"github.com/cortezaproject/corteza-server/internal/test"
+)
+
+func TestMapify(t *testing.T) {
+	t.Run("Fail if lengths missmatch", func(t *testing.T) {
+		var h []string
+		h = append(h, "h1", "h2")
+
+		var v []string
+		v = append(v, "v1")
+
+		test.Assert(t,
+			mapify(h, v) == nil,
+			"Value should be nil",
+		)
+
+		test.Assert(t,
+			mapify(v, h) == nil,
+			"Value should be nil",
+		)
+	})
+
+	t.Run("Successfully mapped", func(t *testing.T) {
+		var h []string
+		h = append(h, "h1", "h2")
+
+		var v []string
+		v = append(v, "v1", "v2")
+
+		mpd := mapify(h, v)
+		test.Assert(t,
+			len(mpd) == 2,
+			fmt.Sprintf("Invalid length %d; should be %d", len(mpd), 2),
+		)
+
+		test.Assert(t,
+			mpd["h1"] == "v1" && mpd["h2"] == "v2",
+			"Invalid values",
+		)
+	})
+}
+
+func TestSetSystemField(t *testing.T) {
+	t.Run("Correctly determine & set", func(t *testing.T) {
+		r := &types.Record{}
+		name := "recordID"
+		value := "123"
+		is, err := setSystemField(r, name, value)
+		test.Assert(t,
+			err == nil,
+			"Returned with error",
+		)
+
+		test.Assert(t,
+			is,
+			"Couldn't determine it's a system field",
+		)
+
+		test.Assert(t,
+			r.ID == 123,
+			fmt.Sprintf("Determined value (%d) not valid; should be %s", r.ID, value),
+		)
+	})
+
+	t.Run("Correctly determine that it's not", func(t *testing.T) {
+		r := &types.Record{}
+		name := "customField"
+		value := "123"
+		is, err := setSystemField(r, name, value)
+		test.Assert(t,
+			err == nil,
+			"Returned with error",
+		)
+
+		test.Assert(t,
+			!is,
+			"Couldn't determine it's not a system field",
+		)
+	})
+}
+
+func TestRecords(t *testing.T) {
+	testFields := make(map[string]string)
+	testFields["f1"] = "f1"
+	testFields["f2"] = "f2"
+	testFields["ID"] = "ID"
+
+	t.Run("Flat reader", func(t *testing.T) {
+		fr := NewFlatReader(csv.NewReader(strings.NewReader(testCSV)), nil)
+		row := 0
+		fr.Records(testFields, func(mod *types.Record) error {
+			row++
+			test.Assert(t,
+				len(mod.Values) == 2,
+				"Not enough values",
+			)
+
+			test.Assert(t,
+				mod.ID == uint64(row),
+				"Not enough values",
+			)
+
+			return nil
+		})
+	})
+
+	t.Run("Structured decoder", func(t *testing.T) {
+		sd := NewStructuredDecoder(json.NewDecoder(strings.NewReader(testJSONL)), nil)
+		row := 0
+		sd.Records(testFields, func(mod *types.Record) error {
+			row++
+			test.Assert(t,
+				len(mod.Values) == 2,
+				"Not enough values",
+			)
+
+			test.Assert(t,
+				mod.ID == uint64(row),
+				"Not enough values",
+			)
+
+			return nil
+		})
+	})
+}

compose/internal/service/import_session.go

@@ -0,0 +1,133 @@
+package service
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/cortezaproject/corteza-server/internal/auth"
+	"github.com/titpetric/factory"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+)
+
+type (
+	recordSet []*RecordImportSession
+
+	importSession struct {
+		l      sync.Mutex
+		logger *zap.Logger
+
+		records recordSet
+	}
+
+	ImportSessionService interface {
+		FindRecordByID(ctx context.Context, sessionID uint64) (*RecordImportSession, error)
+		SetRecordByID(ctx context.Context, sessionID, namespaceID, moduleID uint64, fields map[string]string, progress *RecordImportProgress, decoder Decoder) (*RecordImportSession, error)
+		DeleteRecordByID(ctx context.Context, sessionID uint64) error
+	}
+)
+
+func ImportSession() *importSession {
+	return &importSession{
+		logger:  DefaultLogger.Named("importSession"),
+		records: recordSet{},
+	}
+}
+
+func (svc importSession) indexOf(userID, sessionID uint64) int {
+	for i, r := range svc.records {
+		if r.SessionID == sessionID && r.UserID == userID {
+			return i
+		}
+	}
+
+	return -1
+}
+
+func (svc *importSession) FindRecordByID(ctx context.Context, sessionID uint64) (*RecordImportSession, error) {
+	svc.l.Lock()
+	defer svc.l.Unlock()
+
+	userID := auth.GetIdentityFromContext(ctx).Identity()
+	i := svc.indexOf(userID, sessionID)
+	if i >= 0 {
+		return svc.records[i], nil
+	}
+	return nil, errors.New("Can't access session: session not found")
+}
+
+func (svc *importSession) SetRecordByID(ctx context.Context, sessionID, namespaceID, moduleID uint64, fields map[string]string, progress *RecordImportProgress, decoder Decoder) (*RecordImportSession, error) {
+	svc.l.Lock()
+	defer svc.l.Unlock()
+
+	userID := auth.GetIdentityFromContext(ctx).Identity()
+	i := svc.indexOf(userID, sessionID)
+	var ris *RecordImportSession
+
+	if i >= 0 {
+		ris = svc.records[i]
+	} else {
+		ris = &RecordImportSession{
+			SessionID: factory.Sonyflake.NextID(),
+			CreatedAt: time.Now(),
+		}
+		svc.records = append(svc.records, ris)
+		ris.UserID = userID
+	}
+	ris.UpdatedAt = time.Now()
+
+	if namespaceID > 0 {
+		ris.NamespaceID = namespaceID
+	}
+	if moduleID > 0 {
+		ris.ModuleID = moduleID
+	}
+	if fields != nil {
+		ris.Fields = fields
+	}
+	if progress != nil {
+		ris.Progress = *progress
+	}
+
+	if ris.Progress.FinishedAt != nil {
+		ris.Decoder = nil
+	} else if decoder != nil {
+		ris.Decoder = decoder
+	}
+
+	return ris, nil
+}
+
+// https://stackoverflow.com/a/37335777
+func remove(s recordSet, i int) recordSet {
+	s[len(s)-1], s[i] = s[i], s[len(s)-1]
+	return s[:len(s)-1]
+}
+
+func (svc *importSession) DeleteRecordByID(ctx context.Context, sessionID uint64) error {
+	svc.l.Lock()
+	defer svc.l.Unlock()
+
+	userID := auth.GetIdentityFromContext(ctx).Identity()
+	i := svc.indexOf(userID, sessionID)
+
+	if i >= 0 {
+		svc.records = remove(svc.records, i)
+	}
+	return nil
+}
+
+// @todo run this in some interval
+func (svc *importSession) clean(ctx context.Context) {
+	svc.l.Lock()
+	defer svc.l.Unlock()
+
+	for i := len(svc.records) - 1; i >= 0; i-- {
+		r := svc.records[i]
+		if time.Now().After(r.UpdatedAt.Add(time.Hour * 24 * 3)) {
+			svc.records = remove(svc.records, i)
+		}
+	}
+}

compose/internal/service/import_session_test.go

@@ -0,0 +1,111 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	"go.uber.org/zap"
+
+	"github.com/cortezaproject/corteza-server/internal/test"
+)
+
+var ctx context.Context = context.WithValue(context.Background(), "testing", true)
+
+func TestFindRecordByID(t *testing.T) {
+	DefaultLogger = zap.New(nil, nil)
+	svc := ImportSession()
+	ss, _ := svc.SetRecordByID(ctx, 1, 0, 0, nil, nil, nil)
+	sid := ss.SessionID
+
+	t.Run("Found", func(t *testing.T) {
+		s, err := svc.FindRecordByID(ctx, sid)
+		test.Assert(t,
+			s != nil,
+			"Session should be found",
+		)
+
+		test.Assert(t,
+			err == nil,
+			"Returned with error",
+		)
+	})
+
+	t.Run("Not found", func(t *testing.T) {
+		s, err := svc.FindRecordByID(ctx, sid+1)
+		test.Assert(t,
+			s == nil,
+			"Session should not be found",
+		)
+
+		test.Assert(t,
+			err != nil,
+			"Error should not be nil",
+		)
+	})
+}
+
+func TestSetRecordByID(t *testing.T) {
+	DefaultLogger = zap.New(nil, nil)
+	svc := ImportSession()
+
+	t.Run("New", func(t *testing.T) {
+		ss, err := svc.SetRecordByID(ctx, 1, 0, 0, nil, nil, nil)
+		test.Assert(t,
+			len(svc.records) == 1 && ss != nil,
+			"Session should be created",
+		)
+
+		test.Assert(t,
+			err == nil,
+			"Returned with error",
+		)
+	})
+
+	t.Run("Existing", func(t *testing.T) {
+		svc := ImportSession()
+		ss, err := svc.SetRecordByID(ctx, 1, 0, 0, nil, nil, nil)
+		ns, err := svc.SetRecordByID(ctx, ss.SessionID, 0, 0, nil, nil, nil)
+		test.Assert(t,
+			len(svc.records) == 1 && ns != nil && ss.SessionID == ns.SessionID,
+			"Existing session should be edited",
+		)
+
+		test.Assert(t,
+			err == nil,
+			"Returned with error",
+		)
+	})
+}
+
+func TestDeleteRecordByID(t *testing.T) {
+	DefaultLogger = zap.New(nil, nil)
+	svc := ImportSession()
+	ss, _ := svc.SetRecordByID(ctx, 1, 0, 0, nil, nil, nil)
+
+	t.Run("Delete existing", func(t *testing.T) {
+		err := svc.DeleteRecordByID(ctx, ss.SessionID)
+		test.Assert(t,
+			len(svc.records) == 0,
+			"Session should be deleted",
+		)
+
+		test.Assert(t,
+			err == nil,
+			"Returned with error",
+		)
+	})
+
+	t.Run("Session not found", func(t *testing.T) {
+		ss, _ := svc.SetRecordByID(ctx, 1, 0, 0, nil, nil, nil)
+		err := svc.DeleteRecordByID(ctx, ss.SessionID+1)
+		test.Assert(t,
+			len(svc.records) == 1,
+			"Session should not deleted",
+		)
+
+		test.Assert(t,
+			err == nil,
+			"Returned with error",
+		)
+	})
+}

compose/internal/service/record.go

@@ -10,11 +10,17 @@ import (
 	"github.com/titpetric/factory"
 	"go.uber.org/zap"
 
+	"github.com/cortezaproject/corteza-server/compose/decoder"
 	"github.com/cortezaproject/corteza-server/compose/internal/repository"
 	"github.com/cortezaproject/corteza-server/compose/types"
 	"github.com/cortezaproject/corteza-server/internal/auth"
 )
 
+const (
+	IMPORT_ON_ERROR_SKIP = "SKIP"
+	IMPORT_ON_ERROR_FAIL = "FAIL"
+)
+
 type (
 	record struct {
 		db     *factory.DB
@@ -57,6 +63,7 @@ type (
 		Report(namespaceID, moduleID uint64, metrics, dimensions, filter string) (interface{}, error)
 		Find(filter types.RecordFilter) (set types.RecordSet, f types.RecordFilter, err error)
 		Export(types.RecordFilter, Encoder) error
+		Import(*RecordImportSession, ImportSessionService) error
 
 		Create(record *types.Record) (*types.Record, error)
 		Update(record *types.Record) (*types.Record, error)
@@ -67,6 +74,34 @@ type (
 	Encoder interface {
 		Record(*types.Record) error
 	}
+
+	Decoder interface {
+		Header() []string
+		EntryCount() (uint64, error)
+		Records(fields map[string]string, Create decoder.RecordCreator) error
+	}
+
+	RecordImportSession struct {
+		Decoder     Decoder              `json:"-"`
+		CreatedAt   time.Time            `json:"createdAt"`
+		UpdatedAt   time.Time            `json:"updatedAt"`
+		OnError     string               `json:"onError"`
+		SessionID   uint64               `json:"sessionID,string"`
+		UserID      uint64               `json:"userID,string"`
+		NamespaceID uint64               `json:"namespaceID,string"`
+		ModuleID    uint64               `json:"moduleID,string"`
+		Fields      map[string]string    `json:"fields"`
+		Progress    RecordImportProgress `json:"progress"`
+	}
+
+	RecordImportProgress struct {
+		StartedAt  *time.Time `json:"startedAt"`
+		FinishedAt *time.Time `json:"finishedAt"`
+		EntryCount uint64     `json:"entryCount"`
+		Completed  uint64     `json:"completed"`
+		Failed     uint64     `json:"failed"`
+		FailReason string     `json:"failReason,omitempty"`
+	}
 )
 
 func Record() RecordService {
@@ -184,6 +219,49 @@ func (svc record) Find(filter types.RecordFilter) (set types.RecordSet, f types.
 	return
 }
 
+func (svc record) Import(ses *RecordImportSession, ssvc ImportSessionService) error {
+	if ses.Decoder == nil {
+		return nil
+	}
+
+	if ses.Progress.StartedAt != nil {
+		return errors.New("Unable to start import: Import session already active")
+	}
+
+	sa := time.Now()
+	ses.Progress.StartedAt = &sa
+	ssvc.SetRecordByID(svc.ctx, ses.SessionID, 0, 0, nil, &ses.Progress, nil)
+
+	return svc.db.Transaction(func() (err error) {
+		err = ses.Decoder.Records(ses.Fields, func(mod *types.Record) error {
+			mod.NamespaceID = ses.NamespaceID
+			mod.ModuleID = ses.ModuleID
+			mod.OwnedBy = ses.UserID
+
+			_, err := svc.Create(mod)
+			if err != nil {
+				ses.Progress.Failed++
+				ses.Progress.FailReason = err.Error()
+
+				if ses.OnError == IMPORT_ON_ERROR_FAIL {
+					fa := time.Now()
+					ses.Progress.FinishedAt = &fa
+					ssvc.SetRecordByID(svc.ctx, ses.SessionID, 0, 0, nil, &ses.Progress, nil)
+					return err
+				}
+			} else {
+				ses.Progress.Completed++
+			}
+			return nil
+		})
+
+		fa := time.Now()
+		ses.Progress.FinishedAt = &fa
+		ssvc.SetRecordByID(svc.ctx, ses.SessionID, 0, 0, nil, &ses.Progress, nil)
+		return
+	})
+}
+
 // Export returns all records
 //
 // @todo better value handling

compose/internal/service/service.go

@@ -47,14 +47,14 @@ var (
 	// DefaultAutomationRunner runs automation scripts by listening to triggerManager and invoking Corredor service
 	DefaultAutomationRunner automationRunner
 
-	DefaultNamespace NamespaceService
-	DefaultRecord    RecordService
-	DefaultModule    ModuleService
-	DefaultChart     ChartService
-	DefaultPage      PageService
-
-	DefaultAttachment   AttachmentService
-	DefaultNotification NotificationService
+	DefaultNamespace     NamespaceService
+	DefaultImportSession ImportSessionService
+	DefaultRecord        RecordService
+	DefaultModule        ModuleService
+	DefaultChart         ChartService
+	DefaultPage          PageService
+	DefaultAttachment    AttachmentService
+	DefaultNotification  NotificationService
 
 	DefaultSystemUser *systemUser
 )
@@ -130,6 +130,7 @@ func Init(ctx context.Context, log *zap.Logger, c Config) (err error) {
 		)
 	}
 
+	DefaultImportSession = ImportSession()
 	DefaultRecord = Record()
 	DefaultPage = Page()
 	DefaultChart = Chart()

compose/rest/handlers/record.go

@@ -31,6 +31,9 @@ import (
 type RecordAPI interface {
 	Report(context.Context, *request.RecordReport) (interface{}, error)
 	List(context.Context, *request.RecordList) (interface{}, error)
+	ImportInit(context.Context, *request.RecordImportInit) (interface{}, error)
+	ImportRun(context.Context, *request.RecordImportRun) (interface{}, error)
+	ImportProgress(context.Context, *request.RecordImportProgress) (interface{}, error)
 	Export(context.Context, *request.RecordExport) (interface{}, error)
 	Create(context.Context, *request.RecordCreate) (interface{}, error)
 	Read(context.Context, *request.RecordRead) (interface{}, error)
@@ -41,14 +44,17 @@ type RecordAPI interface {
 
 // HTTP API interface
 type Record struct {
-	Report func(http.ResponseWriter, *http.Request)
-	List   func(http.ResponseWriter, *http.Request)
-	Export func(http.ResponseWriter, *http.Request)
-	Create func(http.ResponseWriter, *http.Request)
-	Read   func(http.ResponseWriter, *http.Request)
-	Update func(http.ResponseWriter, *http.Request)
-	Delete func(http.ResponseWriter, *http.Request)
-	Upload func(http.ResponseWriter, *http.Request)
+	Report         func(http.ResponseWriter, *http.Request)
+	List           func(http.ResponseWriter, *http.Request)
+	ImportInit     func(http.ResponseWriter, *http.Request)
+	ImportRun      func(http.ResponseWriter, *http.Request)
+	ImportProgress func(http.ResponseWriter, *http.Request)
+	Export         func(http.ResponseWriter, *http.Request)
+	Create         func(http.ResponseWriter, *http.Request)
+	Read           func(http.ResponseWriter, *http.Request)
+	Update         func(http.ResponseWriter, *http.Request)
+	Delete         func(http.ResponseWriter, *http.Request)
+	Upload         func(http.ResponseWriter, *http.Request)
 }
 
 func NewRecord(h RecordAPI) *Record {
@@ -93,6 +99,66 @@ func NewRecord(h RecordAPI) *Record {
 				resputil.JSON(w, value)
 			}
 		},
+		ImportInit: func(w http.ResponseWriter, r *http.Request) {
+			defer r.Body.Close()
+			params := request.NewRecordImportInit()
+			if err := params.Fill(r); err != nil {
+				logger.LogParamError("Record.ImportInit", r, err)
+				resputil.JSON(w, err)
+				return
+			}
+
+			value, err := h.ImportInit(r.Context(), params)
+			if err != nil {
+				logger.LogControllerError("Record.ImportInit", r, err, params.Auditable())
+				resputil.JSON(w, err)
+				return
+			}
+			logger.LogControllerCall("Record.ImportInit", r, params.Auditable())
+			if !serveHTTP(value, w, r) {
+				resputil.JSON(w, value)
+			}
+		},
+		ImportRun: func(w http.ResponseWriter, r *http.Request) {
+			defer r.Body.Close()
+			params := request.NewRecordImportRun()
+			if err := params.Fill(r); err != nil {
+				logger.LogParamError("Record.ImportRun", r, err)
+				resputil.JSON(w, err)
+				return
+			}
+
+			value, err := h.ImportRun(r.Context(), params)
+			if err != nil {
+				logger.LogControllerError("Record.ImportRun", r, err, params.Auditable())
+				resputil.JSON(w, err)
+				return
+			}
+			logger.LogControllerCall("Record.ImportRun", r, params.Auditable())
+			if !serveHTTP(value, w, r) {
+				resputil.JSON(w, value)
+			}
+		},
+		ImportProgress: func(w http.ResponseWriter, r *http.Request) {
+			defer r.Body.Close()
+			params := request.NewRecordImportProgress()
+			if err := params.Fill(r); err != nil {
+				logger.LogParamError("Record.ImportProgress", r, err)
+				resputil.JSON(w, err)
+				return
+			}
+
+			value, err := h.ImportProgress(r.Context(), params)
+			if err != nil {
+				logger.LogControllerError("Record.ImportProgress", r, err, params.Auditable())
+				resputil.JSON(w, err)
+				return
+			}
+			logger.LogControllerCall("Record.ImportProgress", r, params.Auditable())
+			if !serveHTTP(value, w, r) {
+				resputil.JSON(w, value)
+			}
+		},
 		Export: func(w http.ResponseWriter, r *http.Request) {
 			defer r.Body.Close()
 			params := request.NewRecordExport()
@@ -221,6 +287,9 @@ func (h Record) MountRoutes(r chi.Router, middlewares ...func(http.Handler) http
 		r.Use(middlewares...)
 		r.Get("/namespace/{namespaceID}/module/{moduleID}/record/report", h.Report)
 		r.Get("/namespace/{namespaceID}/module/{moduleID}/record/", h.List)
+		r.Post("/namespace/{namespaceID}/module/{moduleID}/record/import", h.ImportInit)
+		r.Patch("/namespace/{namespaceID}/module/{moduleID}/record/import/{sessionID}", h.ImportRun)
+		r.Get("/namespace/{namespaceID}/module/{moduleID}/record/import/{sessionID}", h.ImportProgress)
 		r.Get("/namespace/{namespaceID}/module/{moduleID}/record/export{filename}.{ext}", h.Export)
 		r.Post("/namespace/{namespaceID}/module/{moduleID}/record/", h.Create)
 		r.Get("/namespace/{namespaceID}/module/{moduleID}/record/{recordID}", h.Read)

compose/rest/record.go

@@ -12,11 +12,13 @@ import (
 
 	"github.com/pkg/errors"
 
+	"github.com/cortezaproject/corteza-server/compose/decoder"
 	"github.com/cortezaproject/corteza-server/compose/encoder"
 	"github.com/cortezaproject/corteza-server/compose/internal/repository"
 	"github.com/cortezaproject/corteza-server/compose/internal/service"
 	"github.com/cortezaproject/corteza-server/compose/rest/request"
 	"github.com/cortezaproject/corteza-server/compose/types"
+	"github.com/cortezaproject/corteza-server/pkg/mime"
 	"github.com/cortezaproject/corteza-server/pkg/rh"
 )
 
@@ -36,10 +38,11 @@ type (
 	}
 
 	Record struct {
-		record     service.RecordService
-		module     service.ModuleService
-		attachment service.AttachmentService
-		ac         recordAccessController
+		importSession service.ImportSessionService
+		record        service.RecordService
+		module        service.ModuleService
+		attachment    service.AttachmentService
+		ac            recordAccessController
 	}
 
 	recordAccessController interface {
@@ -50,10 +53,11 @@ type (
 
 func (Record) New() *Record {
 	return &Record{
-		record:     service.DefaultRecord,
-		module:     service.DefaultModule,
-		attachment: service.DefaultAttachment,
-		ac:         service.DefaultAccessControl,
+		importSession: service.DefaultImportSession,
+		record:        service.DefaultRecord,
+		module:        service.DefaultModule,
+		attachment:    service.DefaultAttachment,
+		ac:            service.DefaultAccessControl,
 	}
 }
 
@@ -167,6 +171,114 @@ func (ctrl *Record) Upload(ctx context.Context, r *request.RecordUpload) (interf
 	return makeAttachmentPayload(ctx, a, err)
 }
 
+func (ctrl *Record) ImportInit(ctx context.Context, r *request.RecordImportInit) (interface{}, error) {
+	var (
+		err           error
+		recordDecoder service.Decoder
+		entryCount    uint64
+	)
+
+	// Access control.
+	if _, err = ctrl.module.With(ctx).FindByID(r.NamespaceID, r.ModuleID); err != nil {
+		return nil, err
+	}
+
+	f, err := r.Upload.Open()
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	_, ext, err := mime.Type(f)
+	if err != nil {
+		return nil, err
+	}
+
+	if ext == "txt" {
+		if is, err := mime.JsonL(f); err != nil {
+			return nil, err
+		} else if is {
+			ext = "jsonl"
+		}
+	}
+
+	// determine decoder
+	switch strings.ToLower(ext) {
+	case "json", "jsonl", "ldjson", "ndjson":
+		recordDecoder = decoder.NewStructuredDecoder(json.NewDecoder(f), f)
+
+	case "csv":
+		recordDecoder = decoder.NewFlatReader(csv.NewReader(f), f)
+
+	default:
+		return nil, errors.New(fmt.Sprintf("unsupported format (\"%s\")", ext))
+
+	}
+	entryCount, err = recordDecoder.EntryCount()
+	if err != nil {
+		return nil, err
+	}
+
+	header := recordDecoder.Header()
+	hh := make(map[string]string)
+	for _, h := range header {
+		hh[h] = ""
+	}
+
+	return ctrl.importSession.SetRecordByID(
+		ctx,
+		0,
+		r.NamespaceID,
+		r.ModuleID,
+		hh,
+		&service.RecordImportProgress{EntryCount: entryCount},
+		recordDecoder)
+}
+
+func (ctrl *Record) ImportRun(ctx context.Context, r *request.RecordImportRun) (interface{}, error) {
+	var (
+		err error
+	)
+
+	// Access control.
+	if _, err = ctrl.module.With(ctx).FindByID(r.NamespaceID, r.ModuleID); err != nil {
+		return nil, err
+	}
+
+	// Check if session ok
+	ses, err := ctrl.importSession.FindRecordByID(ctx, r.SessionID)
+	if err != nil {
+		return nil, err
+	}
+
+	if ses.Progress.StartedAt != nil {
+		return nil, errors.New("Unable to start import: Import session already active")
+	}
+
+	ses.Fields = make(map[string]string)
+	err = json.Unmarshal(r.Fields, &ses.Fields)
+	if err != nil {
+		return nil, err
+	}
+
+	ses.OnError = r.OnError
+
+	// @todo routine
+	ctrl.record.With(ctx).Import(ses, ctrl.importSession)
+
+	return ses, nil
+}
+
+func (ctrl *Record) ImportProgress(ctx context.Context, r *request.RecordImportProgress) (interface{}, error) {
+	// Get session
+	ses, err := ctrl.importSession.FindRecordByID(ctx, r.SessionID)
+	if err != nil {
+		return nil, err
+	}
+
+	return ses, nil
+}
+
 func (ctrl *Record) Export(ctx context.Context, r *request.RecordExport) (interface{}, error) {
 	type (
 		// ad-hoc interface for our encoder

compose/rest/request/record.go

@@ -175,6 +175,192 @@ func (r *RecordList) Fill(req *http.Request) (err error) {
 
 var _ RequestFiller = NewRecordList()
 
+// Record importInit request parameters
+type RecordImportInit struct {
+	Upload      *multipart.FileHeader
+	NamespaceID uint64 `json:",string"`
+	ModuleID    uint64 `json:",string"`
+}
+
+func NewRecordImportInit() *RecordImportInit {
+	return &RecordImportInit{}
+}
+
+func (r RecordImportInit) Auditable() map[string]interface{} {
+	var out = map[string]interface{}{}
+
+	out["upload.size"] = r.Upload.Size
+	out["upload.filename"] = r.Upload.Filename
+
+	out["namespaceID"] = r.NamespaceID
+	out["moduleID"] = r.ModuleID
+
+	return out
+}
+
+func (r *RecordImportInit) Fill(req *http.Request) (err error) {
+	if strings.ToLower(req.Header.Get("content-type")) == "application/json" {
+		err = json.NewDecoder(req.Body).Decode(r)
+
+		switch {
+		case err == io.EOF:
+			err = nil
+		case err != nil:
+			return errors.Wrap(err, "error parsing http request body")
+		}
+	}
+
+	if err = req.ParseMultipartForm(32 << 20); err != nil {
+		return err
+	}
+
+	get := map[string]string{}
+	post := map[string]string{}
+	urlQuery := req.URL.Query()
+	for name, param := range urlQuery {
+		get[name] = string(param[0])
+	}
+	postVars := req.Form
+	for name, param := range postVars {
+		post[name] = string(param[0])
+	}
+
+	if _, r.Upload, err = req.FormFile("upload"); err != nil {
+		return errors.Wrap(err, "error procesing uploaded file")
+	}
+
+	r.NamespaceID = parseUInt64(chi.URLParam(req, "namespaceID"))
+	r.ModuleID = parseUInt64(chi.URLParam(req, "moduleID"))
+
+	return err
+}
+
+var _ RequestFiller = NewRecordImportInit()
+
+// Record importRun request parameters
+type RecordImportRun struct {
+	SessionID   uint64 `json:",string"`
+	NamespaceID uint64 `json:",string"`
+	ModuleID    uint64 `json:",string"`
+	Fields      json.RawMessage
+	OnError     string
+}
+
+func NewRecordImportRun() *RecordImportRun {
+	return &RecordImportRun{}
+}
+
+func (r RecordImportRun) Auditable() map[string]interface{} {
+	var out = map[string]interface{}{}
+
+	out["sessionID"] = r.SessionID
+	out["namespaceID"] = r.NamespaceID
+	out["moduleID"] = r.ModuleID
+	out["fields"] = r.Fields
+	out["onError"] = r.OnError
+
+	return out
+}
+
+func (r *RecordImportRun) Fill(req *http.Request) (err error) {
+	if strings.ToLower(req.Header.Get("content-type")) == "application/json" {
+		err = json.NewDecoder(req.Body).Decode(r)
+
+		switch {
+		case err == io.EOF:
+			err = nil
+		case err != nil:
+			return errors.Wrap(err, "error parsing http request body")
+		}
+	}
+
+	if err = req.ParseForm(); err != nil {
+		return err
+	}
+
+	get := map[string]string{}
+	post := map[string]string{}
+	urlQuery := req.URL.Query()
+	for name, param := range urlQuery {
+		get[name] = string(param[0])
+	}
+	postVars := req.Form
+	for name, param := range postVars {
+		post[name] = string(param[0])
+	}
+
+	r.SessionID = parseUInt64(chi.URLParam(req, "sessionID"))
+	r.NamespaceID = parseUInt64(chi.URLParam(req, "namespaceID"))
+	r.ModuleID = parseUInt64(chi.URLParam(req, "moduleID"))
+	if val, ok := post["fields"]; ok {
+		r.Fields = json.RawMessage(val)
+	}
+	if val, ok := post["onError"]; ok {
+		r.OnError = val
+	}
+
+	return err
+}
+
+var _ RequestFiller = NewRecordImportRun()
+
+// Record importProgress request parameters
+type RecordImportProgress struct {
+	SessionID   uint64 `json:",string"`
+	NamespaceID uint64 `json:",string"`
+	ModuleID    uint64 `json:",string"`
+}
+
+func NewRecordImportProgress() *RecordImportProgress {
+	return &RecordImportProgress{}
+}
+
+func (r RecordImportProgress) Auditable() map[string]interface{} {
+	var out = map[string]interface{}{}
+
+	out["sessionID"] = r.SessionID
+	out["namespaceID"] = r.NamespaceID
+	out["moduleID"] = r.ModuleID
+
+	return out
+}
+
+func (r *RecordImportProgress) Fill(req *http.Request) (err error) {
+	if strings.ToLower(req.Header.Get("content-type")) == "application/json" {
+		err = json.NewDecoder(req.Body).Decode(r)
+
+		switch {
+		case err == io.EOF:
+			err = nil
+		case err != nil:
+			return errors.Wrap(err, "error parsing http request body")
+		}
+	}
+
+	if err = req.ParseForm(); err != nil {
+		return err
+	}
+
+	get := map[string]string{}
+	post := map[string]string{}
+	urlQuery := req.URL.Query()
+	for name, param := range urlQuery {
+		get[name] = string(param[0])
+	}
+	postVars := req.Form
+	for name, param := range postVars {
+		post[name] = string(param[0])
+	}
+
+	r.SessionID = parseUInt64(chi.URLParam(req, "sessionID"))
+	r.NamespaceID = parseUInt64(chi.URLParam(req, "namespaceID"))
+	r.ModuleID = parseUInt64(chi.URLParam(req, "moduleID"))
+
+	return err
+}
+
+var _ RequestFiller = NewRecordImportProgress()
+
 // Record export request parameters
 type RecordExport struct {
 	Filter      string

docs/compose/README.md

@@ -956,6 +956,9 @@ Compose records
 | ------ | -------- | ------- |
 | `GET` | `/namespace/{namespaceID}/module/{moduleID}/record/report` | Generates report from module records |
 | `GET` | `/namespace/{namespaceID}/module/{moduleID}/record/` | List/read records from module section |
+| `POST` | `/namespace/{namespaceID}/module/{moduleID}/record/import` | Initiate record import session |
+| `PATCH` | `/namespace/{namespaceID}/module/{moduleID}/record/import/{sessionID}` | Run record import |
+| `GET` | `/namespace/{namespaceID}/module/{moduleID}/record/import/{sessionID}` | Get import progress |
 | `GET` | `/namespace/{namespaceID}/module/{moduleID}/record/export{filename}.{ext}` | Exports records that match  |
 | `POST` | `/namespace/{namespaceID}/module/{moduleID}/record/` | Create record in module section |
 | `GET` | `/namespace/{namespaceID}/module/{moduleID}/record/{recordID}` | Read records by ID from module section |
@@ -1000,6 +1003,56 @@ Compose records
 | namespaceID | uint64 | PATH | Namespace ID | N/A | YES |
 | moduleID | uint64 | PATH | Module ID | N/A | YES |
 
+## Initiate record import session
+
+#### Method
+
+| URI | Protocol | Method | Authentication |
+| --- | -------- | ------ | -------------- |
+| `/namespace/{namespaceID}/module/{moduleID}/record/import` | HTTP/S | POST |  |
+
+#### Request parameters
+
+| Parameter | Type | Method | Description | Default | Required? |
+| --------- | ---- | ------ | ----------- | ------- | --------- |
+| upload | *multipart.FileHeader | POST | File import | N/A | YES |
+| namespaceID | uint64 | PATH | Namespace ID | N/A | YES |
+| moduleID | uint64 | PATH | Module ID | N/A | YES |
+
+## Run record import
+
+#### Method
+
+| URI | Protocol | Method | Authentication |
+| --- | -------- | ------ | -------------- |
+| `/namespace/{namespaceID}/module/{moduleID}/record/import/{sessionID}` | HTTP/S | PATCH |  |
+
+#### Request parameters
+
+| Parameter | Type | Method | Description | Default | Required? |
+| --------- | ---- | ------ | ----------- | ------- | --------- |
+| sessionID | uint64 | PATH | Import session | N/A | YES |
+| namespaceID | uint64 | PATH | Namespace ID | N/A | YES |
+| moduleID | uint64 | PATH | Module ID | N/A | YES |
+| fields | json.RawMessage | POST | Fields defined by import file | N/A | YES |
+| onError | string | POST | What happens if record fails to import | N/A | YES |
+
+## Get import progress
+
+#### Method
+
+| URI | Protocol | Method | Authentication |
+| --- | -------- | ------ | -------------- |
+| `/namespace/{namespaceID}/module/{moduleID}/record/import/{sessionID}` | HTTP/S | GET |  |
+
+#### Request parameters
+
+| Parameter | Type | Method | Description | Default | Required? |
+| --------- | ---- | ------ | ----------- | ------- | --------- |
+| sessionID | uint64 | PATH | Import session | N/A | YES |
+| namespaceID | uint64 | PATH | Namespace ID | N/A | YES |
+| moduleID | uint64 | PATH | Module ID | N/A | YES |
+
 ## Exports records that match
 
 #### Method

pkg/count/file.go

@@ -0,0 +1,28 @@
+package count
+
+import (
+	"bytes"
+	"io"
+)
+
+// Lines provides a line count
+//
+// https://stackoverflow.com/a/24563853
+func Lines(r io.ReadSeeker) (count uint64, err error) {
+	defer r.Seek(0, 0)
+	buf := make([]byte, 32*1024)
+	lineSep := []byte{'\n'}
+
+	for {
+		c, err := r.Read(buf)
+		count += uint64(bytes.Count(buf[:c], lineSep))
+
+		switch {
+		case err == io.EOF:
+			return count, nil
+
+		case err != nil:
+			return count, err
+		}
+	}
+}

pkg/mime/mime.go

@@ -0,0 +1,34 @@
+package mime
+
+import (
+	"bufio"
+	"io"
+
+	"github.com/gabriel-vasile/mimetype"
+)
+
+func Type(file io.ReadSeeker) (mt string, ext string, err error) {
+	if _, err = file.Seek(0, 0); err != nil {
+		return
+	}
+
+	// Make sure we rewind when we're done
+	defer file.Seek(0, 0)
+	return mimetype.DetectReader(file)
+}
+
+func JsonL(file io.ReadSeeker) (bool, error) {
+	// ExtractMimetype fails to detect json if jsonl is used
+	// For now check if first rune is {
+	r := bufio.NewReader(file)
+	rn, _, err := r.ReadRune()
+	defer file.Seek(0, 0)
+	if err != nil {
+		return false, err
+	}
+
+	if string(rn) == "{" {
+		return true, nil
+	}
+	return false, nil
+}

vendor/github.com/gabriel-vasile/mimetype/.gitattributes

@@ -0,0 +1 @@
+testdata/* linguist-vendored

vendor/github.com/gabriel-vasile/mimetype/.travis.yml

@@ -0,0 +1,14 @@
+language: go
+go:
+  - "1.9"
+  - "1.10"
+before_install:
+  - go get github.com/mattn/goveralls
+  - go get github.com/client9/misspell/cmd/misspell
+before_script:
+  - go tool vet .
+script:
+  - diff -u <(echo -n) <(gofmt -d ./)
+  - go test -v
+  - $GOPATH/bin/goveralls -service=travis-ci
+  - misspell -locale US -error *.md *.go

vendor/github.com/gabriel-vasile/mimetype/CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at vasile.gabriel@email.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

vendor/github.com/gabriel-vasile/mimetype/CONTRIBUTING.md

@@ -0,0 +1,12 @@
+## Contribute
+Contributions to **mimetype** are welcome. If you find an issue and you consider
+contributing, you can use the [Github issues tracker](https://github.com/gabriel-vasile/mimetype/issues)
+in order to report it, or better yet, open a pull request.
+
+Code contributions must respect these rules:
+ - code must be test covered
+ - code must be formatted using gofmt tool
+ - exported names must be documented
+
+**Important**: By submitting a pull request, you agree to allow the project
+owner to license your work under the same license as that used by the project.

vendor/github.com/gabriel-vasile/mimetype/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018, 2019 Gabriel Vasile
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/gabriel-vasile/mimetype/README.md

@@ -0,0 +1,64 @@
+<h1 align="center">
+  mimetype
+</h1>
+
+<h4 align="center">
+  A package for detecting MIME types and extensions based on magic numbers
+</h4>
+<h6 align="center">
+  No bindings, all written in pure go
+</h6>
+
+<p align="center">
+  <a href="https://travis-ci.org/gabriel-vasile/mimetype">
+    <img alt="Build Status" src="https://travis-ci.org/gabriel-vasile/mimetype.svg?branch=master">
+  </a>
+  <a href="https://godoc.org/github.com/gabriel-vasile/mimetype">
+    <img alt="Documentation" src="https://godoc.org/github.com/gabriel-vasile/mimetype?status.svg">
+  </a>
+  <a href="https://goreportcard.com/report/github.com/gabriel-vasile/mimetype">
+    <img alt="Go report card" src="https://goreportcard.com/badge/github.com/gabriel-vasile/mimetype">
+  </a>
+  <a href="https://coveralls.io/github/gabriel-vasile/mimetype?branch=master">
+    <img alt="Go report card" src="https://coveralls.io/repos/github/gabriel-vasile/mimetype/badge.svg?branch=master">
+  </a>
+  <a href="LICENSE">
+    <img alt="License" src="https://img.shields.io/badge/License-MIT-green.svg">
+  </a>
+</p>
+
+## Install
+```bash
+go get github.com/gabriel-vasile/mimetype
+```
+
+## Use
+The library exposes three functions you can use in order to detect a file type.
+See [Godoc](https://godoc.org/github.com/gabriel-vasile/mimetype) for full reference.
+```go
+func Detect(in []byte) (mime, extension string) {...}
+func DetectReader(r io.Reader) (mime, extension string, err error) {...}
+func DetectFile(file string) (mime, extension string, err error) {...}
+```
+When detecting from a `ReadSeeker` interface, such as `os.File`, make sure
+to reset the offset of the reader to the beginning if needed:
+```go
+_, err = file.Seek(0, io.SeekStart)
+```
+
+## Supported MIME types
+See [supported mimes](supported_mimes.md) for the list of detected MIME types.
+If support is needed for a specific file format, please open an [issue](https://github.com/gabriel-vasile/mimetype/issues/new/choose).
+
+## Structure
+**mimetype** uses an hierarchical structure to keep the matching functions.
+This reduces the number of calls needed for detecting the file type. The reason
+behind this choice is that there are file formats used as containers for other
+file formats. For example, Microsoft office files are just zip archives,
+containing specific metadata files.
+<div align="center">
+  <img alt="structure" src="mimetype.gif" width="88%">
+</div>
+
+## Contributing
+See [CONTRIBUTING.md](CONTRIBUTING.md).

vendor/github.com/gabriel-vasile/mimetype/go.mod

@@ -0,0 +1 @@
+module github.com/gabriel-vasile/mimetype

vendor/github.com/gabriel-vasile/mimetype/internal/json/json.go

@@ -0,0 +1,536 @@
+// Copyright (c) 2009 The Go Authors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//    * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//    * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//    * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// JSON value parser state machine.
+// This package is almost entirely copied from the Go stdlib.
+// Changes made to it permit users of the package to tell
+// if some slice of bytes is a valid beginning of a json string.
+package json
+
+import "fmt"
+
+type (
+	context    int
+	scanStatus int
+)
+
+const (
+	contextKey context = iota
+	contextObj
+	contextArr
+
+	scanContinue     scanStatus = iota // uninteresting byte
+	scanBeginLiteral                   // end implied by next result != scanContinue
+	scanBeginObject                    // begin object
+	scanObjectKey                      // just finished object key (string)
+	scanObjectValue                    // just finished non-last object value
+	scanEndObject                      // end object (implies scanObjectValue if possible)
+	scanBeginArray                     // begin array
+	scanArrayValue                     // just finished array value
+	scanEndArray                       // end array (implies scanArrayValue if possible)
+	scanSkipSpace                      // space byte; can skip; known to be last "continue" result
+	scanEnd                            // top-level value ended *before* this byte; known to be first "stop" result
+	scanError                          // hit an error, scanner.err.
+)
+
+type (
+	scanner struct {
+		step     func(*scanner, byte) scanStatus
+		contexts []context
+		endTop   bool
+		err      error
+		index    int
+	}
+)
+
+// Scan returns the number of bytes scanned and if there was any error
+// in trying to reach the end of data
+func Scan(data []byte) (int, error) {
+	s := &scanner{}
+	_ = checkValid(data, s)
+	return s.index, s.err
+}
+
+// checkValid verifies that data is valid JSON-encoded data.
+// scan is passed in for use by checkValid to avoid an allocation.
+func checkValid(data []byte, scan *scanner) error {
+	scan.reset()
+	for _, c := range data {
+		scan.index++
+		if scan.step(scan, c) == scanError {
+			return scan.err
+		}
+	}
+	if scan.eof() == scanError {
+		return scan.err
+	}
+	return nil
+}
+
+func isSpace(c byte) bool {
+	return c == ' ' || c == '\t' || c == '\r' || c == '\n'
+}
+
+func (s *scanner) reset() {
+	s.step = stateBeginValue
+	s.contexts = s.contexts[0:0]
+	s.err = nil
+}
+
+// eof tells the scanner that the end of input has been reached.
+// It returns a scan status just as s.step does.
+func (s *scanner) eof() scanStatus {
+	if s.err != nil {
+		return scanError
+	}
+	if s.endTop {
+		return scanEnd
+	}
+	s.step(s, ' ')
+	if s.endTop {
+		return scanEnd
+	}
+	if s.err == nil {
+		s.err = fmt.Errorf("unexpected end of JSON input")
+	}
+	return scanError
+}
+
+// pushContext pushes a new parse state p onto the parse stack.
+func (s *scanner) pushParseState(p context) {
+	s.contexts = append(s.contexts, p)
+}
+
+// popParseState pops a parse state (already obtained) off the stack
+// and updates s.step accordingly.
+func (s *scanner) popParseState() {
+	n := len(s.contexts) - 1
+	s.contexts = s.contexts[0:n]
+	if n == 0 {
+		s.step = stateEndTop
+		s.endTop = true
+	} else {
+		s.step = stateEndValue
+	}
+}
+
+// stateBeginValueOrEmpty is the state after reading `[`.
+func stateBeginValueOrEmpty(s *scanner, c byte) scanStatus {
+	if c <= ' ' && isSpace(c) {
+		return scanSkipSpace
+	}
+	if c == ']' {
+		return stateEndValue(s, c)
+	}
+	return stateBeginValue(s, c)
+}
+
+// stateBeginValue is the state at the beginning of the input.
+func stateBeginValue(s *scanner, c byte) scanStatus {
+	if c <= ' ' && isSpace(c) {
+		return scanSkipSpace
+	}
+	switch c {
+	case '{':
+		s.step = stateBeginStringOrEmpty
+		s.pushParseState(contextKey)
+		return scanBeginObject
+	case '[':
+		s.step = stateBeginValueOrEmpty
+		s.pushParseState(contextArr)
+		return scanBeginArray
+	case '"':
+		s.step = stateInString
+		return scanBeginLiteral
+	case '-':
+		s.step = stateNeg
+		return scanBeginLiteral
+	case '0': // beginning of 0.123
+		s.step = state0
+		return scanBeginLiteral
+	case 't': // beginning of true
+		s.step = stateT
+		return scanBeginLiteral
+	case 'f': // beginning of false
+		s.step = stateF
+		return scanBeginLiteral
+	case 'n': // beginning of null
+		s.step = stateN
+		return scanBeginLiteral
+	}
+	if '1' <= c && c <= '9' { // beginning of 1234.5
+		s.step = state1
+		return scanBeginLiteral
+	}
+	return s.error(c, "looking for beginning of value")
+}
+
+// stateBeginStringOrEmpty is the state after reading `{`.
+func stateBeginStringOrEmpty(s *scanner, c byte) scanStatus {
+	if c <= ' ' && isSpace(c) {
+		return scanSkipSpace
+	}
+	if c == '}' {
+		n := len(s.contexts)
+		s.contexts[n-1] = contextObj
+		return stateEndValue(s, c)
+	}
+	return stateBeginString(s, c)
+}
+
+// stateBeginString is the state after reading `{"key": value,`.
+func stateBeginString(s *scanner, c byte) scanStatus {
+	if c <= ' ' && isSpace(c) {
+		return scanSkipSpace
+	}
+	if c == '"' {
+		s.step = stateInString
+		return scanBeginLiteral
+	}
+	return s.error(c, "looking for beginning of object key string")
+}
+
+// stateEndValue is the state after completing a value,
+// such as after reading `{}` or `true` or `["x"`.
+func stateEndValue(s *scanner, c byte) scanStatus {
+	n := len(s.contexts)
+	if n == 0 {
+		// Completed top-level before the current byte.
+		s.step = stateEndTop
+		s.endTop = true
+		return stateEndTop(s, c)
+	}
+	if c <= ' ' && isSpace(c) {
+		s.step = stateEndValue
+		return scanSkipSpace
+	}
+	ps := s.contexts[n-1]
+	switch ps {
+	case contextKey:
+		if c == ':' {
+			s.contexts[n-1] = contextObj
+			s.step = stateBeginValue
+			return scanObjectKey
+		}
+		return s.error(c, "after object key")
+	case contextObj:
+		if c == ',' {
+			s.contexts[n-1] = contextKey
+			s.step = stateBeginString
+			return scanObjectValue
+		}
+		if c == '}' {
+			s.popParseState()
+			return scanEndObject
+		}
+		return s.error(c, "after object key:value pair")
+	case contextArr:
+		if c == ',' {
+			s.step = stateBeginValue
+			return scanArrayValue
+		}
+		if c == ']' {
+			s.popParseState()
+			return scanEndArray
+		}
+		return s.error(c, "after array element")
+	}
+	return s.error(c, "")
+}
+
+// stateEndTop is the state after finishing the top-level value,
+// such as after reading `{}` or `[1,2,3]`.
+// Only space characters should be seen now.
+func stateEndTop(s *scanner, c byte) scanStatus {
+	if c != ' ' && c != '\t' && c != '\r' && c != '\n' {
+		// Complain about non-space byte on next call.
+		s.error(c, "after top-level value")
+	}
+	return scanEnd
+}
+
+// stateInString is the state after reading `"`.
+func stateInString(s *scanner, c byte) scanStatus {
+	if c == '"' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	if c == '\\' {
+		s.step = stateInStringEsc
+		return scanContinue
+	}
+	if c < 0x20 {
+		return s.error(c, "in string literal")
+	}
+	return scanContinue
+}
+
+// stateInStringEsc is the state after reading `"\` during a quoted string.
+func stateInStringEsc(s *scanner, c byte) scanStatus {
+	switch c {
+	case 'b', 'f', 'n', 'r', 't', '\\', '/', '"':
+		s.step = stateInString
+		return scanContinue
+	case 'u':
+		s.step = stateInStringEscU
+		return scanContinue
+	}
+	return s.error(c, "in string escape code")
+}
+
+// stateInStringEscU is the state after reading `"\u` during a quoted string.
+func stateInStringEscU(s *scanner, c byte) scanStatus {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInStringEscU1
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateInStringEscU1 is the state after reading `"\u1` during a quoted string.
+func stateInStringEscU1(s *scanner, c byte) scanStatus {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInStringEscU12
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateInStringEscU12 is the state after reading `"\u12` during a quoted string.
+func stateInStringEscU12(s *scanner, c byte) scanStatus {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInStringEscU123
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateInStringEscU123 is the state after reading `"\u123` during a quoted string.
+func stateInStringEscU123(s *scanner, c byte) scanStatus {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInString
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateNeg is the state after reading `-` during a number.
+func stateNeg(s *scanner, c byte) scanStatus {
+	if c == '0' {
+		s.step = state0
+		return scanContinue
+	}
+	if '1' <= c && c <= '9' {
+		s.step = state1
+		return scanContinue
+	}
+	return s.error(c, "in numeric literal")
+}
+
+// state1 is the state after reading a non-zero integer during a number,
+// such as after reading `1` or `100` but not `0`.
+func state1(s *scanner, c byte) scanStatus {
+	if '0' <= c && c <= '9' {
+		s.step = state1
+		return scanContinue
+	}
+	return state0(s, c)
+}
+
+// state0 is the state after reading `0` during a number.
+func state0(s *scanner, c byte) scanStatus {
+	if c == '.' {
+		s.step = stateDot
+		return scanContinue
+	}
+	if c == 'e' || c == 'E' {
+		s.step = stateE
+		return scanContinue
+	}
+	return stateEndValue(s, c)
+}
+
+// stateDot is the state after reading the integer and decimal point in a number,
+// such as after reading `1.`.
+func stateDot(s *scanner, c byte) scanStatus {
+	if '0' <= c && c <= '9' {
+		s.step = stateDot0
+		return scanContinue
+	}
+	return s.error(c, "after decimal point in numeric literal")
+}
+
+// stateDot0 is the state after reading the integer, decimal point, and subsequent
+// digits of a number, such as after reading `3.14`.
+func stateDot0(s *scanner, c byte) scanStatus {
+	if '0' <= c && c <= '9' {
+		return scanContinue
+	}
+	if c == 'e' || c == 'E' {
+		s.step = stateE
+		return scanContinue
+	}
+	return stateEndValue(s, c)
+}
+
+// stateE is the state after reading the mantissa and e in a number,
+// such as after reading `314e` or `0.314e`.
+func stateE(s *scanner, c byte) scanStatus {
+	if c == '+' || c == '-' {
+		s.step = stateESign
+		return scanContinue
+	}
+	return stateESign(s, c)
+}
+
+// stateESign is the state after reading the mantissa, e, and sign in a number,
+// such as after reading `314e-` or `0.314e+`.
+func stateESign(s *scanner, c byte) scanStatus {
+	if '0' <= c && c <= '9' {
+		s.step = stateE0
+		return scanContinue
+	}
+	return s.error(c, "in exponent of numeric literal")
+}
+
+// stateE0 is the state after reading the mantissa, e, optional sign,
+// and at least one digit of the exponent in a number,
+// such as after reading `314e-2` or `0.314e+1` or `3.14e0`.
+func stateE0(s *scanner, c byte) scanStatus {
+	if '0' <= c && c <= '9' {
+		return scanContinue
+	}
+	return stateEndValue(s, c)
+}
+
+// stateT is the state after reading `t`.
+func stateT(s *scanner, c byte) scanStatus {
+	if c == 'r' {
+		s.step = stateTr
+		return scanContinue
+	}
+	return s.error(c, "in literal true (expecting 'r')")
+}
+
+// stateTr is the state after reading `tr`.
+func stateTr(s *scanner, c byte) scanStatus {
+	if c == 'u' {
+		s.step = stateTru
+		return scanContinue
+	}
+	return s.error(c, "in literal true (expecting 'u')")
+}
+
+// stateTru is the state after reading `tru`.
+func stateTru(s *scanner, c byte) scanStatus {
+	if c == 'e' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	return s.error(c, "in literal true (expecting 'e')")
+}
+
+// stateF is the state after reading `f`.
+func stateF(s *scanner, c byte) scanStatus {
+	if c == 'a' {
+		s.step = stateFa
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 'a')")
+}
+
+// stateFa is the state after reading `fa`.
+func stateFa(s *scanner, c byte) scanStatus {
+	if c == 'l' {
+		s.step = stateFal
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 'l')")
+}
+
+// stateFal is the state after reading `fal`.
+func stateFal(s *scanner, c byte) scanStatus {
+	if c == 's' {
+		s.step = stateFals
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 's')")
+}
+
+// stateFals is the state after reading `fals`.
+func stateFals(s *scanner, c byte) scanStatus {
+	if c == 'e' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 'e')")
+}
+
+// stateN is the state after reading `n`.
+func stateN(s *scanner, c byte) scanStatus {
+	if c == 'u' {
+		s.step = stateNu
+		return scanContinue
+	}
+	return s.error(c, "in literal null (expecting 'u')")
+}
+
+// stateNu is the state after reading `nu`.
+func stateNu(s *scanner, c byte) scanStatus {
+	if c == 'l' {
+		s.step = stateNul
+		return scanContinue
+	}
+	return s.error(c, "in literal null (expecting 'l')")
+}
+
+// stateNul is the state after reading `nul`.
+func stateNul(s *scanner, c byte) scanStatus {
+	if c == 'l' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	return s.error(c, "in literal null (expecting 'l')")
+}
+
+// stateError is the state after reaching a syntax error,
+// such as after reading `[1}` or `5.1.2`.
+func stateError(s *scanner, c byte) scanStatus {
+	return scanError
+}
+
+// error records an error and switches to the error state.
+func (s *scanner) error(c byte, context string) scanStatus {
+	s.step = stateError
+	s.err = fmt.Errorf("invalid character <<%c>> %s", c, context)
+	return scanError
+}

vendor/github.com/gabriel-vasile/mimetype/internal/matchers/archive.go

@@ -0,0 +1,78 @@
+package matchers
+
+import "bytes"
+
+// Zip matches a zip archive.
+func Zip(in []byte) bool {
+	return len(in) > 3 &&
+		in[0] == 0x50 && in[1] == 0x4B &&
+		(in[2] == 0x3 || in[2] == 0x5 || in[2] == 0x7) &&
+		(in[3] == 0x4 || in[3] == 0x6 || in[3] == 0x8)
+}
+
+// SevenZ matches a 7z archive.
+func SevenZ(in []byte) bool {
+	return len(in) > 6 &&
+		bytes.Equal(in[:6], []byte{0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C})
+}
+
+// Epub matches an EPUB file.
+func Epub(in []byte) bool {
+	return len(in) > 58 && bytes.Equal(in[30:58], []byte("mimetypeapplication/epub+zip"))
+}
+
+// Jar matches a Java archive file.
+func Jar(in []byte) bool {
+	return bytes.Contains(in, []byte("META-INF/MANIFEST.MF"))
+}
+
+// Gzip matched gzip files based on http://www.zlib.org/rfc-gzip.html#header-trailer.
+func Gzip(in []byte) bool {
+	return len(in) > 2 && bytes.Equal(in[:2], []byte{0x1f, 0x8b})
+}
+
+// Crx matches a Chrome extension file: a zip archive prepended by "Cr24".
+func Crx(in []byte) bool {
+	return bytes.HasPrefix(in, []byte("Cr24"))
+}
+
+// Tar matches a (t)ape (ar)chive file.
+func Tar(in []byte) bool {
+	return len(in) > 262 && bytes.Equal(in[257:262], []byte("ustar"))
+}
+
+// Fits matches an Flexible Image Transport System file.
+func Fits(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0x53, 0x49, 0x4D, 0x50, 0x4C, 0x45, 0x20,
+		0x20, 0x3D, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+		0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x54})
+}
+
+// Xar matches an eXtensible ARchive format file.
+func Xar(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0x78, 0x61, 0x72, 0x21})
+}
+
+// Bz2 matches a bzip2 file.
+func Bz2(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0x42, 0x5A, 0x68})
+}
+
+// Ar matches an ar (Unix) archive file.
+func Ar(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0x21, 0x3C, 0x61, 0x72, 0x63, 0x68, 0x3E})
+}
+
+// Deb matches a Debian package file
+func Deb(in []byte) bool {
+	return len(in) > 8 && bytes.HasPrefix(in[8:], []byte{0x64, 0x65, 0x62, 0x69,
+		0x61, 0x6E, 0x2D, 0x62, 0x69, 0x6E, 0x61, 0x72, 0x79})
+}
+
+// Rar matches a RAR archive file
+func Rar(in []byte) bool {
+	if !bytes.HasPrefix(in, []byte{0x52, 0x61, 0x72, 0x21, 0x1A, 0x07}) {
+		return false
+	}
+	return len(in) > 8 && (bytes.Equal(in[6:8], []byte{0x01, 0x00}) || in[6] == 0x00)
+}

vendor/github.com/gabriel-vasile/mimetype/internal/matchers/audio.go

@@ -0,0 +1,69 @@
+package matchers
+
+import (
+	"bytes"
+)
+
+// Mp3 matches an mp3 file.
+func Mp3(in []byte) bool {
+	return bytes.HasPrefix(in, []byte("\x49\x44\x33"))
+}
+
+// Flac matches a Free Lossless Audio Codec file.
+func Flac(in []byte) bool {
+	return bytes.HasPrefix(in, []byte("\x66\x4C\x61\x43\x00\x00\x00\x22"))
+}
+
+// Midi matches a Musical Instrument Digital Interface file.
+func Midi(in []byte) bool {
+	return bytes.HasPrefix(in, []byte("\x4D\x54\x68\x64"))
+}
+
+// Ape matches a Monkey's Audio file.
+func Ape(in []byte) bool {
+	return bytes.HasPrefix(in, []byte("\x4D\x41\x43\x20\x96\x0F\x00\x00\x34\x00\x00\x00\x18\x00\x00\x00\x90\xE3"))
+}
+
+// MusePack matches a Musepack file.
+func MusePack(in []byte) bool {
+	return bytes.HasPrefix(in, []byte("MPCK"))
+}
+
+// Wav matches a Waveform Audio File Format file.
+func Wav(in []byte) bool {
+	return len(in) > 12 &&
+		bytes.Equal(in[:4], []byte("\x52\x49\x46\x46")) &&
+		bytes.Equal(in[8:12], []byte("\x57\x41\x56\x45"))
+}
+
+// Aiff matches Audio Interchange File Format file.
+func Aiff(in []byte) bool {
+	return len(in) > 12 &&
+		bytes.Equal(in[:4], []byte("\x46\x4F\x52\x4D")) &&
+		bytes.Equal(in[8:12], []byte("\x41\x49\x46\x46"))
+}
+
+// Ogg matches an Ogg file.
+func Ogg(in []byte) bool {
+	return len(in) > 5 && bytes.Equal(in[:5], []byte("\x4F\x67\x67\x53\x00"))
+}
+
+// Au matches a Sun Microsystems au file.
+func Au(in []byte) bool {
+	return len(in) > 4 && bytes.Equal(in[:4], []byte("\x2E\x73\x6E\x64"))
+}
+
+// Amr matches an Adaptive Multi-Rate file.
+func Amr(in []byte) bool {
+	return len(in) > 5 && bytes.Equal(in[:5], []byte("\x23\x21\x41\x4D\x52"))
+}
+
+// Aac matches an Advanced Audio Coding file.
+func Aac(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0xFF, 0xF1}) || bytes.HasPrefix(in, []byte{0xFF, 0xF9})
+}
+
+// Voc matches a Creative Voice file.
+func Voc(in []byte) bool {
+	return bytes.HasPrefix(in, []byte("Creative Voice File"))
+}

vendor/github.com/gabriel-vasile/mimetype/internal/matchers/binary.go

@@ -0,0 +1,88 @@
+package matchers
+
+import (
+	"bytes"
+)
+
+// Class matches an java class file.
+func Class(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0xCA, 0xFE, 0xBA, 0xBE})
+}
+
+// Swf matches an Adobe Flash swf file.
+func Swf(in []byte) bool {
+	return bytes.HasPrefix(in, []byte("CWS")) ||
+		bytes.HasPrefix(in, []byte("FWS")) ||
+		bytes.HasPrefix(in, []byte("ZWS"))
+}
+
+// Wasm matches a web assembly File Format file.
+func Wasm(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0x00, 0x61, 0x73, 0x6D})
+}
+
+// Dbf matches a dBase file.
+// https://www.dbase.com/Knowledgebase/INT/db7_file_fmt.htm
+func Dbf(in []byte) bool {
+	if len(in) < 4 {
+		return false
+	}
+
+	// 3rd and 4th bytes contain the last update month and day of month
+	if !(0 < in[2] && in[2] < 13 && 0 < in[3] && in[3] < 32) {
+		return false
+	}
+
+	// dbf type is dictated by the first byte
+	dbfTypes := []byte{
+		0x02, 0x03, 0x04, 0x05, 0x30, 0x31, 0x32, 0x42, 0x62, 0x7B, 0x82,
+		0x83, 0x87, 0x8A, 0x8B, 0x8E, 0xB3, 0xCB, 0xE5, 0xF5, 0xF4, 0xFB,
+	}
+	for _, b := range dbfTypes {
+		if in[0] == b {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Exe matches a Windows/DOS executable file.
+func Exe(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0x4D, 0x5A})
+}
+
+// Elf matches an Executable and Linkable Format file.
+func Elf(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0x7F, 0x45, 0x4C, 0x46})
+}
+
+// ElfObj matches an object file.
+func ElfObj(in []byte) bool {
+	return len(in) > 17 && ((in[16] == 0x01 && in[17] == 0x00) ||
+		(in[16] == 0x00 && in[17] == 0x01))
+}
+
+// ElfExe matches an executable file.
+func ElfExe(in []byte) bool {
+	return len(in) > 17 && ((in[16] == 0x02 && in[17] == 0x00) ||
+		(in[16] == 0x00 && in[17] == 0x02))
+}
+
+// ElfLib matches a shared library file.
+func ElfLib(in []byte) bool {
+	return len(in) > 17 && ((in[16] == 0x03 && in[17] == 0x00) ||
+		(in[16] == 0x00 && in[17] == 0x03))
+}
+
+// ElfDump matches a core dump file.
+func ElfDump(in []byte) bool {
+	return len(in) > 17 && ((in[16] == 0x04 && in[17] == 0x00) ||
+		(in[16] == 0x00 && in[17] == 0x04))
+}
+
+// Dcm matches a DICOM medical format file.
+func Dcm(in []byte) bool {
+	return len(in) > 131 &&
+		bytes.Equal(in[128:132], []byte{0x44, 0x49, 0x43, 0x4D})
+}

vendor/github.com/gabriel-vasile/mimetype/internal/matchers/database.go

@@ -0,0 +1,10 @@
+package matchers
+
+import "bytes"
+
+// Sqlite matches an SQLite database file.
+func Sqlite(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{
+		0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00,
+	})
+}

vendor/github.com/gabriel-vasile/mimetype/internal/matchers/document.go

@@ -0,0 +1,35 @@
+package matchers
+
+import "bytes"
+
+// Pdf matches a Portable Document Format file.
+func Pdf(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0x25, 0x50, 0x44, 0x46})
+}
+
+// DjVu matches a DjVu file
+func DjVu(in []byte) bool {
+	if !bytes.HasPrefix(in, []byte{0x41, 0x54, 0x26, 0x54, 0x46, 0x4F, 0x52, 0x4D}) {
+		return false
+	}
+	if len(in) < 15 {
+		return false
+	}
+	return bytes.HasPrefix(in[12:], []byte("DJVM")) ||
+		bytes.HasPrefix(in[12:], []byte("DJVU")) ||
+		bytes.HasPrefix(in[12:], []byte("DJVI")) ||
+		bytes.HasPrefix(in[12:], []byte("THUM"))
+}
+
+// Mobi matches a Mobi file
+func Mobi(in []byte) bool {
+	if len(in) < 68 {
+		return false
+	}
+	return bytes.Equal(in[60:68], []byte("BOOKMOBI"))
+}
+
+// Lit matches a Microsoft Lit file
+func Lit(in []byte) bool {
+	return bytes.HasPrefix(in, []byte("ITOLITLS"))
+}

vendor/github.com/gabriel-vasile/mimetype/internal/matchers/font.go

@@ -0,0 +1,27 @@
+package matchers
+
+import "bytes"
+
+// Woff matches a Web Open Font Format file.
+func Woff(in []byte) bool {
+	return bytes.HasPrefix(in, []byte("wOFF"))
+}
+
+// Woff2 matches a Web Open Font Format version 2 file.
+func Woff2(in []byte) bool {
+	return bytes.HasPrefix(in, []byte("wOF2"))
+}
+
+// Otf matches an OpenType font file.
+func Otf(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0x4F, 0x54, 0x54, 0x4F, 0x00})
+}
+
+// Eot matches an Embedded OpenType font file.
+func Eot(in []byte) bool {
+	return len(in) > 35 &&
+		bytes.Equal(in[34:36], []byte{0x4C, 0x50}) &&
+		(bytes.Equal(in[8:11], []byte{0x02, 0x00, 0x01}) ||
+			bytes.Equal(in[8:11], []byte{0x01, 0x00, 0x00}) ||
+			bytes.Equal(in[8:11], []byte{0x02, 0x00, 0x02}))
+}

vendor/github.com/gabriel-vasile/mimetype/internal/matchers/geo.go

@@ -0,0 +1,44 @@
+package matchers
+
+import (
+	"bytes"
+	"encoding/binary"
+)
+
+// Shp matches a shape format file.
+// https://www.esri.com/library/whitepapers/pdfs/shapefile.pdf
+func Shp(in []byte) bool {
+	if len(in) < 112 {
+		return false
+	}
+	shapeTypes := []int{
+		0,  // Null shape
+		1,  // Point
+		3,  // Polyline
+		5,  // Polygon
+		8,  // MultiPoint
+		11, // PointZ
+		13, // PolylineZ
+		15, // PolygonZ
+		18, // MultiPointZ
+		21, // PointM
+		23, // PolylineM
+		25, // PolygonM
+		28, // MultiPointM
+		31, // MultiPatch
+	}
+
+	for _, st := range shapeTypes {
+		if st == int(binary.LittleEndian.Uint32(in[108:112])) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Shx matches a shape index format file.
+// https://www.esri.com/library/whitepapers/pdfs/shapefile.pdf
+func Shx(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0x00, 0x00, 0x27, 0x0A})
+}

vendor/github.com/gabriel-vasile/mimetype/internal/matchers/image.go

@@ -0,0 +1,91 @@
+package matchers
+
+import "bytes"
+
+// Png matches a Portable Network Graphics file.
+func Png(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A})
+}
+
+// Jpg matches a Joint Photographic Experts Group file.
+func Jpg(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0xFF, 0xD8, 0xFF})
+}
+
+// Gif matches a Graphics Interchange Format file.
+func Gif(in []byte) bool {
+	return bytes.HasPrefix(in, []byte("GIF87a")) ||
+		bytes.HasPrefix(in, []byte("GIF89a"))
+}
+
+// Webp matches a WebP file.
+func Webp(in []byte) bool {
+	return len(in) > 12 &&
+		bytes.Equal(in[0:4], []byte{0x52, 0x49, 0x46, 0x46}) &&
+		bytes.Equal(in[8:12], []byte{0x57, 0x45, 0x42, 0x50})
+}
+
+// Bmp matches a bitmap image file.
+func Bmp(in []byte) bool {
+	return len(in) > 1 && in[0] == 0x42 && in[1] == 0x4D
+}
+
+// Ps matches a PostScript file.
+func Ps(in []byte) bool {
+	return bytes.HasPrefix(in, []byte("%!PS-Adobe-"))
+}
+
+// Psd matches a Photoshop Document file.
+func Psd(in []byte) bool {
+	return bytes.HasPrefix(in, []byte("8BPS"))
+}
+
+// Ico matches an ICO file.
+func Ico(in []byte) bool {
+	return len(in) > 3 &&
+		in[0] == 0x00 && in[1] == 0x00 &&
+		in[2] == 0x01 && in[3] == 0x00
+}
+
+// Tiff matches a Tagged Image File Format file.
+func Tiff(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0x49, 0x49, 0x2A, 0x00}) ||
+		bytes.HasPrefix(in, []byte{0x4D, 0x4D, 0x00, 0x2A})
+}
+
+// Bpg matches a Better Portable Graphics file.
+func Bpg(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0x42, 0x50, 0x47, 0xFB})
+}
+
+// Dwg matches a CAD drawing file.
+func Dwg(in []byte) bool {
+	if len(in) < 6 || in[0] != 0x41 || in[1] != 0x43 {
+		return false
+	}
+	dwgVersions := [][]byte{
+		{0x31, 0x2E, 0x34, 0x30},
+		{0x31, 0x2E, 0x35, 0x30},
+		{0x32, 0x2E, 0x31, 0x30},
+		{0x31, 0x30, 0x30, 0x32},
+		{0x31, 0x30, 0x30, 0x33},
+		{0x31, 0x30, 0x30, 0x34},
+		{0x31, 0x30, 0x30, 0x36},
+		{0x31, 0x30, 0x30, 0x39},
+		{0x31, 0x30, 0x31, 0x32},
+		{0x31, 0x30, 0x31, 0x34},
+		{0x31, 0x30, 0x31, 0x35},
+		{0x31, 0x30, 0x31, 0x38},
+		{0x31, 0x30, 0x32, 0x31},
+		{0x31, 0x30, 0x32, 0x34},
+		{0x31, 0x30, 0x33, 0x32},
+	}
+
+	for _, d := range dwgVersions {
+		if bytes.Equal(in[2:6], d) {
+			return true
+		}
+	}
+
+	return false
+}

vendor/github.com/gabriel-vasile/mimetype/internal/matchers/matchers.go

@@ -0,0 +1,46 @@
+// Package matchers holds the matching functions used to find mime types.
+package matchers
+
+// ReadLimit is the maximum number of bytes read
+// from the input when detecting a reader.
+const ReadLimit = 2048
+
+// True is a dummy matching function used to match any input.
+func True([]byte) bool {
+	return true
+}
+
+// False is a dummy matching function used to never match input.
+func False([]byte) bool {
+	return false
+}
+
+// trimLWS trims whitespace from beginning of the input.
+func trimLWS(in []byte) []byte {
+	firstNonWS := 0
+	for ; firstNonWS < len(in) && isWS(in[firstNonWS]); firstNonWS++ {
+	}
+
+	return in[firstNonWS:]
+}
+
+// trimRWS trims whitespace from the end of the input.
+func trimRWS(in []byte) []byte {
+	lastNonWS := len(in) - 1
+	for ; lastNonWS > 0 && isWS(in[lastNonWS]); lastNonWS-- {
+	}
+
+	return in[:lastNonWS+1]
+}
+
+func firstLine(in []byte) []byte {
+	lineEnd := 0
+	for ; lineEnd < len(in) && in[lineEnd] != '\n'; lineEnd++ {
+	}
+
+	return in[:lineEnd]
+}
+
+func isWS(b byte) bool {
+	return b == '\t' || b == '\n' || b == '\x0c' || b == '\r' || b == ' '
+}

vendor/github.com/gabriel-vasile/mimetype/internal/matchers/ms_office.go

@@ -0,0 +1,73 @@
+package matchers
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+)
+
+// Xlsx matches a Microsoft Excel 2007 file.
+func Xlsx(in []byte) bool {
+	return bytes.Contains(in, []byte("xl/"))
+}
+
+// Docx matches a Microsoft Office 2007 file.
+func Docx(in []byte) bool {
+	return bytes.Contains(in, []byte("word/"))
+}
+
+// Pptx matches a Microsoft PowerPoint 2007 file.
+func Pptx(in []byte) bool {
+	return bytes.Contains(in, []byte("ppt/"))
+}
+
+// Doc matches a Microsoft Office 97-2003 file.
+func Doc(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1})
+}
+
+// Ppt  matches a Microsoft PowerPoint 97-2003 file.
+func Ppt(in []byte) bool {
+	if len(in) < 520 {
+		return false
+	}
+
+	if fmt.Sprintf("%X", in[:8]) == "D0CF11E0A1B11AE1" {
+		offset512 := fmt.Sprintf("%X", in[512:516])
+		if offset512 == "A0461DF0" || offset512 == "006E1EF0" || offset512 == "0F00E803" {
+			return true
+		}
+		if offset512 == "FDFFFFFF" && fmt.Sprintf("%x", in[518:520]) == "0000" {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Xls  matches a Microsoft Excel 97-2003 file.
+func Xls(in []byte) bool {
+	if len(in) < 520 {
+		return false
+	}
+
+	if fmt.Sprintf("%X", in[:8]) == "D0CF11E0A1B11AE1" {
+		offset512 := fmt.Sprintf("%X", in[512:520])
+		subheaders := []string{
+			"0908100000060500",
+			"FDFFFFFF10",
+			"FDFFFFFF1F",
+			"FDFFFFFF22",
+			"FDFFFFFF23",
+			"FDFFFFFF28",
+			"FDFFFFFF29",
+		}
+		for _, h := range subheaders {
+			if strings.HasPrefix(offset512, h) {
+				return true
+			}
+		}
+	}
+
+	return false
+}

vendor/github.com/gabriel-vasile/mimetype/internal/matchers/odf.go

@@ -0,0 +1,48 @@
+package matchers
+
+import "bytes"
+
+// Odt matches an OpenDocument Text file.
+func Odt(in []byte) bool {
+	return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.text"))
+}
+
+// Ott matches an OpenDocument Text Template file.
+func Ott(in []byte) bool {
+	return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.text-template"))
+}
+
+// Ods matches an OpenDocument Spreadsheet file.
+func Ods(in []byte) bool {
+	return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.spreadsheet"))
+}
+
+// Ots matches an OpenDocument Spreadsheet Template file.
+func Ots(in []byte) bool {
+	return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.spreadsheet-template"))
+}
+
+// Odp matches an OpenDocument Presentation file.
+func Odp(in []byte) bool {
+	return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.presentation"))
+}
+
+// Otp matches an OpenDocument Presentation Template file.
+func Otp(in []byte) bool {
+	return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.presentation-template"))
+}
+
+// Odg matches an OpenDocument Drawing file.
+func Odg(in []byte) bool {
+	return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.graphics"))
+}
+
+// Otg matches an OpenDocument Drawing Template file.
+func Otg(in []byte) bool {
+	return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.graphics-template"))
+}
+
+// Odf matches an OpenDocument Formula file.
+func Odf(in []byte) bool {
+	return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.formula"))
+}

vendor/github.com/gabriel-vasile/mimetype/internal/matchers/signature.go

@@ -0,0 +1,128 @@
+package matchers
+
+import (
+	"bytes"
+	"fmt"
+)
+
+type (
+	markupSig  []byte
+	ciSig      []byte // case insensitive signature
+	shebangSig []byte // matches !# followed by the signature
+	ftypSig    []byte // matches audio/video files. www.ftyps.com
+	xmlSig     struct {
+		// the local name of the root tag
+		localName []byte
+		// the namespace of the XML document
+		xmlns []byte
+	}
+	sig interface {
+		detect([]byte) bool
+	}
+)
+
+func newXmlSig(localName, xmlns string) xmlSig {
+	ret := xmlSig{xmlns: []byte(xmlns)}
+	if localName != "" {
+		ret.localName = []byte(fmt.Sprintf("<%s", localName))
+	}
+
+	return ret
+}
+
+// Implement sig interface.
+func (hSig markupSig) detect(in []byte) bool {
+	if len(in) < len(hSig)+1 {
+		return false
+	}
+
+	// perform case insensitive check
+	for i, b := range hSig {
+		db := in[i]
+		if 'A' <= b && b <= 'Z' {
+			db &= 0xDF
+		}
+		if b != db {
+			return false
+		}
+	}
+	// Next byte must be space or right angle bracket.
+	if db := in[len(hSig)]; db != ' ' && db != '>' {
+		return false
+	}
+
+	return true
+}
+
+// Implement sig interface.
+func (tSig ciSig) detect(in []byte) bool {
+	if len(in) < len(tSig)+1 {
+		return false
+	}
+
+	// perform case insensitive check
+	for i, b := range tSig {
+		db := in[i]
+		if 'A' <= b && b <= 'Z' {
+			db &= 0xDF
+		}
+		if b != db {
+			return false
+		}
+	}
+
+	return true
+}
+
+// a valid shebang starts with the "#!" characters
+// followed by any number of spaces
+// followed by the path to the interpreter and optionally, the args for the interpreter
+func (sSig shebangSig) detect(in []byte) bool {
+	in = firstLine(in)
+
+	if len(in) < len(sSig)+2 {
+		return false
+	}
+	if in[0] != '#' || in[1] != '!' {
+		return false
+	}
+
+	in = trimLWS(trimRWS(in[2:]))
+
+	return bytes.Equal(in, sSig)
+}
+
+// Implement sig interface.
+func (fSig ftypSig) detect(in []byte) bool {
+	return len(in) > 12 &&
+		bytes.Equal(in[4:8], []byte("ftyp")) &&
+		bytes.Equal(in[8:12], fSig)
+}
+
+func (xSig xmlSig) detect(in []byte) bool {
+	l := 512
+	if len(in) < l {
+		l = len(in)
+	}
+	in = in[:l]
+
+	if len(xSig.localName) == 0 {
+		return bytes.Index(in, xSig.xmlns) > 0
+	}
+	if len(xSig.xmlns) == 0 {
+		return bytes.Index(in, xSig.localName) > 0
+	}
+
+	localNameIndex := bytes.Index(in, xSig.localName)
+	return localNameIndex != -1 && localNameIndex < bytes.Index(in, xSig.xmlns)
+}
+
+func detect(in []byte, sigs []sig) bool {
+	for _, sig := range sigs {
+		if sig.detect(in) {
+			return true
+		}
+	}
+
+	return false
+}

vendor/github.com/gabriel-vasile/mimetype/internal/matchers/text.go

@@ -0,0 +1,304 @@
+package matchers
+
+import (
+	"bytes"
+
+	"github.com/gabriel-vasile/mimetype/internal/json"
+)
+
+var (
+	htmlSigs = []sig{
+		markupSig("<!DOCTYPE HTML"),
+		markupSig("<HTML"),
+		markupSig("<HEAD"),
+		markupSig("<SCRIPT"),
+		markupSig("<IFRAME"),
+		markupSig("<H1"),
+		markupSig("<DIV"),
+		markupSig("<FONT"),
+		markupSig("<TABLE"),
+		markupSig("<A"),
+		markupSig("<STYLE"),
+		markupSig("<TITLE"),
+		markupSig("<B"),
+		markupSig("<BODY"),
+		markupSig("<BR"),
+		markupSig("<P"),
+		markupSig("<!--"),
+	}
+	xmlSigs = []sig{
+		markupSig("<?XML"),
+	}
+	rssSigs = []sig{
+		newXmlSig("rss", ""),
+	}
+	atomSigs = []sig{
+		newXmlSig("feed", `xmlns="http://www.w3.org/2005/Atom"`),
+	}
+	kmlSigs = []sig{
+		newXmlSig("kml", `xmlns="http://www.opengis.net/kml/2.2"`),
+		newXmlSig("kml", `xmlns="http://earth.google.com/kml/2.0"`),
+		newXmlSig("kml", `xmlns="http://earth.google.com/kml/2.1"`),
+		newXmlSig("kml", `xmlns="http://earth.google.com/kml/2.2"`),
+	}
+	xliffSigs = []sig{
+		newXmlSig("xliff", `xmlns="urn:oasis:names:tc:xliff:document:1.2"`),
+	}
+	colladaSigs = []sig{
+		newXmlSig("COLLADA", `xmlns="http://www.collada.org/2005/11/COLLADASchema"`),
+	}
+	gmlSigs = []sig{
+		newXmlSig("", `xmlns:gml="http://www.opengis.net/gml"`),
+		newXmlSig("", `xmlns:gml="http://www.opengis.net/gml/3.2"`),
+		newXmlSig("", `xmlns:gml="http://www.opengis.net/gml/3.3/exr"`),
+	}
+	gpxSigs = []sig{
+		newXmlSig("gpx", `xmlns="http://www.topografix.com/GPX/1/1"`),
+	}
+	tcxSigs = []sig{
+		newXmlSig("TrainingCenterDatabase", `xmlns="http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2"`),
+	}
+	x3dSigs = []sig{
+		newXmlSig("X3D", `xmlns:xsd="http://www.w3.org/2001/XMLSchema-instance"`),
+	}
+	amfSigs = []sig{
+		newXmlSig("amf", ""),
+	}
+	threemfSigs = []sig{
+		newXmlSig("model", `xmlns="http://schemas.microsoft.com/3dmanufacturing/core/2015/02"`),
+	}
+	vCardSigs = []sig{
+		ciSig("BEGIN:VCARD\n"),
+	}
+	phpSigs = []sig{
+		ciSig("<?PHP"),
+		ciSig("<?\n"),
+		ciSig("<?\r"),
+		ciSig("<? "),
+		shebangSig("/usr/local/bin/php"),
+		shebangSig("/usr/bin/php"),
+		shebangSig("/usr/bin/env php"),
+	}
+	jsSigs = []sig{
+		shebangSig("/bin/node"),
+		shebangSig("/usr/bin/node"),
+		shebangSig("/bin/nodejs"),
+		shebangSig("/usr/bin/nodejs"),
+		shebangSig("/usr/bin/env node"),
+		shebangSig("/usr/bin/env nodejs"),
+	}
+	luaSigs = []sig{
+		shebangSig("/usr/bin/lua"),
+		shebangSig("/usr/local/bin/lua"),
+		shebangSig("/usr/bin/env lua"),
+	}
+	perlSigs = []sig{
+		shebangSig("/usr/bin/perl"),
+		shebangSig("/usr/bin/env perl"),
+	}
+	pythonSigs = []sig{
+		shebangSig("/usr/bin/python"),
+		shebangSig("/usr/local/bin/python"),
+		shebangSig("/usr/bin/env python"),
+	}
+	tclSigs = []sig{
+		shebangSig("/usr/bin/tcl"),
+		shebangSig("/usr/local/bin/tcl"),
+		shebangSig("/usr/bin/env tcl"),
+		shebangSig("/usr/bin/tclsh"),
+		shebangSig("/usr/local/bin/tclsh"),
+		shebangSig("/usr/bin/env tclsh"),
+		shebangSig("/usr/bin/wish"),
+		shebangSig("/usr/local/bin/wish"),
+		shebangSig("/usr/bin/env wish"),
+	}
+)
+
+// Txt matches a text file.
+func Txt(in []byte) bool {
+	in = trimLWS(in)
+	for _, b := range in {
+		if b <= 0x08 ||
+			b == 0x0B ||
+			0x0E <= b && b <= 0x1A ||
+			0x1C <= b && b <= 0x1F {
+			return false
+		}
+	}
+
+	return true
+}
+
+// Html matches a Hypertext Markup Language file.
+func Html(in []byte) bool {
+	return detect(in, htmlSigs)
+}
+
+// Xml matches an Extensible Markup Language file.
+func Xml(in []byte) bool {
+	return detect(in, xmlSigs)
+}
+
+// Php matches a PHP: Hypertext Preprocessor file.
+func Php(in []byte) bool {
+	return detect(in, phpSigs)
+}
+
+// Json matches a JavaScript Object Notation file.
+func Json(in []byte) bool {
+	parsed, err := json.Scan(in)
+	if len(in) < ReadLimit {
+		return err == nil
+	}
+
+	return parsed == len(in)
+}
+
+// GeoJson matches a RFC 7946 GeoJSON file.
+//
+// BUG(gabriel-vasile): The "type" key should be searched for in the root object.
+func GeoJson(in []byte) bool {
+	if len(in) == 0 {
+		return false
+	}
+	in = trimLWS(in)
+	// geojson is always an object
+	if in[0] != '{' {
+		return false
+	}
+
+	s := []byte(`"type"`)
+	si := bytes.Index(in, s)
+	sl := len(s)
+
+	if si == -1 {
+		return false
+	}
+
+	// if the "type" string is the suffix of the input
+	// there is no need to search for the value of the key
+	if si+sl == len(in) {
+		return false
+	}
+	// skip the "type" part
+	in = in[si+sl:]
+	// skip any whitespace before the colon
+	in = trimLWS(in)
+	// skip any whitesapce after the colon
+	// not checking if char is colon because json matcher already did check
+	in = trimLWS(in[1:])
+
+	geoJsonTypes := [][]byte{
+		[]byte(`"Feature"`),
+		[]byte(`"FeatureCollection"`),
+		[]byte(`"Point"`),
+		[]byte(`"LineString"`),
+		[]byte(`"Polygon"`),
+		[]byte(`"MultiPoint"`),
+		[]byte(`"MultiLineString"`),
+		[]byte(`"MultiPolygon"`),
+		[]byte(`"GeometryCollection"`),
+	}
+	for _, t := range geoJsonTypes {
+		if bytes.HasPrefix(in, t) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Js matches a Javascript file.
+func Js(in []byte) bool {
+	return detect(in, jsSigs)
+}
+
+// Lua matches a Lua programming language file.
+func Lua(in []byte) bool {
+	return detect(in, luaSigs)
+}
+
+// Perl matches a Perl programming language file.
+func Perl(in []byte) bool {
+	return detect(in, perlSigs)
+}
+
+// Python matches a Python programming language file.
+func Python(in []byte) bool {
+	return detect(in, pythonSigs)
+}
+
+// Tcl matches a Tcl programming language file.
+func Tcl(in []byte) bool {
+	return detect(in, tclSigs)
+}
+
+// Rtf matches a Rich Text Format file.
+func Rtf(in []byte) bool {
+	return bytes.HasPrefix(in, []byte("{\\rtf1"))
+}
+
+// Svg matches a SVG file.
+func Svg(in []byte) bool {
+	return bytes.Contains(in, []byte("<svg"))
+}
+
+// Rss matches a Rich Site Summary file.
+func Rss(in []byte) bool {
+	return detect(in, rssSigs)
+}
+
+// Atom matches an Atom Syndication Format file.
+func Atom(in []byte) bool {
+	return detect(in, atomSigs)
+}
+
+// Kml matches a Keyhole Markup Language file.
+func Kml(in []byte) bool {
+	return detect(in, kmlSigs)
+}
+
+// Xliff matches a XML Localization Interchange File Format file.
+func Xliff(in []byte) bool {
+	return detect(in, xliffSigs)
+}
+
+// Collada matches a COLLAborative Design Activity file.
+func Collada(in []byte) bool {
+	return detect(in, colladaSigs)
+}
+
+// Gml matches a Geography Markup Language file.
+func Gml(in []byte) bool {
+	return detect(in, gmlSigs)
+}
+
+// Gpx matches a GPS Exchange Format file.
+func Gpx(in []byte) bool {
+	return detect(in, gpxSigs)
+}
+
+// Tcx matches a Training Center XML file.
+func Tcx(in []byte) bool {
+	return detect(in, tcxSigs)
+}
+
+// Amf matches an Additive Manufacturing XML file.
+func Amf(in []byte) bool {
+	return detect(in, amfSigs)
+}
+
+// Threemf matches a 3D Manufacturing Format file.
+func Threemf(in []byte) bool {
+	return detect(in, threemfSigs)
+}
+
+// X3d matches an Extensible 3D Graphics file.
+func X3d(in []byte) bool {
+	return detect(in, x3dSigs)
+}
+
+// VCard matches a Virtual Contact File.
+func VCard(in []byte) bool {
+	return detect(in, vCardSigs)
+}

vendor/github.com/gabriel-vasile/mimetype/internal/matchers/text_csv.go

@@ -0,0 +1,46 @@
+package matchers
+
+import (
+	"bytes"
+	"encoding/csv"
+	"io"
+)
+
+// Csv matches a comma-separated values file.
+func Csv(in []byte) bool {
+	return sv(in, ',')
+}
+
+// Tsv matches a tab-separated values file.
+func Tsv(in []byte) bool {
+	return sv(in, '\t')
+}
+
+func sv(in []byte, comma rune) bool {
+	r := csv.NewReader(butLastLineReader(in, ReadLimit))
+	r.Comma = comma
+	r.TrimLeadingSpace = true
+	r.LazyQuotes = true
+	r.Comment = '#'
+
+	lines, err := r.ReadAll()
+	return err == nil && r.FieldsPerRecord > 1 && len(lines) > 1
+}
+
+// butLastLineReader returns a reader to the provided byte slice.
+// the reader is guaranteed to reach EOF before it reads `cutAt` bytes.
+// bytes after the last newline are dropped from the input.
+func butLastLineReader(in []byte, cutAt int) io.Reader {
+	if len(in) >= cutAt {
+		for i := cutAt - 1; i > 0; i-- {
+			if in[i] == '\n' {
+				return bytes.NewReader(in[:i])
+			}
+		}
+
+		// no newline was found between the 0 index and cutAt
+		return bytes.NewReader(in[:cutAt])
+	}
+
+	return bytes.NewReader(in)
+}

vendor/github.com/gabriel-vasile/mimetype/internal/matchers/video.go

@@ -0,0 +1,70 @@
+package matchers
+
+import (
+	"bytes"
+)
+
+// WebM matches a WebM file.
+func WebM(in []byte) bool {
+	return isMatroskaFileTypeMatched(in, "webm")
+}
+
+// Mkv matches a mkv file.
+func Mkv(in []byte) bool {
+	return isMatroskaFileTypeMatched(in, "matroska")
+}
+
+// isMatroskaFileTypeMatched is used for webm and mkv file matching.
+// It checks for .Eߣ sequence. If the sequence is found,
+// then it means it is Matroska media container, including WebM.
+// Then it verifies which of the file type it is representing by matching the
+// file specific string.
+func isMatroskaFileTypeMatched(in []byte, flType string) bool {
+	if bytes.HasPrefix(in, []byte("\x1A\x45\xDF\xA3")) {
+		return isFileTypeNamePresent(in, flType)
+	}
+	return false
+}
+
+// isFileTypeNamePresent accepts the matroska input data stream and searches
+// for the given file type in the stream. Return whether a match is found.
+// The logic of search is: find first instance of \x42\x82 and then
+// search for given string after one byte of above instance.
+func isFileTypeNamePresent(in []byte, flType string) bool {
+	var ind int
+	if len(in) >= 4096 { // restricting length to 4096
+		ind = bytes.Index(in[0:4096], []byte("\x42\x82"))
+	} else {
+		ind = bytes.Index(in, []byte("\x42\x82"))
+	}
+	if ind > 0 {
+		// filetype name will be present exactly
+		// one byte after the match of the two bytes "\x42\x82"
+		return bytes.HasPrefix(in[ind+3:], []byte(flType))
+	}
+	return false
+}
+
+// Flv matches a Flash video file.
+func Flv(in []byte) bool {
+	return bytes.HasPrefix(in, []byte("\x46\x4C\x56\x01"))
+}
+
+// Mpeg matches a Moving Picture Experts Group file.
+func Mpeg(in []byte) bool {
+	return bytes.HasPrefix(in, []byte{0x00, 0x00, 0x01}) &&
+		in[3] >= 0xB0 && in[3] <= 0xBF
+}
+
+// Avi matches an Audio Video Interleaved file.
+func Avi(in []byte) bool {
+	return len(in) > 16 &&
+		bytes.Equal(in[:4], []byte("RIFF")) &&
+		bytes.Equal(in[8:16], []byte("AVI LIST"))
+}
+
+// Asf matches an Advanced Systems Format file.
+func Asf(in []byte) bool {
+	return len(in) > 16 && bytes.Equal(in[:16], []byte{0x30, 0x26, 0xB2, 0x75,
+		0x8E, 0x66, 0xCF, 0x11, 0xA6, 0xD9, 0x00, 0xAA, 0x00, 0x62, 0xCE, 0x6C})
+}

vendor/github.com/gabriel-vasile/mimetype/internal/matchers/video_ftyp.go

@@ -0,0 +1,70 @@
+package matchers
+
+var (
+	mp4Sigs = []sig{
+		ftypSig("avc1"), ftypSig("dash"), ftypSig("iso2"), ftypSig("iso3"),
+		ftypSig("iso4"), ftypSig("iso5"), ftypSig("iso6"), ftypSig("isom"),
+		ftypSig("mmp4"), ftypSig("mp41"), ftypSig("mp42"), ftypSig("mp4v"),
+		ftypSig("mp71"), ftypSig("MSNV"), ftypSig("NDAS"), ftypSig("NDSC"),
+		ftypSig("NSDC"), ftypSig("NSDH"), ftypSig("NDSM"), ftypSig("NDSP"),
+		ftypSig("NDSS"), ftypSig("NDXC"), ftypSig("NDXH"), ftypSig("NDXM"),
+		ftypSig("NDXP"), ftypSig("NDXS"), ftypSig("F4V "), ftypSig("F4P "),
+	}
+	threeGPSigs = []sig{
+		ftypSig("3gp1"), ftypSig("3gp2"), ftypSig("3gp3"), ftypSig("3gp4"),
+		ftypSig("3gp5"), ftypSig("3gp6"), ftypSig("3gs7"), ftypSig("3ge6"),
+		ftypSig("3ge7"), ftypSig("3gg6"),
+	}
+	threeG2Sigs = []sig{
+		ftypSig("3g2a"), ftypSig("3g2b"), ftypSig("3g2c"), ftypSig("KDDI"),
+	}
+	amp4Sigs = []sig{
+		// audio for Adobe Flash Player 9+
+		ftypSig("F4A "), ftypSig("F4B "),
+		// Apple iTunes AAC-LC (.M4A) Audio
+		ftypSig("M4B "), ftypSig("M4P "),
+		// MPEG-4 (.MP4) for SonyPSP
+		ftypSig("MSNV"),
+		// Nero Digital AAC Audio
+		ftypSig("NDAS"),
+	}
+	qtSigs  = []sig{ftypSig("qt  "), ftypSig("moov")}
+	mqvSigs = []sig{ftypSig("mqt ")}
+	m4aSigs = []sig{ftypSig("M4A ")}
+	// TODO: add support for remaining video formats at ftyps.com.
+)
+
+// Mp4 matches an MP4 file.
+func Mp4(in []byte) bool {
+	return detect(in, mp4Sigs)
+}
+
+// ThreeGP matches a 3GPP file.
+func ThreeGP(in []byte) bool {
+	return detect(in, threeGPSigs)
+}
+
+// ThreeG2 matches a 3GPP2 file.
+func ThreeG2(in []byte) bool {
+	return detect(in, threeG2Sigs)
+}
+
+// AMp4 matches an audio MP4 file.
+func AMp4(in []byte) bool {
+	return detect(in, amp4Sigs)
+}
+
+// QuickTime matches a QuickTime File Format file.
+func QuickTime(in []byte) bool {
+	return detect(in, qtSigs)
+}
+
+// Mqv matches a Sony / Mobile QuickTime  file.
+func Mqv(in []byte) bool {
+	return detect(in, mqvSigs)
+}
+
+// M4a matches an audio M4A file.
+func M4a(in []byte) bool {
+	return detect(in, m4aSigs)
+}

vendor/github.com/gabriel-vasile/mimetype/mime.go

@@ -0,0 +1,55 @@
+// Package mimetype uses magic number signatures
+// to detect the MIME type and extension of a file.
+package mimetype
+
+import (
+	"io"
+	"os"
+
+	"github.com/gabriel-vasile/mimetype/internal/matchers"
+)
+
+// Detect returns the MIME type and extension of the provided byte slice.
+//
+// mime is always a valid MIME type, with application/octet-stream as fallback.
+// extension is empty string if detected file format does not have an extension.
+func Detect(in []byte) (mime, extension string) {
+	if len(in) == 0 {
+		return "inode/x-empty", ""
+	}
+	n := root.match(in, root)
+	return n.mime, n.extension
+}
+
+// DetectReader returns the MIME type and extension
+// of the byte slice read from the provided reader.
+//
+// mime is always a valid MIME type, with application/octet-stream as fallback.
+// extension is empty string if detection failed with an error or
+// detected file format does not have an extension.
+func DetectReader(r io.Reader) (mime, extension string, err error) {
+	in := make([]byte, matchers.ReadLimit)
+	n, err := r.Read(in)
+	if err != nil && err != io.EOF {
+		return root.mime, root.extension, err
+	}
+	in = in[:n]
+
+	mime, extension = Detect(in)
+	return mime, extension, nil
+}
+
+// DetectFile returns the MIME type and extension of the provided file.
+//
+// mime is always a valid MIME type, with application/octet-stream as fallback.
+// extension is empty string if detection failed with an error or
+// detected file format does not have an extension.
+func DetectFile(file string) (mime, extension string, err error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return root.mime, root.extension, err
+	}
+	defer f.Close()
+
+	return DetectReader(f)
+}

vendor/github.com/gabriel-vasile/mimetype/node.go

@@ -0,0 +1,43 @@
+package mimetype
+
+type (
+	// node represents a vertex in the matchers tree structure.
+	// It holds the mime type, the extension and the function
+	// to check whether a byte slice has the mime type.
+	node struct {
+		mime      string
+		extension string
+		matchFunc func([]byte) bool
+		children  []*node
+	}
+)
+
+func newNode(mime, extension string, matchFunc func([]byte) bool, children ...*node) *node {
+	return &node{
+		mime:      mime,
+		extension: extension,
+		matchFunc: matchFunc,
+		children:  children,
+	}
+}
+
+// match does a depth-first search on the matchers tree.
+// it returns the deepest successful matcher for which all the children fail.
+func (n *node) match(in []byte, deepestMatch *node) *node {
+	for _, c := range n.children {
+		if c.matchFunc(in) {
+			return c.match(in, c)
+		}
+	}
+
+	return deepestMatch
+}
+
+func (n *node) flatten() []*node {
+	out := []*node{n}
+	for _, c := range n.children {
+		out = append(out, c.flatten()...)
+	}
+
+	return out
+}

vendor/github.com/gabriel-vasile/mimetype/supported_mimes.md

@@ -0,0 +1,119 @@
+## 114 Supported MIME types
+This file is automatically generated when running tests. Do not edit manually.
+
+Extension | MIME type
+--------- | --------
+**n/a** | application/octet-stream
+**7z** | application/x-7z-compressed
+**zip** | application/zip
+**xlsx** | application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+**docx** | application/vnd.openxmlformats-officedocument.wordprocessingml.document
+**pptx** | application/vnd.openxmlformats-officedocument.presentationml.presentation
+**epub** | application/epub+zip
+**jar** | application/jar
+**odt** | application/vnd.oasis.opendocument.text
+**ott** | application/vnd.oasis.opendocument.text-template
+**ods** | application/vnd.oasis.opendocument.spreadsheet
+**ots** | application/vnd.oasis.opendocument.spreadsheet-template
+**odp** | application/vnd.oasis.opendocument.presentation
+**otp** | application/vnd.oasis.opendocument.presentation-template
+**odg** | application/vnd.oasis.opendocument.graphics
+**otg** | application/vnd.oasis.opendocument.graphics-template
+**odf** | application/vnd.oasis.opendocument.formula
+**pdf** | application/pdf
+**xls** | application/vnd.ms-excel
+**ppt** | application/vnd.ms-powerpoint
+**doc** | application/msword
+**ps** | application/postscript
+**psd** | application/x-photoshop
+**ogg** | application/ogg
+**png** | image/png
+**jpg** | image/jpeg
+**gif** | image/gif
+**webp** | image/webp
+**exe** | application/vnd.microsoft.portable-executable
+**n/a** | application/x-elf
+**n/a** | application/x-object
+**n/a** | application/x-executable
+**so** | application/x-sharedlib
+**n/a** | application/x-coredump
+**a** | application/x-archive
+**deb** | application/vnd.debian.binary-package
+**tar** | application/x-tar
+**xar** | application/x-xar
+**bz2** | application/x-bzip2
+**fits** | application/fits
+**tiff** | image/tiff
+**bmp** | image/bmp
+**ico** | image/x-icon
+**mp3** | audio/mpeg
+**flac** | audio/flac
+**midi** | audio/midi
+**ape** | audio/ape
+**mpc** | audio/musepack
+**amr** | audio/amr
+**wav** | audio/wav
+**aiff** | audio/aiff
+**au** | audio/basic
+**mpeg** | video/mpeg
+**mov** | video/quicktime
+**mqv** | video/quicktime
+**mp4** | video/mp4
+**webm** | video/webm
+**3gp** | video/3gpp
+**3g2** | video/3gpp2
+**avi** | video/x-msvideo
+**flv** | video/x-flv
+**mkv** | video/x-matroska
+**asf** | video/x-ms-asf
+**aac** | audio/aac
+**voc** | audio/x-unknown
+**mp4** | audio/mp4
+**m4a** | audio/x-m4a
+**txt** | text/plain
+**html** | text/html; charset=utf-8
+**svg** | image/svg+xml
+**xml** | text/xml; charset=utf-8
+**rss** | application/rss+xml
+**atom** | application/atom+xml
+**x3d** | model/x3d+xml
+**kml** | application/vnd.google-earth.kml+xml
+**xlf** | application/x-xliff+xml
+**dae** | model/vnd.collada+xml
+**gml** | application/gml+xml
+**gpx** | application/gpx+xml
+**tcx** | application/vnd.garmin.tcx+xml
+**amf** | application/x-amf
+**3mf** | application/vnd.ms-package.3dmanufacturing-3dmodel+xml
+**php** | text/x-php; charset=utf-8
+**js** | application/javascript
+**lua** | text/x-lua
+**pl** | text/x-perl
+**py** | application/x-python
+**json** | application/json
+**geojson** | application/geo+json
+**rtf** | text/rtf
+**tcl** | text/x-tcl
+**csv** | text/csv
+**tsv** | text/tab-separated-values
+**vcf** | text/vcard
+**gz** | application/gzip
+**class** | application/x-java-applet; charset=binary
+**swf** | application/x-shockwave-flash
+**crx** | application/x-chrome-extension
+**woff** | font/woff
+**woff2** | font/woff2
+**otf** | font/otf
+**eot** | application/vnd.ms-fontobject
+**wasm** | application/wasm
+**shx** | application/octet-stream
+**shp** | application/octet-stream
+**dbf** | application/x-dbf
+**dcm** | application/dicom
+**rar** | application/x-rar-compressed
+**djvu** | image/vnd.djvu
+**mobi** | application/x-mobipocket-ebook
+**lit** | application/x-ms-reader
+**bpg** | image/bpg
+**sqlite** | application/x-sqlite3
+**dwg** | image/vnd.dwg

vendor/github.com/gabriel-vasile/mimetype/tree.go

@@ -0,0 +1,131 @@
+package mimetype
+
+import "github.com/gabriel-vasile/mimetype/internal/matchers"
+
+// root is a matcher which passes for any slice of bytes.
+// When a matcher passes the check, the children matchers
+// are tried in order to find a more accurate mime type.
+var root = newNode("application/octet-stream", "", matchers.True,
+	sevenZ, zip, pdf, xls, ppt, doc, ps, psd, ogg, png, jpg, gif, webp, exe, elf,
+	ar, tar, xar, bz2, fits, tiff, bmp, ico, mp3, flac, midi, ape, musePack, amr,
+	wav, aiff, au, mpeg, quickTime, mqv, mp4, webM, threeGP, threeG2, avi, flv,
+	mkv, asf, aac, voc, aMp4, m4a, txt, gzip, class, swf, crx, woff, woff2, otf,
+	eot, wasm, shx, dbf, dcm, rar, djvu, mobi, lit, bpg, sqlite3, dwg,
+)
+
+// The list of nodes appended to the root node
+var (
+	gzip      = newNode("application/gzip", "gz", matchers.Gzip)
+	sevenZ    = newNode("application/x-7z-compressed", "7z", matchers.SevenZ)
+	zip       = newNode("application/zip", "zip", matchers.Zip, xlsx, docx, pptx, epub, jar, odt, ods, odp, odg, odf)
+	tar       = newNode("application/x-tar", "tar", matchers.Tar)
+	xar       = newNode("application/x-xar", "xar", matchers.Xar)
+	bz2       = newNode("application/x-bzip2", "bz2", matchers.Bz2)
+	pdf       = newNode("application/pdf", "pdf", matchers.Pdf)
+	xlsx      = newNode("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xlsx", matchers.Xlsx)
+	docx      = newNode("application/vnd.openxmlformats-officedocument.wordprocessingml.document", "docx", matchers.Docx)
+	pptx      = newNode("application/vnd.openxmlformats-officedocument.presentationml.presentation", "pptx", matchers.Pptx)
+	epub      = newNode("application/epub+zip", "epub", matchers.Epub)
+	jar       = newNode("application/jar", "jar", matchers.Jar)
+	doc       = newNode("application/msword", "doc", matchers.Doc)
+	ppt       = newNode("application/vnd.ms-powerpoint", "ppt", matchers.Ppt)
+	xls       = newNode("application/vnd.ms-excel", "xls", matchers.Xls)
+	ps        = newNode("application/postscript", "ps", matchers.Ps)
+	psd       = newNode("application/x-photoshop", "psd", matchers.Psd)
+	fits      = newNode("application/fits", "fits", matchers.Fits)
+	ogg       = newNode("application/ogg", "ogg", matchers.Ogg)
+	txt       = newNode("text/plain", "txt", matchers.Txt, html, svg, xml, php, js, lua, perl, python, json, rtf, tcl, csv, tsv, vCard)
+	xml       = newNode("text/xml; charset=utf-8", "xml", matchers.Xml, rss, atom, x3d, kml, xliff, collada, gml, gpx, tcx, amf, threemf)
+	json      = newNode("application/json", "json", matchers.Json, geoJson)
+	csv       = newNode("text/csv", "csv", matchers.Csv)
+	tsv       = newNode("text/tab-separated-values", "tsv", matchers.Tsv)
+	geoJson   = newNode("application/geo+json", "geojson", matchers.GeoJson)
+	html      = newNode("text/html; charset=utf-8", "html", matchers.Html)
+	php       = newNode("text/x-php; charset=utf-8", "php", matchers.Php)
+	rtf       = newNode("text/rtf", "rtf", matchers.Rtf)
+	js        = newNode("application/javascript", "js", matchers.Js)
+	lua       = newNode("text/x-lua", "lua", matchers.Lua)
+	perl      = newNode("text/x-perl", "pl", matchers.Perl)
+	python    = newNode("application/x-python", "py", matchers.Python)
+	tcl       = newNode("text/x-tcl", "tcl", matchers.Tcl)
+	vCard     = newNode("text/vcard", "vcf", matchers.VCard)
+	svg       = newNode("image/svg+xml", "svg", matchers.Svg)
+	rss       = newNode("application/rss+xml", "rss", matchers.Rss)
+	atom      = newNode("application/atom+xml", "atom", matchers.Atom)
+	x3d       = newNode("model/x3d+xml", "x3d", matchers.X3d)
+	kml       = newNode("application/vnd.google-earth.kml+xml", "kml", matchers.Kml)
+	xliff     = newNode("application/x-xliff+xml", "xlf", matchers.Xliff)
+	collada   = newNode("model/vnd.collada+xml", "dae", matchers.Collada)
+	gml       = newNode("application/gml+xml", "gml", matchers.Gml)
+	gpx       = newNode("application/gpx+xml", "gpx", matchers.Gpx)
+	tcx       = newNode("application/vnd.garmin.tcx+xml", "tcx", matchers.Tcx)
+	amf       = newNode("application/x-amf", "amf", matchers.Amf)
+	threemf   = newNode("application/vnd.ms-package.3dmanufacturing-3dmodel+xml", "3mf", matchers.Threemf)
+	png       = newNode("image/png", "png", matchers.Png)
+	jpg       = newNode("image/jpeg", "jpg", matchers.Jpg)
+	bpg       = newNode("image/bpg", "bpg", matchers.Bpg)
+	gif       = newNode("image/gif", "gif", matchers.Gif)
+	webp      = newNode("image/webp", "webp", matchers.Webp)
+	tiff      = newNode("image/tiff", "tiff", matchers.Tiff)
+	bmp       = newNode("image/bmp", "bmp", matchers.Bmp)
+	ico       = newNode("image/x-icon", "ico", matchers.Ico)
+	mp3       = newNode("audio/mpeg", "mp3", matchers.Mp3)
+	flac      = newNode("audio/flac", "flac", matchers.Flac)
+	midi      = newNode("audio/midi", "midi", matchers.Midi)
+	ape       = newNode("audio/ape", "ape", matchers.Ape)
+	musePack  = newNode("audio/musepack", "mpc", matchers.MusePack)
+	wav       = newNode("audio/wav", "wav", matchers.Wav)
+	aiff      = newNode("audio/aiff", "aiff", matchers.Aiff)
+	au        = newNode("audio/basic", "au", matchers.Au)
+	amr       = newNode("audio/amr", "amr", matchers.Amr)
+	aac       = newNode("audio/aac", "aac", matchers.Aac)
+	voc       = newNode("audio/x-unknown", "voc", matchers.Voc)
+	aMp4      = newNode("audio/mp4", "mp4", matchers.AMp4)
+	m4a       = newNode("audio/x-m4a", "m4a", matchers.M4a)
+	mp4       = newNode("video/mp4", "mp4", matchers.Mp4)
+	webM      = newNode("video/webm", "webm", matchers.WebM)
+	mpeg      = newNode("video/mpeg", "mpeg", matchers.Mpeg)
+	quickTime = newNode("video/quicktime", "mov", matchers.QuickTime)
+	mqv       = newNode("video/quicktime", "mqv", matchers.Mqv)
+	threeGP   = newNode("video/3gpp", "3gp", matchers.ThreeGP)
+	threeG2   = newNode("video/3gpp2", "3g2", matchers.ThreeG2)
+	avi       = newNode("video/x-msvideo", "avi", matchers.Avi)
+	flv       = newNode("video/x-flv", "flv", matchers.Flv)
+	mkv       = newNode("video/x-matroska", "mkv", matchers.Mkv)
+	asf       = newNode("video/x-ms-asf", "asf", matchers.Asf)
+	class     = newNode("application/x-java-applet; charset=binary", "class", matchers.Class)
+	swf       = newNode("application/x-shockwave-flash", "swf", matchers.Swf)
+	crx       = newNode("application/x-chrome-extension", "crx", matchers.Crx)
+	woff      = newNode("font/woff", "woff", matchers.Woff)
+	woff2     = newNode("font/woff2", "woff2", matchers.Woff2)
+	otf       = newNode("font/otf", "otf", matchers.Otf)
+	eot       = newNode("application/vnd.ms-fontobject", "eot", matchers.Eot)
+	wasm      = newNode("application/wasm", "wasm", matchers.Wasm)
+	shp       = newNode("application/octet-stream", "shp", matchers.Shp)
+	shx       = newNode("application/octet-stream", "shx", matchers.Shx, shp)
+	dbf       = newNode("application/x-dbf", "dbf", matchers.Dbf)
+	exe       = newNode("application/vnd.microsoft.portable-executable", "exe", matchers.Exe)
+	elf       = newNode("application/x-elf", "", matchers.Elf, elfObj, elfExe, elfLib, elfDump)
+	elfObj    = newNode("application/x-object", "", matchers.ElfObj)
+	elfExe    = newNode("application/x-executable", "", matchers.ElfExe)
+	elfLib    = newNode("application/x-sharedlib", "so", matchers.ElfLib)
+	elfDump   = newNode("application/x-coredump", "", matchers.ElfDump)
+	ar        = newNode("application/x-archive", "a", matchers.Ar, deb)
+	deb       = newNode("application/vnd.debian.binary-package", "deb", matchers.Deb)
+	dcm       = newNode("application/dicom", "dcm", matchers.Dcm)
+	odt       = newNode("application/vnd.oasis.opendocument.text", "odt", matchers.Odt, ott)
+	ott       = newNode("application/vnd.oasis.opendocument.text-template", "ott", matchers.Ott)
+	ods       = newNode("application/vnd.oasis.opendocument.spreadsheet", "ods", matchers.Ods, ots)
+	ots       = newNode("application/vnd.oasis.opendocument.spreadsheet-template", "ots", matchers.Ots)
+	odp       = newNode("application/vnd.oasis.opendocument.presentation", "odp", matchers.Odp, otp)
+	otp       = newNode("application/vnd.oasis.opendocument.presentation-template", "otp", matchers.Otp)
+	odg       = newNode("application/vnd.oasis.opendocument.graphics", "odg", matchers.Odg, otg)
+	otg       = newNode("application/vnd.oasis.opendocument.graphics-template", "otg", matchers.Otg)
+	odf       = newNode("application/vnd.oasis.opendocument.formula", "odf", matchers.Odf)
+	rar       = newNode("application/x-rar-compressed", "rar", matchers.Rar)
+	djvu      = newNode("image/vnd.djvu", "djvu", matchers.DjVu)
+	mobi      = newNode("application/x-mobipocket-ebook", "mobi", matchers.Mobi)
+	lit       = newNode("application/x-ms-reader", "lit", matchers.Lit)
+	sqlite3   = newNode("application/x-sqlite3", "sqlite", matchers.Sqlite)
+	dwg       = newNode("image/vnd.dwg", "dwg", matchers.Dwg)
+)