diff --git a/Gopkg.lock b/Gopkg.lock index 72bf7ff6b..0e8dfe6a5 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -100,6 +100,18 @@ pruneopts = "UT" revision = "b57537c92a6b4155c5219ce0d7d2a9c0cc74b1c4" +[[projects]] + digest = "1:04904b7a6f24221e7999b554094e347885b4837beeba9933efadc93f4f5421ff" + name = "github.com/gabriel-vasile/mimetype" + packages = [ + ".", + "internal/json", + "internal/matchers", + ] + pruneopts = "UT" + revision = "b9686d36d26efccccd34eff91a19f42a61bcdc95" + version = "v0.3.17" + [[projects]] digest = "1:f59f34bb582fbc13885cd0338570bc0f21738fc711065527c36928213de62c1b" name = "github.com/getsentry/sentry-go" @@ -690,6 +702,7 @@ "github.com/dgrijalva/jwt-go", "github.com/disintegration/imaging", "github.com/edwvee/exiffix", + "github.com/gabriel-vasile/mimetype", "github.com/getsentry/sentry-go", "github.com/getsentry/sentry-go/http", "github.com/go-chi/chi", diff --git a/Gopkg.toml b/Gopkg.toml index 244d52ecf..c751eac86 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -29,3 +29,7 @@ required = [ [[prune.project]] name = "github.com/cortezaproject/corteza-protobuf" unused-packages = false + +[[constraint]] + name = "github.com/gabriel-vasile/mimetype" + version = "0.3.17" diff --git a/api/compose/spec.json b/api/compose/spec.json index 6729ef0d0..18b3166bb 100644 --- a/api/compose/spec.json +++ b/api/compose/spec.json @@ -667,6 +667,68 @@ ] } }, + { + "name": "importInit", + "path": "/import", + "method": "POST", + "title": "Initiate record import session", + "parameters": { + "post": [ + { + "name": "upload", + "type": "*multipart.FileHeader", + "required": true, + "title": "File import" + } + ] + } + }, + { + "name": "importRun", + "path": "/import/{sessionID}", + "method": "PATCH", + "title": "Run record import", + "parameters": { + "path": [ + { + "name": "sessionID", + "type": "uint64", + "required": true, + "title": "Import session" + } + ], + "post": [ + { + "name": "fields", + "type": "json.RawMessage", + "required": true, + "title": "Fields defined by import file" + }, + { + "name": "onError", + "type": "string", + "required": true, + "title": "What happens if record fails to import" + } + ] + } + }, + { + "name": "importProgress", + "path": "/import/{sessionID}", + "method": "GET", + "title": "Get import progress", + "parameters": { + "path": [ + { + "name": "sessionID", + "type": "uint64", + "required": true, + "title": "Import session" + } + ] + } + }, { "name": "export", "path": "/export{filename}.{ext}", diff --git a/api/compose/spec/record.json b/api/compose/spec/record.json index f6aacf563..9b26fd875 100644 --- a/api/compose/spec/record.json +++ b/api/compose/spec/record.json @@ -91,6 +91,68 @@ ] } }, + { + "Name": "importInit", + "Method": "POST", + "Title": "Initiate record import session", + "Path": "/import", + "Parameters": { + "post": [ + { + "name": "upload", + "required": true, + "title": "File import", + "type": "*multipart.FileHeader" + } + ] + } + }, + { + "Name": "importRun", + "Method": "PATCH", + "Title": "Run record import", + "Path": "/import/{sessionID}", + "Parameters": { + "path": [ + { + "name": "sessionID", + "required": true, + "title": "Import session", + "type": "uint64" + } + ], + "post": [ + { + "name": "fields", + "required": true, + "title": "Fields defined by import file", + "type": "json.RawMessage" + }, + { + "name": "onError", + "required": true, + "title": "What happens if record fails to import", + "type": "string" + } + ] + } + }, + { + "Name": "importProgress", + "Method": "GET", + "Title": "Get import progress", + "Path": "/import/{sessionID}", + "Parameters": { + "path": [ + { + "name": "sessionID", + "required": true, + "title": "Import session", + "type": "uint64" + } + ] + } + }, { "Name": "export", "Method": "GET", diff --git a/compose/decoder/decoder.go b/compose/decoder/decoder.go new file mode 100644 index 000000000..4a9387644 --- /dev/null +++ b/compose/decoder/decoder.go @@ -0,0 +1,169 @@ +package decoder + +import ( + "io" + + "github.com/cortezaproject/corteza-server/pkg/count" +) + +type ( + multiple uint + + FlatReader interface { + Read() ([]string, error) + } + + StructuredDecoder interface { + Decode(interface{}) error + More() bool + } + + flatReader struct { + f io.ReadSeeker + r FlatReader + header []string + more bool + } + + structuredDecoder struct { + f io.ReadSeeker + header []string + d StructuredDecoder + buf []map[string]interface{} + } + + // callbacks + sdCallback func(map[string]interface{}) error + fdCallback func([]string) error +) + +// flat reader +func NewFlatReader(r FlatReader, f io.ReadSeeker) *flatReader { + return &flatReader{ + f: f, + r: r, + more: true, + } +} + +func (dec *flatReader) EntryCount() (uint64, error) { + defer dec.f.Seek(0, 0) + + c, err := count.Lines(dec.f) + if err != nil { + return 0, err + } + if c <= 0 { + return 0, nil + } + return c - 1, nil +} + +func (dec *flatReader) get(fnc fdCallback) error { + v, err := dec.r.Read() + if err == io.EOF { + dec.more = false + return nil + } else if err != nil { + return err + } + + return fnc(v) +} + +func (dec *flatReader) walk(fnc fdCallback) error { + for dec.more { + if err := dec.get(fnc); err != nil { + return err + } + } + return nil +} + +func (dec *flatReader) Header() []string { + if len(dec.header) > 0 { + return dec.header + } + + dec.get(func(rtr []string) error { + dec.header = rtr + return nil + }) + + return dec.header +} + +// structured decoder +func NewStructuredDecoder(d StructuredDecoder, f io.ReadSeeker) *structuredDecoder { + return &structuredDecoder{ + f: f, + d: d, + } +} + +func (dec *structuredDecoder) EntryCount() (uint64, error) { + defer dec.f.Seek(0, 0) + return count.Lines(dec.f) +} + +func (dec *structuredDecoder) get(fnc sdCallback) error { + if !dec.d.More() { + return nil + } + + var tmp map[string]interface{} + err := dec.d.Decode(&tmp) + if err != nil { + return err + } + + return fnc(tmp) +} + +func (dec *structuredDecoder) exhaustBuffer(fnc sdCallback) error { + if dec.buf != nil { + for _, b := range dec.buf { + fnc(b) + } + dec.buf = nil + } + return nil +} + +func (dec *structuredDecoder) walk(fnc sdCallback) error { + if err := dec.exhaustBuffer(fnc); err != nil { + return err + } + + for dec.d.More() { + if err := dec.get(fnc); err != nil { + return err + } + } + + return nil +} + +func (dec *structuredDecoder) Header() []string { + if len(dec.header) > 0 { + return dec.header + } + + var tmp []string + dec.get(func(rtr map[string]interface{}) error { + // buffer first row or else it will be lost + dec.buf = append(dec.buf, rtr) + + tmp = make([]string, len(rtr)) + i := 0 + for k := range rtr { + tmp[i] = k + i++ + } + + return nil + }) + + dec.header = tmp + return tmp +} diff --git a/compose/decoder/decoder_test.go b/compose/decoder/decoder_test.go new file mode 100644 index 000000000..b9da09e46 --- /dev/null +++ b/compose/decoder/decoder_test.go @@ -0,0 +1,179 @@ +package decoder + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "io" + "testing" + + "github.com/cortezaproject/corteza-server/internal/test" +) + +func makeReadSeeker(c string) io.ReadSeeker { + bb := []byte(c) + return bytes.NewReader(bb) +} + +const ( + testCSV string = "f1,f2,ID\nr1v1,r1v2,1\n" + testJSONL string = "{ \"f1\": \"nr1v1\", \"f2\": \"r1v2\", \"ID\": \"1\" }\n" +) + +func TestEntryCount(t *testing.T) { + t.Run("Flat reader", func(t *testing.T) { + rs := makeReadSeeker(testCSV) + fr := NewFlatReader(csv.NewReader(rs), rs) + + c, err := fr.EntryCount() + test.Assert(t, + c == 1, + fmt.Sprintf("Invalid number of entries determines; found %d, expected %d", c, 1), + ) + + test.Assert(t, + err == nil, + "Returned with error", + ) + }) + + t.Run("Structured decoder", func(t *testing.T) { + rs := makeReadSeeker(testJSONL) + sd := NewStructuredDecoder(json.NewDecoder(rs), rs) + + c, err := sd.EntryCount() + test.Assert(t, + c == 1, + fmt.Sprintf("Invalid number of entries determines; found %d, expected %d", c, 1), + ) + + test.Assert(t, + err == nil, + "Returned with error", + ) + }) +} + +func TestGet(t *testing.T) { + t.Run("Flat reader", func(t *testing.T) { + rs := makeReadSeeker(testCSV) + fr := NewFlatReader(csv.NewReader(rs), rs) + + // dump first line + fr.get(func(f []string) error { return nil }) + err := fr.get(func(f []string) error { + return errors.New("called") + }) + test.Assert(t, + err != nil, + "Error should be returned to indicate that get did read", + ) + + err = fr.get(func(f []string) error { + return errors.New("called") + }) + test.Assert(t, + err == nil, + "Error should NOT be returned to indicate that get didn't read", + ) + }) + + t.Run("Structured decoder", func(t *testing.T) { + rs := makeReadSeeker(testJSONL) + sd := NewStructuredDecoder(json.NewDecoder(rs), rs) + + err := sd.get(func(f map[string]interface{}) error { + return errors.New("called") + }) + test.Assert(t, + err != nil, + "Error should be returned to indicate that get did read", + ) + + err = sd.get(func(f map[string]interface{}) error { + return errors.New("called") + }) + test.Assert(t, + err == nil, + "Error should NOT be returned to indicate that get didn't read", + ) + }) +} + +func TestWalk(t *testing.T) { + t.Run("Flat reader", func(t *testing.T) { + rs := makeReadSeeker(testCSV) + fr := NewFlatReader(csv.NewReader(rs), rs) + + i := 0 + err := fr.walk(func(f []string) error { + i++ + return nil + }) + + test.Assert(t, + i == 2, + "Invalid number of reads", + ) + + test.Assert(t, + err == nil, + "Returned with error", + ) + }) + + t.Run("Structured decoder", func(t *testing.T) { + rs := makeReadSeeker(testJSONL) + sd := NewStructuredDecoder(json.NewDecoder(rs), rs) + + i := 0 + err := sd.walk(func(f map[string]interface{}) error { + i++ + return nil + }) + + test.Assert(t, + i == 1, + "Invalid number of reads", + ) + + test.Assert(t, + err == nil, + "Returned with error", + ) + }) +} + +func TestHeader(t *testing.T) { + t.Run("Flat reader", func(t *testing.T) { + rs := makeReadSeeker(testCSV) + fr := NewFlatReader(csv.NewReader(rs), rs) + + h := fr.Header() + test.Assert(t, + len(h) == 3, + "Invalid number of header fields", + ) + + expect := [...]string{"f1", "f2", "ID"} + for i, h := range h { + test.Assert(t, + h == expect[i], + "Invalid header value", + ) + } + }) + + t.Run("Structured decoder", func(t *testing.T) { + rs := makeReadSeeker(testJSONL) + sd := NewStructuredDecoder(json.NewDecoder(rs), rs) + + h := sd.Header() + test.Assert(t, + len(h) == 3, + "Invalid number of header fields", + ) + }) +} diff --git a/compose/decoder/record.go b/compose/decoder/record.go new file mode 100644 index 000000000..6b338eaa2 --- /dev/null +++ b/compose/decoder/record.go @@ -0,0 +1,135 @@ +package decoder + +import ( + "errors" + "fmt" + "strconv" + "time" + + "github.com/cortezaproject/corteza-server/compose/types" +) + +type ( + RecordCreator func(mod *types.Record) error +) + +func fmtTime(tp string) (time.Time, error) { + return time.Parse(time.RFC3339, tp) +} +func fmtTimePtr(tp string) (*time.Time, error) { + t, err := fmtTime(tp) + if err != nil { + return nil, err + } + return &t, nil +} + +func mapify(header, values []string) map[string]string { + if len(header) != len(values) { + return nil + } + + rtr := make(map[string]string) + for i, v := range values { + rtr[header[i]] = v + } + + return rtr +} + +func setSystemField(r *types.Record, name, value string) (is bool, err error) { + switch name { + case "recordID", "ID": + r.ID, err = strconv.ParseUint(value, 10, 64) + case "moduleID": + r.ModuleID, err = strconv.ParseUint(value, 10, 64) + case "namespaceID": + r.NamespaceID, err = strconv.ParseUint(value, 10, 64) + case "ownedBy": + r.OwnedBy, err = strconv.ParseUint(value, 10, 64) + case "createdBy": + r.CreatedBy, err = strconv.ParseUint(value, 10, 64) + case "createdAt": + r.CreatedAt, err = fmtTime(value) + case "updatedBy": + r.UpdatedBy, err = strconv.ParseUint(value, 10, 64) + case "updatedAt": + r.UpdatedAt, err = fmtTimePtr(value) + case "deletedBy": + r.DeletedBy, err = strconv.ParseUint(value, 10, 64) + case "deletedAt": + r.DeletedAt, err = fmtTimePtr(value) + default: + return false, err + } + return true, err +} + +func (dec flatReader) Records(fields map[string]string, Create RecordCreator) error { + header := dec.Header() + + err := dec.walk(func(row []string) error { + mapped := mapify(header, row) + r := types.Record{} + rvs := types.RecordValueSet{} + + i := 0 + for imp, rec := range fields { + if rec == "" { + return errors.New("Can not import record: Record field not defined") + } + + val := mapped[imp] + if system, err := setSystemField(&r, rec, val); err != nil { + return err + } else if !system { + rv := types.RecordValue{ + Name: rec, + Value: val, + Place: uint(i), + } + i++ + + rvs = append(rvs, &rv) + } + } + + r.Values = rvs + return Create(&r) + }) + + return err +} + +func (dec structuredDecoder) Records(fields map[string]string, Create RecordCreator) error { + err := dec.walk(func(entry map[string]interface{}) error { + r := types.Record{} + rvs := types.RecordValueSet{} + + i := 0 + for imp, rec := range fields { + if rec == "" { + return errors.New("Can not import record: Record field not defined") + } + + val := fmt.Sprintf("%v", entry[imp]) + if system, err := setSystemField(&r, rec, val); err != nil { + return err + } else if !system { + rv := types.RecordValue{ + Name: rec, + Value: val, + Place: uint(i), + } + i++ + + rvs = append(rvs, &rv) + } + } + + r.Values = rvs + return Create(&r) + }) + + return err +} diff --git a/compose/decoder/record_test.go b/compose/decoder/record_test.go new file mode 100644 index 000000000..c1a05f900 --- /dev/null +++ b/compose/decoder/record_test.go @@ -0,0 +1,135 @@ +package decoder + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/cortezaproject/corteza-server/compose/types" + "github.com/cortezaproject/corteza-server/internal/test" +) + +func TestMapify(t *testing.T) { + t.Run("Fail if lengths missmatch", func(t *testing.T) { + var h []string + h = append(h, "h1", "h2") + + var v []string + v = append(v, "v1") + + test.Assert(t, + mapify(h, v) == nil, + "Value should be nil", + ) + + test.Assert(t, + mapify(v, h) == nil, + "Value should be nil", + ) + }) + + t.Run("Successfully mapped", func(t *testing.T) { + var h []string + h = append(h, "h1", "h2") + + var v []string + v = append(v, "v1", "v2") + + mpd := mapify(h, v) + test.Assert(t, + len(mpd) == 2, + fmt.Sprintf("Invalid length %d; should be %d", len(mpd), 2), + ) + + test.Assert(t, + mpd["h1"] == "v1" && mpd["h2"] == "v2", + "Invalid values", + ) + }) +} + +func TestSetSystemField(t *testing.T) { + t.Run("Correctly determine & set", func(t *testing.T) { + r := &types.Record{} + name := "recordID" + value := "123" + is, err := setSystemField(r, name, value) + test.Assert(t, + err == nil, + "Returned with error", + ) + + test.Assert(t, + is, + "Couldn't determine it's a system field", + ) + + test.Assert(t, + r.ID == 123, + fmt.Sprintf("Determined value (%d) not valid; should be %s", r.ID, value), + ) + }) + + t.Run("Correctly determine that it's not", func(t *testing.T) { + r := &types.Record{} + name := "customField" + value := "123" + is, err := setSystemField(r, name, value) + test.Assert(t, + err == nil, + "Returned with error", + ) + + test.Assert(t, + !is, + "Couldn't determine it's not a system field", + ) + }) +} + +func TestRecords(t *testing.T) { + testFields := make(map[string]string) + testFields["f1"] = "f1" + testFields["f2"] = "f2" + testFields["ID"] = "ID" + + t.Run("Flat reader", func(t *testing.T) { + fr := NewFlatReader(csv.NewReader(strings.NewReader(testCSV)), nil) + row := 0 + fr.Records(testFields, func(mod *types.Record) error { + row++ + test.Assert(t, + len(mod.Values) == 2, + "Not enough values", + ) + + test.Assert(t, + mod.ID == uint64(row), + "Not enough values", + ) + + return nil + }) + }) + + t.Run("Structured decoder", func(t *testing.T) { + sd := NewStructuredDecoder(json.NewDecoder(strings.NewReader(testJSONL)), nil) + row := 0 + sd.Records(testFields, func(mod *types.Record) error { + row++ + test.Assert(t, + len(mod.Values) == 2, + "Not enough values", + ) + + test.Assert(t, + mod.ID == uint64(row), + "Not enough values", + ) + + return nil + }) + }) +} diff --git a/compose/internal/service/import_session.go b/compose/internal/service/import_session.go new file mode 100644 index 000000000..a27eef4cd --- /dev/null +++ b/compose/internal/service/import_session.go @@ -0,0 +1,133 @@ +package service + +import ( + "context" + "sync" + "time" + + "github.com/cortezaproject/corteza-server/internal/auth" + "github.com/titpetric/factory" + + "github.com/pkg/errors" + "go.uber.org/zap" +) + +type ( + recordSet []*RecordImportSession + + importSession struct { + l sync.Mutex + logger *zap.Logger + + records recordSet + } + + ImportSessionService interface { + FindRecordByID(ctx context.Context, sessionID uint64) (*RecordImportSession, error) + SetRecordByID(ctx context.Context, sessionID, namespaceID, moduleID uint64, fields map[string]string, progress *RecordImportProgress, decoder Decoder) (*RecordImportSession, error) + DeleteRecordByID(ctx context.Context, sessionID uint64) error + } +) + +func ImportSession() *importSession { + return &importSession{ + logger: DefaultLogger.Named("importSession"), + records: recordSet{}, + } +} + +func (svc importSession) indexOf(userID, sessionID uint64) int { + for i, r := range svc.records { + if r.SessionID == sessionID && r.UserID == userID { + return i + } + } + + return -1 +} + +func (svc *importSession) FindRecordByID(ctx context.Context, sessionID uint64) (*RecordImportSession, error) { + svc.l.Lock() + defer svc.l.Unlock() + + userID := auth.GetIdentityFromContext(ctx).Identity() + i := svc.indexOf(userID, sessionID) + if i >= 0 { + return svc.records[i], nil + } + return nil, errors.New("Can't access session: session not found") +} + +func (svc *importSession) SetRecordByID(ctx context.Context, sessionID, namespaceID, moduleID uint64, fields map[string]string, progress *RecordImportProgress, decoder Decoder) (*RecordImportSession, error) { + svc.l.Lock() + defer svc.l.Unlock() + + userID := auth.GetIdentityFromContext(ctx).Identity() + i := svc.indexOf(userID, sessionID) + var ris *RecordImportSession + + if i >= 0 { + ris = svc.records[i] + } else { + ris = &RecordImportSession{ + SessionID: factory.Sonyflake.NextID(), + CreatedAt: time.Now(), + } + svc.records = append(svc.records, ris) + ris.UserID = userID + } + ris.UpdatedAt = time.Now() + + if namespaceID > 0 { + ris.NamespaceID = namespaceID + } + if moduleID > 0 { + ris.ModuleID = moduleID + } + if fields != nil { + ris.Fields = fields + } + if progress != nil { + ris.Progress = *progress + } + + if ris.Progress.FinishedAt != nil { + ris.Decoder = nil + } else if decoder != nil { + ris.Decoder = decoder + } + + return ris, nil +} + +// https://stackoverflow.com/a/37335777 +func remove(s recordSet, i int) recordSet { + s[len(s)-1], s[i] = s[i], s[len(s)-1] + return s[:len(s)-1] +} + +func (svc *importSession) DeleteRecordByID(ctx context.Context, sessionID uint64) error { + svc.l.Lock() + defer svc.l.Unlock() + + userID := auth.GetIdentityFromContext(ctx).Identity() + i := svc.indexOf(userID, sessionID) + + if i >= 0 { + svc.records = remove(svc.records, i) + } + return nil +} + +// @todo run this in some interval +func (svc *importSession) clean(ctx context.Context) { + svc.l.Lock() + defer svc.l.Unlock() + + for i := len(svc.records) - 1; i >= 0; i-- { + r := svc.records[i] + if time.Now().After(r.UpdatedAt.Add(time.Hour * 24 * 3)) { + svc.records = remove(svc.records, i) + } + } +} diff --git a/compose/internal/service/import_session_test.go b/compose/internal/service/import_session_test.go new file mode 100644 index 000000000..02097e3cc --- /dev/null +++ b/compose/internal/service/import_session_test.go @@ -0,0 +1,111 @@ +package service + +import ( + "context" + "testing" + + "go.uber.org/zap" + + "github.com/cortezaproject/corteza-server/internal/test" +) + +var ctx context.Context = context.WithValue(context.Background(), "testing", true) + +func TestFindRecordByID(t *testing.T) { + DefaultLogger = zap.New(nil, nil) + svc := ImportSession() + ss, _ := svc.SetRecordByID(ctx, 1, 0, 0, nil, nil, nil) + sid := ss.SessionID + + t.Run("Found", func(t *testing.T) { + s, err := svc.FindRecordByID(ctx, sid) + test.Assert(t, + s != nil, + "Session should be found", + ) + + test.Assert(t, + err == nil, + "Returned with error", + ) + }) + + t.Run("Not found", func(t *testing.T) { + s, err := svc.FindRecordByID(ctx, sid+1) + test.Assert(t, + s == nil, + "Session should not be found", + ) + + test.Assert(t, + err != nil, + "Error should not be nil", + ) + }) +} + +func TestSetRecordByID(t *testing.T) { + DefaultLogger = zap.New(nil, nil) + svc := ImportSession() + + t.Run("New", func(t *testing.T) { + ss, err := svc.SetRecordByID(ctx, 1, 0, 0, nil, nil, nil) + test.Assert(t, + len(svc.records) == 1 && ss != nil, + "Session should be created", + ) + + test.Assert(t, + err == nil, + "Returned with error", + ) + }) + + t.Run("Existing", func(t *testing.T) { + svc := ImportSession() + ss, err := svc.SetRecordByID(ctx, 1, 0, 0, nil, nil, nil) + ns, err := svc.SetRecordByID(ctx, ss.SessionID, 0, 0, nil, nil, nil) + test.Assert(t, + len(svc.records) == 1 && ns != nil && ss.SessionID == ns.SessionID, + "Existing session should be edited", + ) + + test.Assert(t, + err == nil, + "Returned with error", + ) + }) +} + +func TestDeleteRecordByID(t *testing.T) { + DefaultLogger = zap.New(nil, nil) + svc := ImportSession() + ss, _ := svc.SetRecordByID(ctx, 1, 0, 0, nil, nil, nil) + + t.Run("Delete existing", func(t *testing.T) { + err := svc.DeleteRecordByID(ctx, ss.SessionID) + test.Assert(t, + len(svc.records) == 0, + "Session should be deleted", + ) + + test.Assert(t, + err == nil, + "Returned with error", + ) + }) + + t.Run("Session not found", func(t *testing.T) { + ss, _ := svc.SetRecordByID(ctx, 1, 0, 0, nil, nil, nil) + err := svc.DeleteRecordByID(ctx, ss.SessionID+1) + test.Assert(t, + len(svc.records) == 1, + "Session should not deleted", + ) + + test.Assert(t, + err == nil, + "Returned with error", + ) + }) +} diff --git a/compose/internal/service/record.go b/compose/internal/service/record.go index 846d8bf85..9ab4a1b83 100644 --- a/compose/internal/service/record.go +++ b/compose/internal/service/record.go @@ -10,11 +10,17 @@ import ( "github.com/titpetric/factory" "go.uber.org/zap" + "github.com/cortezaproject/corteza-server/compose/decoder" "github.com/cortezaproject/corteza-server/compose/internal/repository" "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/internal/auth" ) +const ( + IMPORT_ON_ERROR_SKIP = "SKIP" + IMPORT_ON_ERROR_FAIL = "FAIL" +) + type ( record struct { db *factory.DB @@ -57,6 +63,7 @@ type ( Report(namespaceID, moduleID uint64, metrics, dimensions, filter string) (interface{}, error) Find(filter types.RecordFilter) (set types.RecordSet, f types.RecordFilter, err error) Export(types.RecordFilter, Encoder) error + Import(*RecordImportSession, ImportSessionService) error Create(record *types.Record) (*types.Record, error) Update(record *types.Record) (*types.Record, error) @@ -67,6 +74,34 @@ type ( Encoder interface { Record(*types.Record) error } + + Decoder interface { + Header() []string + EntryCount() (uint64, error) + Records(fields map[string]string, Create decoder.RecordCreator) error + } + + RecordImportSession struct { + Decoder Decoder `json:"-"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + OnError string `json:"onError"` + SessionID uint64 `json:"sessionID,string"` + UserID uint64 `json:"userID,string"` + NamespaceID uint64 `json:"namespaceID,string"` + ModuleID uint64 `json:"moduleID,string"` + Fields map[string]string `json:"fields"` + Progress RecordImportProgress `json:"progress"` + } + + RecordImportProgress struct { + StartedAt *time.Time `json:"startedAt"` + FinishedAt *time.Time `json:"finishedAt"` + EntryCount uint64 `json:"entryCount"` + Completed uint64 `json:"completed"` + Failed uint64 `json:"failed"` + FailReason string `json:"failReason,omitempty"` + } ) func Record() RecordService { @@ -184,6 +219,49 @@ func (svc record) Find(filter types.RecordFilter) (set types.RecordSet, f types. return } +func (svc record) Import(ses *RecordImportSession, ssvc ImportSessionService) error { + if ses.Decoder == nil { + return nil + } + + if ses.Progress.StartedAt != nil { + return errors.New("Unable to start import: Import session already active") + } + + sa := time.Now() + ses.Progress.StartedAt = &sa + ssvc.SetRecordByID(svc.ctx, ses.SessionID, 0, 0, nil, &ses.Progress, nil) + + return svc.db.Transaction(func() (err error) { + err = ses.Decoder.Records(ses.Fields, func(mod *types.Record) error { + mod.NamespaceID = ses.NamespaceID + mod.ModuleID = ses.ModuleID + mod.OwnedBy = ses.UserID + + _, err := svc.Create(mod) + if err != nil { + ses.Progress.Failed++ + ses.Progress.FailReason = err.Error() + + if ses.OnError == IMPORT_ON_ERROR_FAIL { + fa := time.Now() + ses.Progress.FinishedAt = &fa + ssvc.SetRecordByID(svc.ctx, ses.SessionID, 0, 0, nil, &ses.Progress, nil) + return err + } + } else { + ses.Progress.Completed++ + } + return nil + }) + + fa := time.Now() + ses.Progress.FinishedAt = &fa + ssvc.SetRecordByID(svc.ctx, ses.SessionID, 0, 0, nil, &ses.Progress, nil) + return + }) +} + // Export returns all records // // @todo better value handling diff --git a/compose/internal/service/service.go b/compose/internal/service/service.go index 02ef1e382..50f3932bc 100644 --- a/compose/internal/service/service.go +++ b/compose/internal/service/service.go @@ -47,14 +47,14 @@ var ( // DefaultAutomationRunner runs automation scripts by listening to triggerManager and invoking Corredor service DefaultAutomationRunner automationRunner - DefaultNamespace NamespaceService - DefaultRecord RecordService - DefaultModule ModuleService - DefaultChart ChartService - DefaultPage PageService - - DefaultAttachment AttachmentService - DefaultNotification NotificationService + DefaultNamespace NamespaceService + DefaultImportSession ImportSessionService + DefaultRecord RecordService + DefaultModule ModuleService + DefaultChart ChartService + DefaultPage PageService + DefaultAttachment AttachmentService + DefaultNotification NotificationService DefaultSystemUser *systemUser ) @@ -130,6 +130,7 @@ func Init(ctx context.Context, log *zap.Logger, c Config) (err error) { ) } + DefaultImportSession = ImportSession() DefaultRecord = Record() DefaultPage = Page() DefaultChart = Chart() diff --git a/compose/rest/handlers/record.go b/compose/rest/handlers/record.go index 46a2a6e13..495a337f0 100644 --- a/compose/rest/handlers/record.go +++ b/compose/rest/handlers/record.go @@ -31,6 +31,9 @@ import ( type RecordAPI interface { Report(context.Context, *request.RecordReport) (interface{}, error) List(context.Context, *request.RecordList) (interface{}, error) + ImportInit(context.Context, *request.RecordImportInit) (interface{}, error) + ImportRun(context.Context, *request.RecordImportRun) (interface{}, error) + ImportProgress(context.Context, *request.RecordImportProgress) (interface{}, error) Export(context.Context, *request.RecordExport) (interface{}, error) Create(context.Context, *request.RecordCreate) (interface{}, error) Read(context.Context, *request.RecordRead) (interface{}, error) @@ -41,14 +44,17 @@ type RecordAPI interface { // HTTP API interface type Record struct { - Report func(http.ResponseWriter, *http.Request) - List func(http.ResponseWriter, *http.Request) - Export func(http.ResponseWriter, *http.Request) - Create func(http.ResponseWriter, *http.Request) - Read func(http.ResponseWriter, *http.Request) - Update func(http.ResponseWriter, *http.Request) - Delete func(http.ResponseWriter, *http.Request) - Upload func(http.ResponseWriter, *http.Request) + Report func(http.ResponseWriter, *http.Request) + List func(http.ResponseWriter, *http.Request) + ImportInit func(http.ResponseWriter, *http.Request) + ImportRun func(http.ResponseWriter, *http.Request) + ImportProgress func(http.ResponseWriter, *http.Request) + Export func(http.ResponseWriter, *http.Request) + Create func(http.ResponseWriter, *http.Request) + Read func(http.ResponseWriter, *http.Request) + Update func(http.ResponseWriter, *http.Request) + Delete func(http.ResponseWriter, *http.Request) + Upload func(http.ResponseWriter, *http.Request) } func NewRecord(h RecordAPI) *Record { @@ -93,6 +99,66 @@ func NewRecord(h RecordAPI) *Record { resputil.JSON(w, value) } }, + ImportInit: func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + params := request.NewRecordImportInit() + if err := params.Fill(r); err != nil { + logger.LogParamError("Record.ImportInit", r, err) + resputil.JSON(w, err) + return + } + + value, err := h.ImportInit(r.Context(), params) + if err != nil { + logger.LogControllerError("Record.ImportInit", r, err, params.Auditable()) + resputil.JSON(w, err) + return + } + logger.LogControllerCall("Record.ImportInit", r, params.Auditable()) + if !serveHTTP(value, w, r) { + resputil.JSON(w, value) + } + }, + ImportRun: func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + params := request.NewRecordImportRun() + if err := params.Fill(r); err != nil { + logger.LogParamError("Record.ImportRun", r, err) + resputil.JSON(w, err) + return + } + + value, err := h.ImportRun(r.Context(), params) + if err != nil { + logger.LogControllerError("Record.ImportRun", r, err, params.Auditable()) + resputil.JSON(w, err) + return + } + logger.LogControllerCall("Record.ImportRun", r, params.Auditable()) + if !serveHTTP(value, w, r) { + resputil.JSON(w, value) + } + }, + ImportProgress: func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + params := request.NewRecordImportProgress() + if err := params.Fill(r); err != nil { + logger.LogParamError("Record.ImportProgress", r, err) + resputil.JSON(w, err) + return + } + + value, err := h.ImportProgress(r.Context(), params) + if err != nil { + logger.LogControllerError("Record.ImportProgress", r, err, params.Auditable()) + resputil.JSON(w, err) + return + } + logger.LogControllerCall("Record.ImportProgress", r, params.Auditable()) + if !serveHTTP(value, w, r) { + resputil.JSON(w, value) + } + }, Export: func(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() params := request.NewRecordExport() @@ -221,6 +287,9 @@ func (h Record) MountRoutes(r chi.Router, middlewares ...func(http.Handler) http r.Use(middlewares...) r.Get("/namespace/{namespaceID}/module/{moduleID}/record/report", h.Report) r.Get("/namespace/{namespaceID}/module/{moduleID}/record/", h.List) + r.Post("/namespace/{namespaceID}/module/{moduleID}/record/import", h.ImportInit) + r.Patch("/namespace/{namespaceID}/module/{moduleID}/record/import/{sessionID}", h.ImportRun) + r.Get("/namespace/{namespaceID}/module/{moduleID}/record/import/{sessionID}", h.ImportProgress) r.Get("/namespace/{namespaceID}/module/{moduleID}/record/export{filename}.{ext}", h.Export) r.Post("/namespace/{namespaceID}/module/{moduleID}/record/", h.Create) r.Get("/namespace/{namespaceID}/module/{moduleID}/record/{recordID}", h.Read) diff --git a/compose/rest/record.go b/compose/rest/record.go index df57fe924..b503f967d 100644 --- a/compose/rest/record.go +++ b/compose/rest/record.go @@ -12,11 +12,13 @@ import ( "github.com/pkg/errors" + "github.com/cortezaproject/corteza-server/compose/decoder" "github.com/cortezaproject/corteza-server/compose/encoder" "github.com/cortezaproject/corteza-server/compose/internal/repository" "github.com/cortezaproject/corteza-server/compose/internal/service" "github.com/cortezaproject/corteza-server/compose/rest/request" "github.com/cortezaproject/corteza-server/compose/types" + "github.com/cortezaproject/corteza-server/pkg/mime" "github.com/cortezaproject/corteza-server/pkg/rh" ) @@ -36,10 +38,11 @@ type ( } Record struct { - record service.RecordService - module service.ModuleService - attachment service.AttachmentService - ac recordAccessController + importSession service.ImportSessionService + record service.RecordService + module service.ModuleService + attachment service.AttachmentService + ac recordAccessController } recordAccessController interface { @@ -50,10 +53,11 @@ type ( func (Record) New() *Record { return &Record{ - record: service.DefaultRecord, - module: service.DefaultModule, - attachment: service.DefaultAttachment, - ac: service.DefaultAccessControl, + importSession: service.DefaultImportSession, + record: service.DefaultRecord, + module: service.DefaultModule, + attachment: service.DefaultAttachment, + ac: service.DefaultAccessControl, } } @@ -167,6 +171,114 @@ func (ctrl *Record) Upload(ctx context.Context, r *request.RecordUpload) (interf return makeAttachmentPayload(ctx, a, err) } +func (ctrl *Record) ImportInit(ctx context.Context, r *request.RecordImportInit) (interface{}, error) { + var ( + err error + recordDecoder service.Decoder + entryCount uint64 + ) + + // Access control. + if _, err = ctrl.module.With(ctx).FindByID(r.NamespaceID, r.ModuleID); err != nil { + return nil, err + } + + f, err := r.Upload.Open() + if err != nil { + return nil, err + } + defer f.Close() + + _, ext, err := mime.Type(f) + if err != nil { + return nil, err + } + + if ext == "txt" { + if is, err := mime.JsonL(f); err != nil { + return nil, err + } else if is { + ext = "jsonl" + } + } + + // determine decoder + switch strings.ToLower(ext) { + case "json", "jsonl", "ldjson", "ndjson": + recordDecoder = decoder.NewStructuredDecoder(json.NewDecoder(f), f) + + case "csv": + recordDecoder = decoder.NewFlatReader(csv.NewReader(f), f) + + default: + return nil, errors.New(fmt.Sprintf("unsupported format (\"%s\")", ext)) + + } + entryCount, err = recordDecoder.EntryCount() + if err != nil { + return nil, err + } + + header := recordDecoder.Header() + hh := make(map[string]string) + for _, h := range header { + hh[h] = "" + } + + return ctrl.importSession.SetRecordByID( + ctx, + 0, + r.NamespaceID, + r.ModuleID, + hh, + &service.RecordImportProgress{EntryCount: entryCount}, + recordDecoder) +} + +func (ctrl *Record) ImportRun(ctx context.Context, r *request.RecordImportRun) (interface{}, error) { + var ( + err error + ) + + // Access control. + if _, err = ctrl.module.With(ctx).FindByID(r.NamespaceID, r.ModuleID); err != nil { + return nil, err + } + + // Check if session ok + ses, err := ctrl.importSession.FindRecordByID(ctx, r.SessionID) + if err != nil { + return nil, err + } + + if ses.Progress.StartedAt != nil { + return nil, errors.New("Unable to start import: Import session already active") + } + + ses.Fields = make(map[string]string) + err = json.Unmarshal(r.Fields, &ses.Fields) + if err != nil { + return nil, err + } + + ses.OnError = r.OnError + + // @todo routine + ctrl.record.With(ctx).Import(ses, ctrl.importSession) + + return ses, nil +} + +func (ctrl *Record) ImportProgress(ctx context.Context, r *request.RecordImportProgress) (interface{}, error) { + // Get session + ses, err := ctrl.importSession.FindRecordByID(ctx, r.SessionID) + if err != nil { + return nil, err + } + + return ses, nil +} + func (ctrl *Record) Export(ctx context.Context, r *request.RecordExport) (interface{}, error) { type ( // ad-hoc interface for our encoder diff --git a/compose/rest/request/record.go b/compose/rest/request/record.go index 8cc5b9d77..bc4bf8a5a 100644 --- a/compose/rest/request/record.go +++ b/compose/rest/request/record.go @@ -175,6 +175,192 @@ func (r *RecordList) Fill(req *http.Request) (err error) { var _ RequestFiller = NewRecordList() +// Record importInit request parameters +type RecordImportInit struct { + Upload *multipart.FileHeader + NamespaceID uint64 `json:",string"` + ModuleID uint64 `json:",string"` +} + +func NewRecordImportInit() *RecordImportInit { + return &RecordImportInit{} +} + +func (r RecordImportInit) Auditable() map[string]interface{} { + var out = map[string]interface{}{} + + out["upload.size"] = r.Upload.Size + out["upload.filename"] = r.Upload.Filename + + out["namespaceID"] = r.NamespaceID + out["moduleID"] = r.ModuleID + + return out +} + +func (r *RecordImportInit) Fill(req *http.Request) (err error) { + if strings.ToLower(req.Header.Get("content-type")) == "application/json" { + err = json.NewDecoder(req.Body).Decode(r) + + switch { + case err == io.EOF: + err = nil + case err != nil: + return errors.Wrap(err, "error parsing http request body") + } + } + + if err = req.ParseMultipartForm(32 << 20); err != nil { + return err + } + + get := map[string]string{} + post := map[string]string{} + urlQuery := req.URL.Query() + for name, param := range urlQuery { + get[name] = string(param[0]) + } + postVars := req.Form + for name, param := range postVars { + post[name] = string(param[0]) + } + + if _, r.Upload, err = req.FormFile("upload"); err != nil { + return errors.Wrap(err, "error procesing uploaded file") + } + + r.NamespaceID = parseUInt64(chi.URLParam(req, "namespaceID")) + r.ModuleID = parseUInt64(chi.URLParam(req, "moduleID")) + + return err +} + +var _ RequestFiller = NewRecordImportInit() + +// Record importRun request parameters +type RecordImportRun struct { + SessionID uint64 `json:",string"` + NamespaceID uint64 `json:",string"` + ModuleID uint64 `json:",string"` + Fields json.RawMessage + OnError string +} + +func NewRecordImportRun() *RecordImportRun { + return &RecordImportRun{} +} + +func (r RecordImportRun) Auditable() map[string]interface{} { + var out = map[string]interface{}{} + + out["sessionID"] = r.SessionID + out["namespaceID"] = r.NamespaceID + out["moduleID"] = r.ModuleID + out["fields"] = r.Fields + out["onError"] = r.OnError + + return out +} + +func (r *RecordImportRun) Fill(req *http.Request) (err error) { + if strings.ToLower(req.Header.Get("content-type")) == "application/json" { + err = json.NewDecoder(req.Body).Decode(r) + + switch { + case err == io.EOF: + err = nil + case err != nil: + return errors.Wrap(err, "error parsing http request body") + } + } + + if err = req.ParseForm(); err != nil { + return err + } + + get := map[string]string{} + post := map[string]string{} + urlQuery := req.URL.Query() + for name, param := range urlQuery { + get[name] = string(param[0]) + } + postVars := req.Form + for name, param := range postVars { + post[name] = string(param[0]) + } + + r.SessionID = parseUInt64(chi.URLParam(req, "sessionID")) + r.NamespaceID = parseUInt64(chi.URLParam(req, "namespaceID")) + r.ModuleID = parseUInt64(chi.URLParam(req, "moduleID")) + if val, ok := post["fields"]; ok { + r.Fields = json.RawMessage(val) + } + if val, ok := post["onError"]; ok { + r.OnError = val + } + + return err +} + +var _ RequestFiller = NewRecordImportRun() + +// Record importProgress request parameters +type RecordImportProgress struct { + SessionID uint64 `json:",string"` + NamespaceID uint64 `json:",string"` + ModuleID uint64 `json:",string"` +} + +func NewRecordImportProgress() *RecordImportProgress { + return &RecordImportProgress{} +} + +func (r RecordImportProgress) Auditable() map[string]interface{} { + var out = map[string]interface{}{} + + out["sessionID"] = r.SessionID + out["namespaceID"] = r.NamespaceID + out["moduleID"] = r.ModuleID + + return out +} + +func (r *RecordImportProgress) Fill(req *http.Request) (err error) { + if strings.ToLower(req.Header.Get("content-type")) == "application/json" { + err = json.NewDecoder(req.Body).Decode(r) + + switch { + case err == io.EOF: + err = nil + case err != nil: + return errors.Wrap(err, "error parsing http request body") + } + } + + if err = req.ParseForm(); err != nil { + return err + } + + get := map[string]string{} + post := map[string]string{} + urlQuery := req.URL.Query() + for name, param := range urlQuery { + get[name] = string(param[0]) + } + postVars := req.Form + for name, param := range postVars { + post[name] = string(param[0]) + } + + r.SessionID = parseUInt64(chi.URLParam(req, "sessionID")) + r.NamespaceID = parseUInt64(chi.URLParam(req, "namespaceID")) + r.ModuleID = parseUInt64(chi.URLParam(req, "moduleID")) + + return err +} + +var _ RequestFiller = NewRecordImportProgress() + // Record export request parameters type RecordExport struct { Filter string diff --git a/docs/compose/README.md b/docs/compose/README.md index 549b87d90..2400c9e89 100644 --- a/docs/compose/README.md +++ b/docs/compose/README.md @@ -956,6 +956,9 @@ Compose records | ------ | -------- | ------- | | `GET` | `/namespace/{namespaceID}/module/{moduleID}/record/report` | Generates report from module records | | `GET` | `/namespace/{namespaceID}/module/{moduleID}/record/` | List/read records from module section | +| `POST` | `/namespace/{namespaceID}/module/{moduleID}/record/import` | Initiate record import session | +| `PATCH` | `/namespace/{namespaceID}/module/{moduleID}/record/import/{sessionID}` | Run record import | +| `GET` | `/namespace/{namespaceID}/module/{moduleID}/record/import/{sessionID}` | Get import progress | | `GET` | `/namespace/{namespaceID}/module/{moduleID}/record/export{filename}.{ext}` | Exports records that match | | `POST` | `/namespace/{namespaceID}/module/{moduleID}/record/` | Create record in module section | | `GET` | `/namespace/{namespaceID}/module/{moduleID}/record/{recordID}` | Read records by ID from module section | @@ -1000,6 +1003,56 @@ Compose records | namespaceID | uint64 | PATH | Namespace ID | N/A | YES | | moduleID | uint64 | PATH | Module ID | N/A | YES | +## Initiate record import session + +#### Method + +| URI | Protocol | Method | Authentication | +| --- | -------- | ------ | -------------- | +| `/namespace/{namespaceID}/module/{moduleID}/record/import` | HTTP/S | POST | | + +#### Request parameters + +| Parameter | Type | Method | Description | Default | Required? | +| --------- | ---- | ------ | ----------- | ------- | --------- | +| upload | *multipart.FileHeader | POST | File import | N/A | YES | +| namespaceID | uint64 | PATH | Namespace ID | N/A | YES | +| moduleID | uint64 | PATH | Module ID | N/A | YES | + +## Run record import + +#### Method + +| URI | Protocol | Method | Authentication | +| --- | -------- | ------ | -------------- | +| `/namespace/{namespaceID}/module/{moduleID}/record/import/{sessionID}` | HTTP/S | PATCH | | + +#### Request parameters + +| Parameter | Type | Method | Description | Default | Required? | +| --------- | ---- | ------ | ----------- | ------- | --------- | +| sessionID | uint64 | PATH | Import session | N/A | YES | +| namespaceID | uint64 | PATH | Namespace ID | N/A | YES | +| moduleID | uint64 | PATH | Module ID | N/A | YES | +| fields | json.RawMessage | POST | Fields defined by import file | N/A | YES | +| onError | string | POST | What happens if record fails to import | N/A | YES | + +## Get import progress + +#### Method + +| URI | Protocol | Method | Authentication | +| --- | -------- | ------ | -------------- | +| `/namespace/{namespaceID}/module/{moduleID}/record/import/{sessionID}` | HTTP/S | GET | | + +#### Request parameters + +| Parameter | Type | Method | Description | Default | Required? | +| --------- | ---- | ------ | ----------- | ------- | --------- | +| sessionID | uint64 | PATH | Import session | N/A | YES | +| namespaceID | uint64 | PATH | Namespace ID | N/A | YES | +| moduleID | uint64 | PATH | Module ID | N/A | YES | + ## Exports records that match #### Method diff --git a/pkg/count/file.go b/pkg/count/file.go new file mode 100644 index 000000000..6c1a88597 --- /dev/null +++ b/pkg/count/file.go @@ -0,0 +1,28 @@ +package count + +import ( + "bytes" + "io" +) + +// Lines provides a line count +// +// https://stackoverflow.com/a/24563853 +func Lines(r io.ReadSeeker) (count uint64, err error) { + defer r.Seek(0, 0) + buf := make([]byte, 32*1024) + lineSep := []byte{'\n'} + + for { + c, err := r.Read(buf) + count += uint64(bytes.Count(buf[:c], lineSep)) + + switch { + case err == io.EOF: + return count, nil + + case err != nil: + return count, err + } + } +} diff --git a/pkg/mime/mime.go b/pkg/mime/mime.go new file mode 100644 index 000000000..063c5adb3 --- /dev/null +++ b/pkg/mime/mime.go @@ -0,0 +1,34 @@ +package mime + +import ( + "bufio" + "io" + + "github.com/gabriel-vasile/mimetype" +) + +func Type(file io.ReadSeeker) (mt string, ext string, err error) { + if _, err = file.Seek(0, 0); err != nil { + return + } + + // Make sure we rewind when we're done + defer file.Seek(0, 0) + return mimetype.DetectReader(file) +} + +func JsonL(file io.ReadSeeker) (bool, error) { + // ExtractMimetype fails to detect json if jsonl is used + // For now check if first rune is { + r := bufio.NewReader(file) + rn, _, err := r.ReadRune() + defer file.Seek(0, 0) + if err != nil { + return false, err + } + + if string(rn) == "{" { + return true, nil + } + return false, nil +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/.gitattributes b/vendor/github.com/gabriel-vasile/mimetype/.gitattributes new file mode 100644 index 000000000..0cc26ec01 --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/.gitattributes @@ -0,0 +1 @@ +testdata/* linguist-vendored diff --git a/vendor/github.com/gabriel-vasile/mimetype/.travis.yml b/vendor/github.com/gabriel-vasile/mimetype/.travis.yml new file mode 100644 index 000000000..9f453191c --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/.travis.yml @@ -0,0 +1,14 @@ +language: go +go: + - "1.9" + - "1.10" +before_install: + - go get github.com/mattn/goveralls + - go get github.com/client9/misspell/cmd/misspell +before_script: + - go tool vet . +script: + - diff -u <(echo -n) <(gofmt -d ./) + - go test -v + - $GOPATH/bin/goveralls -service=travis-ci + - misspell -locale US -error *.md *.go diff --git a/vendor/github.com/gabriel-vasile/mimetype/CODE_OF_CONDUCT.md b/vendor/github.com/gabriel-vasile/mimetype/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..8479cd87d --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at vasile.gabriel@email.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/vendor/github.com/gabriel-vasile/mimetype/CONTRIBUTING.md b/vendor/github.com/gabriel-vasile/mimetype/CONTRIBUTING.md new file mode 100644 index 000000000..56ae4e57c --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/CONTRIBUTING.md @@ -0,0 +1,12 @@ +## Contribute +Contributions to **mimetype** are welcome. If you find an issue and you consider +contributing, you can use the [Github issues tracker](https://github.com/gabriel-vasile/mimetype/issues) +in order to report it, or better yet, open a pull request. + +Code contributions must respect these rules: + - code must be test covered + - code must be formatted using gofmt tool + - exported names must be documented + +**Important**: By submitting a pull request, you agree to allow the project +owner to license your work under the same license as that used by the project. diff --git a/vendor/github.com/gabriel-vasile/mimetype/LICENSE b/vendor/github.com/gabriel-vasile/mimetype/LICENSE new file mode 100644 index 000000000..f1b456e91 --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018, 2019 Gabriel Vasile + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/gabriel-vasile/mimetype/README.md b/vendor/github.com/gabriel-vasile/mimetype/README.md new file mode 100644 index 000000000..3d7f4afc5 --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/README.md @@ -0,0 +1,64 @@ +

+ mimetype +

+ +

+ A package for detecting MIME types and extensions based on magic numbers +

+
+ No bindings, all written in pure go +
+ +

+ + Build Status + + + Documentation + + + Go report card + + + Go report card + + + License + +

+ +## Install +```bash +go get github.com/gabriel-vasile/mimetype +``` + +## Use +The library exposes three functions you can use in order to detect a file type. +See [Godoc](https://godoc.org/github.com/gabriel-vasile/mimetype) for full reference. +```go +func Detect(in []byte) (mime, extension string) {...} +func DetectReader(r io.Reader) (mime, extension string, err error) {...} +func DetectFile(file string) (mime, extension string, err error) {...} +``` +When detecting from a `ReadSeeker` interface, such as `os.File`, make sure +to reset the offset of the reader to the beginning if needed: +```go +_, err = file.Seek(0, io.SeekStart) +``` + +## Supported MIME types +See [supported mimes](supported_mimes.md) for the list of detected MIME types. +If support is needed for a specific file format, please open an [issue](https://github.com/gabriel-vasile/mimetype/issues/new/choose). + +## Structure +**mimetype** uses an hierarchical structure to keep the matching functions. +This reduces the number of calls needed for detecting the file type. The reason +behind this choice is that there are file formats used as containers for other +file formats. For example, Microsoft office files are just zip archives, +containing specific metadata files. +
+ structure +
+ +## Contributing +See [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/vendor/github.com/gabriel-vasile/mimetype/go.mod b/vendor/github.com/gabriel-vasile/mimetype/go.mod new file mode 100644 index 000000000..6f8542d53 --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/go.mod @@ -0,0 +1 @@ +module github.com/gabriel-vasile/mimetype diff --git a/vendor/github.com/gabriel-vasile/mimetype/internal/json/json.go b/vendor/github.com/gabriel-vasile/mimetype/internal/json/json.go new file mode 100644 index 000000000..9aef6cb4c --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/internal/json/json.go @@ -0,0 +1,536 @@ +// Copyright (c) 2009 The Go Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// JSON value parser state machine. +// This package is almost entirely copied from the Go stdlib. +// Changes made to it permit users of the package to tell +// if some slice of bytes is a valid beginning of a json string. +package json + +import "fmt" + +type ( + context int + scanStatus int +) + +const ( + contextKey context = iota + contextObj + contextArr + + scanContinue scanStatus = iota // uninteresting byte + scanBeginLiteral // end implied by next result != scanContinue + scanBeginObject // begin object + scanObjectKey // just finished object key (string) + scanObjectValue // just finished non-last object value + scanEndObject // end object (implies scanObjectValue if possible) + scanBeginArray // begin array + scanArrayValue // just finished array value + scanEndArray // end array (implies scanArrayValue if possible) + scanSkipSpace // space byte; can skip; known to be last "continue" result + scanEnd // top-level value ended *before* this byte; known to be first "stop" result + scanError // hit an error, scanner.err. +) + +type ( + scanner struct { + step func(*scanner, byte) scanStatus + contexts []context + endTop bool + err error + index int + } +) + +// Scan returns the number of bytes scanned and if there was any error +// in trying to reach the end of data +func Scan(data []byte) (int, error) { + s := &scanner{} + _ = checkValid(data, s) + return s.index, s.err +} + +// checkValid verifies that data is valid JSON-encoded data. +// scan is passed in for use by checkValid to avoid an allocation. +func checkValid(data []byte, scan *scanner) error { + scan.reset() + for _, c := range data { + scan.index++ + if scan.step(scan, c) == scanError { + return scan.err + } + } + if scan.eof() == scanError { + return scan.err + } + return nil +} + +func isSpace(c byte) bool { + return c == ' ' || c == '\t' || c == '\r' || c == '\n' +} + +func (s *scanner) reset() { + s.step = stateBeginValue + s.contexts = s.contexts[0:0] + s.err = nil +} + +// eof tells the scanner that the end of input has been reached. +// It returns a scan status just as s.step does. +func (s *scanner) eof() scanStatus { + if s.err != nil { + return scanError + } + if s.endTop { + return scanEnd + } + s.step(s, ' ') + if s.endTop { + return scanEnd + } + if s.err == nil { + s.err = fmt.Errorf("unexpected end of JSON input") + } + return scanError +} + +// pushContext pushes a new parse state p onto the parse stack. +func (s *scanner) pushParseState(p context) { + s.contexts = append(s.contexts, p) +} + +// popParseState pops a parse state (already obtained) off the stack +// and updates s.step accordingly. +func (s *scanner) popParseState() { + n := len(s.contexts) - 1 + s.contexts = s.contexts[0:n] + if n == 0 { + s.step = stateEndTop + s.endTop = true + } else { + s.step = stateEndValue + } +} + +// stateBeginValueOrEmpty is the state after reading `[`. +func stateBeginValueOrEmpty(s *scanner, c byte) scanStatus { + if c <= ' ' && isSpace(c) { + return scanSkipSpace + } + if c == ']' { + return stateEndValue(s, c) + } + return stateBeginValue(s, c) +} + +// stateBeginValue is the state at the beginning of the input. +func stateBeginValue(s *scanner, c byte) scanStatus { + if c <= ' ' && isSpace(c) { + return scanSkipSpace + } + switch c { + case '{': + s.step = stateBeginStringOrEmpty + s.pushParseState(contextKey) + return scanBeginObject + case '[': + s.step = stateBeginValueOrEmpty + s.pushParseState(contextArr) + return scanBeginArray + case '"': + s.step = stateInString + return scanBeginLiteral + case '-': + s.step = stateNeg + return scanBeginLiteral + case '0': // beginning of 0.123 + s.step = state0 + return scanBeginLiteral + case 't': // beginning of true + s.step = stateT + return scanBeginLiteral + case 'f': // beginning of false + s.step = stateF + return scanBeginLiteral + case 'n': // beginning of null + s.step = stateN + return scanBeginLiteral + } + if '1' <= c && c <= '9' { // beginning of 1234.5 + s.step = state1 + return scanBeginLiteral + } + return s.error(c, "looking for beginning of value") +} + +// stateBeginStringOrEmpty is the state after reading `{`. +func stateBeginStringOrEmpty(s *scanner, c byte) scanStatus { + if c <= ' ' && isSpace(c) { + return scanSkipSpace + } + if c == '}' { + n := len(s.contexts) + s.contexts[n-1] = contextObj + return stateEndValue(s, c) + } + return stateBeginString(s, c) +} + +// stateBeginString is the state after reading `{"key": value,`. +func stateBeginString(s *scanner, c byte) scanStatus { + if c <= ' ' && isSpace(c) { + return scanSkipSpace + } + if c == '"' { + s.step = stateInString + return scanBeginLiteral + } + return s.error(c, "looking for beginning of object key string") +} + +// stateEndValue is the state after completing a value, +// such as after reading `{}` or `true` or `["x"`. +func stateEndValue(s *scanner, c byte) scanStatus { + n := len(s.contexts) + if n == 0 { + // Completed top-level before the current byte. + s.step = stateEndTop + s.endTop = true + return stateEndTop(s, c) + } + if c <= ' ' && isSpace(c) { + s.step = stateEndValue + return scanSkipSpace + } + ps := s.contexts[n-1] + switch ps { + case contextKey: + if c == ':' { + s.contexts[n-1] = contextObj + s.step = stateBeginValue + return scanObjectKey + } + return s.error(c, "after object key") + case contextObj: + if c == ',' { + s.contexts[n-1] = contextKey + s.step = stateBeginString + return scanObjectValue + } + if c == '}' { + s.popParseState() + return scanEndObject + } + return s.error(c, "after object key:value pair") + case contextArr: + if c == ',' { + s.step = stateBeginValue + return scanArrayValue + } + if c == ']' { + s.popParseState() + return scanEndArray + } + return s.error(c, "after array element") + } + return s.error(c, "") +} + +// stateEndTop is the state after finishing the top-level value, +// such as after reading `{}` or `[1,2,3]`. +// Only space characters should be seen now. +func stateEndTop(s *scanner, c byte) scanStatus { + if c != ' ' && c != '\t' && c != '\r' && c != '\n' { + // Complain about non-space byte on next call. + s.error(c, "after top-level value") + } + return scanEnd +} + +// stateInString is the state after reading `"`. +func stateInString(s *scanner, c byte) scanStatus { + if c == '"' { + s.step = stateEndValue + return scanContinue + } + if c == '\\' { + s.step = stateInStringEsc + return scanContinue + } + if c < 0x20 { + return s.error(c, "in string literal") + } + return scanContinue +} + +// stateInStringEsc is the state after reading `"\` during a quoted string. +func stateInStringEsc(s *scanner, c byte) scanStatus { + switch c { + case 'b', 'f', 'n', 'r', 't', '\\', '/', '"': + s.step = stateInString + return scanContinue + case 'u': + s.step = stateInStringEscU + return scanContinue + } + return s.error(c, "in string escape code") +} + +// stateInStringEscU is the state after reading `"\u` during a quoted string. +func stateInStringEscU(s *scanner, c byte) scanStatus { + if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { + s.step = stateInStringEscU1 + return scanContinue + } + // numbers + return s.error(c, "in \\u hexadecimal character escape") +} + +// stateInStringEscU1 is the state after reading `"\u1` during a quoted string. +func stateInStringEscU1(s *scanner, c byte) scanStatus { + if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { + s.step = stateInStringEscU12 + return scanContinue + } + // numbers + return s.error(c, "in \\u hexadecimal character escape") +} + +// stateInStringEscU12 is the state after reading `"\u12` during a quoted string. +func stateInStringEscU12(s *scanner, c byte) scanStatus { + if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { + s.step = stateInStringEscU123 + return scanContinue + } + // numbers + return s.error(c, "in \\u hexadecimal character escape") +} + +// stateInStringEscU123 is the state after reading `"\u123` during a quoted string. +func stateInStringEscU123(s *scanner, c byte) scanStatus { + if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { + s.step = stateInString + return scanContinue + } + // numbers + return s.error(c, "in \\u hexadecimal character escape") +} + +// stateNeg is the state after reading `-` during a number. +func stateNeg(s *scanner, c byte) scanStatus { + if c == '0' { + s.step = state0 + return scanContinue + } + if '1' <= c && c <= '9' { + s.step = state1 + return scanContinue + } + return s.error(c, "in numeric literal") +} + +// state1 is the state after reading a non-zero integer during a number, +// such as after reading `1` or `100` but not `0`. +func state1(s *scanner, c byte) scanStatus { + if '0' <= c && c <= '9' { + s.step = state1 + return scanContinue + } + return state0(s, c) +} + +// state0 is the state after reading `0` during a number. +func state0(s *scanner, c byte) scanStatus { + if c == '.' { + s.step = stateDot + return scanContinue + } + if c == 'e' || c == 'E' { + s.step = stateE + return scanContinue + } + return stateEndValue(s, c) +} + +// stateDot is the state after reading the integer and decimal point in a number, +// such as after reading `1.`. +func stateDot(s *scanner, c byte) scanStatus { + if '0' <= c && c <= '9' { + s.step = stateDot0 + return scanContinue + } + return s.error(c, "after decimal point in numeric literal") +} + +// stateDot0 is the state after reading the integer, decimal point, and subsequent +// digits of a number, such as after reading `3.14`. +func stateDot0(s *scanner, c byte) scanStatus { + if '0' <= c && c <= '9' { + return scanContinue + } + if c == 'e' || c == 'E' { + s.step = stateE + return scanContinue + } + return stateEndValue(s, c) +} + +// stateE is the state after reading the mantissa and e in a number, +// such as after reading `314e` or `0.314e`. +func stateE(s *scanner, c byte) scanStatus { + if c == '+' || c == '-' { + s.step = stateESign + return scanContinue + } + return stateESign(s, c) +} + +// stateESign is the state after reading the mantissa, e, and sign in a number, +// such as after reading `314e-` or `0.314e+`. +func stateESign(s *scanner, c byte) scanStatus { + if '0' <= c && c <= '9' { + s.step = stateE0 + return scanContinue + } + return s.error(c, "in exponent of numeric literal") +} + +// stateE0 is the state after reading the mantissa, e, optional sign, +// and at least one digit of the exponent in a number, +// such as after reading `314e-2` or `0.314e+1` or `3.14e0`. +func stateE0(s *scanner, c byte) scanStatus { + if '0' <= c && c <= '9' { + return scanContinue + } + return stateEndValue(s, c) +} + +// stateT is the state after reading `t`. +func stateT(s *scanner, c byte) scanStatus { + if c == 'r' { + s.step = stateTr + return scanContinue + } + return s.error(c, "in literal true (expecting 'r')") +} + +// stateTr is the state after reading `tr`. +func stateTr(s *scanner, c byte) scanStatus { + if c == 'u' { + s.step = stateTru + return scanContinue + } + return s.error(c, "in literal true (expecting 'u')") +} + +// stateTru is the state after reading `tru`. +func stateTru(s *scanner, c byte) scanStatus { + if c == 'e' { + s.step = stateEndValue + return scanContinue + } + return s.error(c, "in literal true (expecting 'e')") +} + +// stateF is the state after reading `f`. +func stateF(s *scanner, c byte) scanStatus { + if c == 'a' { + s.step = stateFa + return scanContinue + } + return s.error(c, "in literal false (expecting 'a')") +} + +// stateFa is the state after reading `fa`. +func stateFa(s *scanner, c byte) scanStatus { + if c == 'l' { + s.step = stateFal + return scanContinue + } + return s.error(c, "in literal false (expecting 'l')") +} + +// stateFal is the state after reading `fal`. +func stateFal(s *scanner, c byte) scanStatus { + if c == 's' { + s.step = stateFals + return scanContinue + } + return s.error(c, "in literal false (expecting 's')") +} + +// stateFals is the state after reading `fals`. +func stateFals(s *scanner, c byte) scanStatus { + if c == 'e' { + s.step = stateEndValue + return scanContinue + } + return s.error(c, "in literal false (expecting 'e')") +} + +// stateN is the state after reading `n`. +func stateN(s *scanner, c byte) scanStatus { + if c == 'u' { + s.step = stateNu + return scanContinue + } + return s.error(c, "in literal null (expecting 'u')") +} + +// stateNu is the state after reading `nu`. +func stateNu(s *scanner, c byte) scanStatus { + if c == 'l' { + s.step = stateNul + return scanContinue + } + return s.error(c, "in literal null (expecting 'l')") +} + +// stateNul is the state after reading `nul`. +func stateNul(s *scanner, c byte) scanStatus { + if c == 'l' { + s.step = stateEndValue + return scanContinue + } + return s.error(c, "in literal null (expecting 'l')") +} + +// stateError is the state after reaching a syntax error, +// such as after reading `[1}` or `5.1.2`. +func stateError(s *scanner, c byte) scanStatus { + return scanError +} + +// error records an error and switches to the error state. +func (s *scanner) error(c byte, context string) scanStatus { + s.step = stateError + s.err = fmt.Errorf("invalid character <<%c>> %s", c, context) + return scanError +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/archive.go b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/archive.go new file mode 100644 index 000000000..db3b2150f --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/archive.go @@ -0,0 +1,78 @@ +package matchers + +import "bytes" + +// Zip matches a zip archive. +func Zip(in []byte) bool { + return len(in) > 3 && + in[0] == 0x50 && in[1] == 0x4B && + (in[2] == 0x3 || in[2] == 0x5 || in[2] == 0x7) && + (in[3] == 0x4 || in[3] == 0x6 || in[3] == 0x8) +} + +// SevenZ matches a 7z archive. +func SevenZ(in []byte) bool { + return len(in) > 6 && + bytes.Equal(in[:6], []byte{0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C}) +} + +// Epub matches an EPUB file. +func Epub(in []byte) bool { + return len(in) > 58 && bytes.Equal(in[30:58], []byte("mimetypeapplication/epub+zip")) +} + +// Jar matches a Java archive file. +func Jar(in []byte) bool { + return bytes.Contains(in, []byte("META-INF/MANIFEST.MF")) +} + +// Gzip matched gzip files based on http://www.zlib.org/rfc-gzip.html#header-trailer. +func Gzip(in []byte) bool { + return len(in) > 2 && bytes.Equal(in[:2], []byte{0x1f, 0x8b}) +} + +// Crx matches a Chrome extension file: a zip archive prepended by "Cr24". +func Crx(in []byte) bool { + return bytes.HasPrefix(in, []byte("Cr24")) +} + +// Tar matches a (t)ape (ar)chive file. +func Tar(in []byte) bool { + return len(in) > 262 && bytes.Equal(in[257:262], []byte("ustar")) +} + +// Fits matches an Flexible Image Transport System file. +func Fits(in []byte) bool { + return bytes.HasPrefix(in, []byte{0x53, 0x49, 0x4D, 0x50, 0x4C, 0x45, 0x20, + 0x20, 0x3D, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x54}) +} + +// Xar matches an eXtensible ARchive format file. +func Xar(in []byte) bool { + return bytes.HasPrefix(in, []byte{0x78, 0x61, 0x72, 0x21}) +} + +// Bz2 matches a bzip2 file. +func Bz2(in []byte) bool { + return bytes.HasPrefix(in, []byte{0x42, 0x5A, 0x68}) +} + +// Ar matches an ar (Unix) archive file. +func Ar(in []byte) bool { + return bytes.HasPrefix(in, []byte{0x21, 0x3C, 0x61, 0x72, 0x63, 0x68, 0x3E}) +} + +// Deb matches a Debian package file +func Deb(in []byte) bool { + return len(in) > 8 && bytes.HasPrefix(in[8:], []byte{0x64, 0x65, 0x62, 0x69, + 0x61, 0x6E, 0x2D, 0x62, 0x69, 0x6E, 0x61, 0x72, 0x79}) +} + +// Rar matches a RAR archive file +func Rar(in []byte) bool { + if !bytes.HasPrefix(in, []byte{0x52, 0x61, 0x72, 0x21, 0x1A, 0x07}) { + return false + } + return len(in) > 8 && (bytes.Equal(in[6:8], []byte{0x01, 0x00}) || in[6] == 0x00) +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/audio.go b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/audio.go new file mode 100644 index 000000000..3b67a39e2 --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/audio.go @@ -0,0 +1,69 @@ +package matchers + +import ( + "bytes" +) + +// Mp3 matches an mp3 file. +func Mp3(in []byte) bool { + return bytes.HasPrefix(in, []byte("\x49\x44\x33")) +} + +// Flac matches a Free Lossless Audio Codec file. +func Flac(in []byte) bool { + return bytes.HasPrefix(in, []byte("\x66\x4C\x61\x43\x00\x00\x00\x22")) +} + +// Midi matches a Musical Instrument Digital Interface file. +func Midi(in []byte) bool { + return bytes.HasPrefix(in, []byte("\x4D\x54\x68\x64")) +} + +// Ape matches a Monkey's Audio file. +func Ape(in []byte) bool { + return bytes.HasPrefix(in, []byte("\x4D\x41\x43\x20\x96\x0F\x00\x00\x34\x00\x00\x00\x18\x00\x00\x00\x90\xE3")) +} + +// MusePack matches a Musepack file. +func MusePack(in []byte) bool { + return bytes.HasPrefix(in, []byte("MPCK")) +} + +// Wav matches a Waveform Audio File Format file. +func Wav(in []byte) bool { + return len(in) > 12 && + bytes.Equal(in[:4], []byte("\x52\x49\x46\x46")) && + bytes.Equal(in[8:12], []byte("\x57\x41\x56\x45")) +} + +// Aiff matches Audio Interchange File Format file. +func Aiff(in []byte) bool { + return len(in) > 12 && + bytes.Equal(in[:4], []byte("\x46\x4F\x52\x4D")) && + bytes.Equal(in[8:12], []byte("\x41\x49\x46\x46")) +} + +// Ogg matches an Ogg file. +func Ogg(in []byte) bool { + return len(in) > 5 && bytes.Equal(in[:5], []byte("\x4F\x67\x67\x53\x00")) +} + +// Au matches a Sun Microsystems au file. +func Au(in []byte) bool { + return len(in) > 4 && bytes.Equal(in[:4], []byte("\x2E\x73\x6E\x64")) +} + +// Amr matches an Adaptive Multi-Rate file. +func Amr(in []byte) bool { + return len(in) > 5 && bytes.Equal(in[:5], []byte("\x23\x21\x41\x4D\x52")) +} + +// Aac matches an Advanced Audio Coding file. +func Aac(in []byte) bool { + return bytes.HasPrefix(in, []byte{0xFF, 0xF1}) || bytes.HasPrefix(in, []byte{0xFF, 0xF9}) +} + +// Voc matches a Creative Voice file. +func Voc(in []byte) bool { + return bytes.HasPrefix(in, []byte("Creative Voice File")) +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/binary.go b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/binary.go new file mode 100644 index 000000000..1f58346eb --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/binary.go @@ -0,0 +1,88 @@ +package matchers + +import ( + "bytes" +) + +// Class matches an java class file. +func Class(in []byte) bool { + return bytes.HasPrefix(in, []byte{0xCA, 0xFE, 0xBA, 0xBE}) +} + +// Swf matches an Adobe Flash swf file. +func Swf(in []byte) bool { + return bytes.HasPrefix(in, []byte("CWS")) || + bytes.HasPrefix(in, []byte("FWS")) || + bytes.HasPrefix(in, []byte("ZWS")) +} + +// Wasm matches a web assembly File Format file. +func Wasm(in []byte) bool { + return bytes.HasPrefix(in, []byte{0x00, 0x61, 0x73, 0x6D}) +} + +// Dbf matches a dBase file. +// https://www.dbase.com/Knowledgebase/INT/db7_file_fmt.htm +func Dbf(in []byte) bool { + if len(in) < 4 { + return false + } + + // 3rd and 4th bytes contain the last update month and day of month + if !(0 < in[2] && in[2] < 13 && 0 < in[3] && in[3] < 32) { + return false + } + + // dbf type is dictated by the first byte + dbfTypes := []byte{ + 0x02, 0x03, 0x04, 0x05, 0x30, 0x31, 0x32, 0x42, 0x62, 0x7B, 0x82, + 0x83, 0x87, 0x8A, 0x8B, 0x8E, 0xB3, 0xCB, 0xE5, 0xF5, 0xF4, 0xFB, + } + for _, b := range dbfTypes { + if in[0] == b { + return true + } + } + + return false +} + +// Exe matches a Windows/DOS executable file. +func Exe(in []byte) bool { + return bytes.HasPrefix(in, []byte{0x4D, 0x5A}) +} + +// Elf matches an Executable and Linkable Format file. +func Elf(in []byte) bool { + return bytes.HasPrefix(in, []byte{0x7F, 0x45, 0x4C, 0x46}) +} + +// ElfObj matches an object file. +func ElfObj(in []byte) bool { + return len(in) > 17 && ((in[16] == 0x01 && in[17] == 0x00) || + (in[16] == 0x00 && in[17] == 0x01)) +} + +// ElfExe matches an executable file. +func ElfExe(in []byte) bool { + return len(in) > 17 && ((in[16] == 0x02 && in[17] == 0x00) || + (in[16] == 0x00 && in[17] == 0x02)) +} + +// ElfLib matches a shared library file. +func ElfLib(in []byte) bool { + return len(in) > 17 && ((in[16] == 0x03 && in[17] == 0x00) || + (in[16] == 0x00 && in[17] == 0x03)) +} + +// ElfDump matches a core dump file. +func ElfDump(in []byte) bool { + return len(in) > 17 && ((in[16] == 0x04 && in[17] == 0x00) || + (in[16] == 0x00 && in[17] == 0x04)) +} + +// Dcm matches a DICOM medical format file. +func Dcm(in []byte) bool { + return len(in) > 131 && + bytes.Equal(in[128:132], []byte{0x44, 0x49, 0x43, 0x4D}) +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/database.go b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/database.go new file mode 100644 index 000000000..89901044a --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/database.go @@ -0,0 +1,10 @@ +package matchers + +import "bytes" + +// Sqlite matches an SQLite database file. +func Sqlite(in []byte) bool { + return bytes.HasPrefix(in, []byte{ + 0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00, + }) +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/document.go b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/document.go new file mode 100644 index 000000000..c853f5850 --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/document.go @@ -0,0 +1,35 @@ +package matchers + +import "bytes" + +// Pdf matches a Portable Document Format file. +func Pdf(in []byte) bool { + return bytes.HasPrefix(in, []byte{0x25, 0x50, 0x44, 0x46}) +} + +// DjVu matches a DjVu file +func DjVu(in []byte) bool { + if !bytes.HasPrefix(in, []byte{0x41, 0x54, 0x26, 0x54, 0x46, 0x4F, 0x52, 0x4D}) { + return false + } + if len(in) < 15 { + return false + } + return bytes.HasPrefix(in[12:], []byte("DJVM")) || + bytes.HasPrefix(in[12:], []byte("DJVU")) || + bytes.HasPrefix(in[12:], []byte("DJVI")) || + bytes.HasPrefix(in[12:], []byte("THUM")) +} + +// Mobi matches a Mobi file +func Mobi(in []byte) bool { + if len(in) < 68 { + return false + } + return bytes.Equal(in[60:68], []byte("BOOKMOBI")) +} + +// Lit matches a Microsoft Lit file +func Lit(in []byte) bool { + return bytes.HasPrefix(in, []byte("ITOLITLS")) +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/font.go b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/font.go new file mode 100644 index 000000000..11ae659d4 --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/font.go @@ -0,0 +1,27 @@ +package matchers + +import "bytes" + +// Woff matches a Web Open Font Format file. +func Woff(in []byte) bool { + return bytes.HasPrefix(in, []byte("wOFF")) +} + +// Woff2 matches a Web Open Font Format version 2 file. +func Woff2(in []byte) bool { + return bytes.HasPrefix(in, []byte("wOF2")) +} + +// Otf matches an OpenType font file. +func Otf(in []byte) bool { + return bytes.HasPrefix(in, []byte{0x4F, 0x54, 0x54, 0x4F, 0x00}) +} + +// Eot matches an Embedded OpenType font file. +func Eot(in []byte) bool { + return len(in) > 35 && + bytes.Equal(in[34:36], []byte{0x4C, 0x50}) && + (bytes.Equal(in[8:11], []byte{0x02, 0x00, 0x01}) || + bytes.Equal(in[8:11], []byte{0x01, 0x00, 0x00}) || + bytes.Equal(in[8:11], []byte{0x02, 0x00, 0x02})) +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/geo.go b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/geo.go new file mode 100644 index 000000000..86f46305a --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/geo.go @@ -0,0 +1,44 @@ +package matchers + +import ( + "bytes" + "encoding/binary" +) + +// Shp matches a shape format file. +// https://www.esri.com/library/whitepapers/pdfs/shapefile.pdf +func Shp(in []byte) bool { + if len(in) < 112 { + return false + } + shapeTypes := []int{ + 0, // Null shape + 1, // Point + 3, // Polyline + 5, // Polygon + 8, // MultiPoint + 11, // PointZ + 13, // PolylineZ + 15, // PolygonZ + 18, // MultiPointZ + 21, // PointM + 23, // PolylineM + 25, // PolygonM + 28, // MultiPointM + 31, // MultiPatch + } + + for _, st := range shapeTypes { + if st == int(binary.LittleEndian.Uint32(in[108:112])) { + return true + } + } + + return false +} + +// Shx matches a shape index format file. +// https://www.esri.com/library/whitepapers/pdfs/shapefile.pdf +func Shx(in []byte) bool { + return bytes.HasPrefix(in, []byte{0x00, 0x00, 0x27, 0x0A}) +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/image.go b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/image.go new file mode 100644 index 000000000..e2a5827ee --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/image.go @@ -0,0 +1,91 @@ +package matchers + +import "bytes" + +// Png matches a Portable Network Graphics file. +func Png(in []byte) bool { + return bytes.HasPrefix(in, []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}) +} + +// Jpg matches a Joint Photographic Experts Group file. +func Jpg(in []byte) bool { + return bytes.HasPrefix(in, []byte{0xFF, 0xD8, 0xFF}) +} + +// Gif matches a Graphics Interchange Format file. +func Gif(in []byte) bool { + return bytes.HasPrefix(in, []byte("GIF87a")) || + bytes.HasPrefix(in, []byte("GIF89a")) +} + +// Webp matches a WebP file. +func Webp(in []byte) bool { + return len(in) > 12 && + bytes.Equal(in[0:4], []byte{0x52, 0x49, 0x46, 0x46}) && + bytes.Equal(in[8:12], []byte{0x57, 0x45, 0x42, 0x50}) +} + +// Bmp matches a bitmap image file. +func Bmp(in []byte) bool { + return len(in) > 1 && in[0] == 0x42 && in[1] == 0x4D +} + +// Ps matches a PostScript file. +func Ps(in []byte) bool { + return bytes.HasPrefix(in, []byte("%!PS-Adobe-")) +} + +// Psd matches a Photoshop Document file. +func Psd(in []byte) bool { + return bytes.HasPrefix(in, []byte("8BPS")) +} + +// Ico matches an ICO file. +func Ico(in []byte) bool { + return len(in) > 3 && + in[0] == 0x00 && in[1] == 0x00 && + in[2] == 0x01 && in[3] == 0x00 +} + +// Tiff matches a Tagged Image File Format file. +func Tiff(in []byte) bool { + return bytes.HasPrefix(in, []byte{0x49, 0x49, 0x2A, 0x00}) || + bytes.HasPrefix(in, []byte{0x4D, 0x4D, 0x00, 0x2A}) +} + +// Bpg matches a Better Portable Graphics file. +func Bpg(in []byte) bool { + return bytes.HasPrefix(in, []byte{0x42, 0x50, 0x47, 0xFB}) +} + +// Dwg matches a CAD drawing file. +func Dwg(in []byte) bool { + if len(in) < 6 || in[0] != 0x41 || in[1] != 0x43 { + return false + } + dwgVersions := [][]byte{ + {0x31, 0x2E, 0x34, 0x30}, + {0x31, 0x2E, 0x35, 0x30}, + {0x32, 0x2E, 0x31, 0x30}, + {0x31, 0x30, 0x30, 0x32}, + {0x31, 0x30, 0x30, 0x33}, + {0x31, 0x30, 0x30, 0x34}, + {0x31, 0x30, 0x30, 0x36}, + {0x31, 0x30, 0x30, 0x39}, + {0x31, 0x30, 0x31, 0x32}, + {0x31, 0x30, 0x31, 0x34}, + {0x31, 0x30, 0x31, 0x35}, + {0x31, 0x30, 0x31, 0x38}, + {0x31, 0x30, 0x32, 0x31}, + {0x31, 0x30, 0x32, 0x34}, + {0x31, 0x30, 0x33, 0x32}, + } + + for _, d := range dwgVersions { + if bytes.Equal(in[2:6], d) { + return true + } + } + + return false +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/matchers.go b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/matchers.go new file mode 100644 index 000000000..6170443a8 --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/matchers.go @@ -0,0 +1,46 @@ +// Package matchers holds the matching functions used to find mime types. +package matchers + +// ReadLimit is the maximum number of bytes read +// from the input when detecting a reader. +const ReadLimit = 2048 + +// True is a dummy matching function used to match any input. +func True([]byte) bool { + return true +} + +// False is a dummy matching function used to never match input. +func False([]byte) bool { + return false +} + +// trimLWS trims whitespace from beginning of the input. +func trimLWS(in []byte) []byte { + firstNonWS := 0 + for ; firstNonWS < len(in) && isWS(in[firstNonWS]); firstNonWS++ { + } + + return in[firstNonWS:] +} + +// trimRWS trims whitespace from the end of the input. +func trimRWS(in []byte) []byte { + lastNonWS := len(in) - 1 + for ; lastNonWS > 0 && isWS(in[lastNonWS]); lastNonWS-- { + } + + return in[:lastNonWS+1] +} + +func firstLine(in []byte) []byte { + lineEnd := 0 + for ; lineEnd < len(in) && in[lineEnd] != '\n'; lineEnd++ { + } + + return in[:lineEnd] +} + +func isWS(b byte) bool { + return b == '\t' || b == '\n' || b == '\x0c' || b == '\r' || b == ' ' +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/ms_office.go b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/ms_office.go new file mode 100644 index 000000000..dadc48ae0 --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/ms_office.go @@ -0,0 +1,73 @@ +package matchers + +import ( + "bytes" + "fmt" + "strings" +) + +// Xlsx matches a Microsoft Excel 2007 file. +func Xlsx(in []byte) bool { + return bytes.Contains(in, []byte("xl/")) +} + +// Docx matches a Microsoft Office 2007 file. +func Docx(in []byte) bool { + return bytes.Contains(in, []byte("word/")) +} + +// Pptx matches a Microsoft PowerPoint 2007 file. +func Pptx(in []byte) bool { + return bytes.Contains(in, []byte("ppt/")) +} + +// Doc matches a Microsoft Office 97-2003 file. +func Doc(in []byte) bool { + return bytes.HasPrefix(in, []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}) +} + +// Ppt matches a Microsoft PowerPoint 97-2003 file. +func Ppt(in []byte) bool { + if len(in) < 520 { + return false + } + + if fmt.Sprintf("%X", in[:8]) == "D0CF11E0A1B11AE1" { + offset512 := fmt.Sprintf("%X", in[512:516]) + if offset512 == "A0461DF0" || offset512 == "006E1EF0" || offset512 == "0F00E803" { + return true + } + if offset512 == "FDFFFFFF" && fmt.Sprintf("%x", in[518:520]) == "0000" { + return true + } + } + + return false +} + +// Xls matches a Microsoft Excel 97-2003 file. +func Xls(in []byte) bool { + if len(in) < 520 { + return false + } + + if fmt.Sprintf("%X", in[:8]) == "D0CF11E0A1B11AE1" { + offset512 := fmt.Sprintf("%X", in[512:520]) + subheaders := []string{ + "0908100000060500", + "FDFFFFFF10", + "FDFFFFFF1F", + "FDFFFFFF22", + "FDFFFFFF23", + "FDFFFFFF28", + "FDFFFFFF29", + } + for _, h := range subheaders { + if strings.HasPrefix(offset512, h) { + return true + } + } + } + + return false +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/odf.go b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/odf.go new file mode 100644 index 000000000..ad31e666b --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/odf.go @@ -0,0 +1,48 @@ +package matchers + +import "bytes" + +// Odt matches an OpenDocument Text file. +func Odt(in []byte) bool { + return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.text")) +} + +// Ott matches an OpenDocument Text Template file. +func Ott(in []byte) bool { + return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.text-template")) +} + +// Ods matches an OpenDocument Spreadsheet file. +func Ods(in []byte) bool { + return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.spreadsheet")) +} + +// Ots matches an OpenDocument Spreadsheet Template file. +func Ots(in []byte) bool { + return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.spreadsheet-template")) +} + +// Odp matches an OpenDocument Presentation file. +func Odp(in []byte) bool { + return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.presentation")) +} + +// Otp matches an OpenDocument Presentation Template file. +func Otp(in []byte) bool { + return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.presentation-template")) +} + +// Odg matches an OpenDocument Drawing file. +func Odg(in []byte) bool { + return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.graphics")) +} + +// Otg matches an OpenDocument Drawing Template file. +func Otg(in []byte) bool { + return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.graphics-template")) +} + +// Odf matches an OpenDocument Formula file. +func Odf(in []byte) bool { + return bytes.Contains(in, []byte("mimetypeapplication/vnd.oasis.opendocument.formula")) +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/signature.go b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/signature.go new file mode 100644 index 000000000..75267090f --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/signature.go @@ -0,0 +1,128 @@ +package matchers + +import ( + "bytes" + "fmt" +) + +type ( + markupSig []byte + ciSig []byte // case insensitive signature + shebangSig []byte // matches !# followed by the signature + ftypSig []byte // matches audio/video files. www.ftyps.com + xmlSig struct { + // the local name of the root tag + localName []byte + // the namespace of the XML document + xmlns []byte + } + sig interface { + detect([]byte) bool + } +) + +func newXmlSig(localName, xmlns string) xmlSig { + ret := xmlSig{xmlns: []byte(xmlns)} + if localName != "" { + ret.localName = []byte(fmt.Sprintf("<%s", localName)) + } + + return ret +} + +// Implement sig interface. +func (hSig markupSig) detect(in []byte) bool { + if len(in) < len(hSig)+1 { + return false + } + + // perform case insensitive check + for i, b := range hSig { + db := in[i] + if 'A' <= b && b <= 'Z' { + db &= 0xDF + } + if b != db { + return false + } + } + // Next byte must be space or right angle bracket. + if db := in[len(hSig)]; db != ' ' && db != '>' { + return false + } + + return true +} + +// Implement sig interface. +func (tSig ciSig) detect(in []byte) bool { + if len(in) < len(tSig)+1 { + return false + } + + // perform case insensitive check + for i, b := range tSig { + db := in[i] + if 'A' <= b && b <= 'Z' { + db &= 0xDF + } + if b != db { + return false + } + } + + return true +} + +// a valid shebang starts with the "#!" characters +// followed by any number of spaces +// followed by the path to the interpreter and optionally, the args for the interpreter +func (sSig shebangSig) detect(in []byte) bool { + in = firstLine(in) + + if len(in) < len(sSig)+2 { + return false + } + if in[0] != '#' || in[1] != '!' { + return false + } + + in = trimLWS(trimRWS(in[2:])) + + return bytes.Equal(in, sSig) +} + +// Implement sig interface. +func (fSig ftypSig) detect(in []byte) bool { + return len(in) > 12 && + bytes.Equal(in[4:8], []byte("ftyp")) && + bytes.Equal(in[8:12], fSig) +} + +func (xSig xmlSig) detect(in []byte) bool { + l := 512 + if len(in) < l { + l = len(in) + } + in = in[:l] + + if len(xSig.localName) == 0 { + return bytes.Index(in, xSig.xmlns) > 0 + } + if len(xSig.xmlns) == 0 { + return bytes.Index(in, xSig.localName) > 0 + } + + localNameIndex := bytes.Index(in, xSig.localName) + return localNameIndex != -1 && localNameIndex < bytes.Index(in, xSig.xmlns) +} + +func detect(in []byte, sigs []sig) bool { + for _, sig := range sigs { + if sig.detect(in) { + return true + } + } + + return false +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/text.go b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/text.go new file mode 100644 index 000000000..e9ca91c44 --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/text.go @@ -0,0 +1,304 @@ +package matchers + +import ( + "bytes" + + "github.com/gabriel-vasile/mimetype/internal/json" +) + +var ( + htmlSigs = []sig{ + markupSig(" 1 && len(lines) > 1 +} + +// butLastLineReader returns a reader to the provided byte slice. +// the reader is guaranteed to reach EOF before it reads `cutAt` bytes. +// bytes after the last newline are dropped from the input. +func butLastLineReader(in []byte, cutAt int) io.Reader { + if len(in) >= cutAt { + for i := cutAt - 1; i > 0; i-- { + if in[i] == '\n' { + return bytes.NewReader(in[:i]) + } + } + + // no newline was found between the 0 index and cutAt + return bytes.NewReader(in[:cutAt]) + } + + return bytes.NewReader(in) +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/video.go b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/video.go new file mode 100644 index 000000000..2c1fcc1a1 --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/video.go @@ -0,0 +1,70 @@ +package matchers + +import ( + "bytes" +) + +// WebM matches a WebM file. +func WebM(in []byte) bool { + return isMatroskaFileTypeMatched(in, "webm") +} + +// Mkv matches a mkv file. +func Mkv(in []byte) bool { + return isMatroskaFileTypeMatched(in, "matroska") +} + +// isMatroskaFileTypeMatched is used for webm and mkv file matching. +// It checks for .Eߣ sequence. If the sequence is found, +// then it means it is Matroska media container, including WebM. +// Then it verifies which of the file type it is representing by matching the +// file specific string. +func isMatroskaFileTypeMatched(in []byte, flType string) bool { + if bytes.HasPrefix(in, []byte("\x1A\x45\xDF\xA3")) { + return isFileTypeNamePresent(in, flType) + } + return false +} + +// isFileTypeNamePresent accepts the matroska input data stream and searches +// for the given file type in the stream. Return whether a match is found. +// The logic of search is: find first instance of \x42\x82 and then +// search for given string after one byte of above instance. +func isFileTypeNamePresent(in []byte, flType string) bool { + var ind int + if len(in) >= 4096 { // restricting length to 4096 + ind = bytes.Index(in[0:4096], []byte("\x42\x82")) + } else { + ind = bytes.Index(in, []byte("\x42\x82")) + } + if ind > 0 { + // filetype name will be present exactly + // one byte after the match of the two bytes "\x42\x82" + return bytes.HasPrefix(in[ind+3:], []byte(flType)) + } + return false +} + +// Flv matches a Flash video file. +func Flv(in []byte) bool { + return bytes.HasPrefix(in, []byte("\x46\x4C\x56\x01")) +} + +// Mpeg matches a Moving Picture Experts Group file. +func Mpeg(in []byte) bool { + return bytes.HasPrefix(in, []byte{0x00, 0x00, 0x01}) && + in[3] >= 0xB0 && in[3] <= 0xBF +} + +// Avi matches an Audio Video Interleaved file. +func Avi(in []byte) bool { + return len(in) > 16 && + bytes.Equal(in[:4], []byte("RIFF")) && + bytes.Equal(in[8:16], []byte("AVI LIST")) +} + +// Asf matches an Advanced Systems Format file. +func Asf(in []byte) bool { + return len(in) > 16 && bytes.Equal(in[:16], []byte{0x30, 0x26, 0xB2, 0x75, + 0x8E, 0x66, 0xCF, 0x11, 0xA6, 0xD9, 0x00, 0xAA, 0x00, 0x62, 0xCE, 0x6C}) +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/video_ftyp.go b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/video_ftyp.go new file mode 100644 index 000000000..0d470cd3c --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/internal/matchers/video_ftyp.go @@ -0,0 +1,70 @@ +package matchers + +var ( + mp4Sigs = []sig{ + ftypSig("avc1"), ftypSig("dash"), ftypSig("iso2"), ftypSig("iso3"), + ftypSig("iso4"), ftypSig("iso5"), ftypSig("iso6"), ftypSig("isom"), + ftypSig("mmp4"), ftypSig("mp41"), ftypSig("mp42"), ftypSig("mp4v"), + ftypSig("mp71"), ftypSig("MSNV"), ftypSig("NDAS"), ftypSig("NDSC"), + ftypSig("NSDC"), ftypSig("NSDH"), ftypSig("NDSM"), ftypSig("NDSP"), + ftypSig("NDSS"), ftypSig("NDXC"), ftypSig("NDXH"), ftypSig("NDXM"), + ftypSig("NDXP"), ftypSig("NDXS"), ftypSig("F4V "), ftypSig("F4P "), + } + threeGPSigs = []sig{ + ftypSig("3gp1"), ftypSig("3gp2"), ftypSig("3gp3"), ftypSig("3gp4"), + ftypSig("3gp5"), ftypSig("3gp6"), ftypSig("3gs7"), ftypSig("3ge6"), + ftypSig("3ge7"), ftypSig("3gg6"), + } + threeG2Sigs = []sig{ + ftypSig("3g2a"), ftypSig("3g2b"), ftypSig("3g2c"), ftypSig("KDDI"), + } + amp4Sigs = []sig{ + // audio for Adobe Flash Player 9+ + ftypSig("F4A "), ftypSig("F4B "), + // Apple iTunes AAC-LC (.M4A) Audio + ftypSig("M4B "), ftypSig("M4P "), + // MPEG-4 (.MP4) for SonyPSP + ftypSig("MSNV"), + // Nero Digital AAC Audio + ftypSig("NDAS"), + } + qtSigs = []sig{ftypSig("qt "), ftypSig("moov")} + mqvSigs = []sig{ftypSig("mqt ")} + m4aSigs = []sig{ftypSig("M4A ")} + // TODO: add support for remaining video formats at ftyps.com. +) + +// Mp4 matches an MP4 file. +func Mp4(in []byte) bool { + return detect(in, mp4Sigs) +} + +// ThreeGP matches a 3GPP file. +func ThreeGP(in []byte) bool { + return detect(in, threeGPSigs) +} + +// ThreeG2 matches a 3GPP2 file. +func ThreeG2(in []byte) bool { + return detect(in, threeG2Sigs) +} + +// AMp4 matches an audio MP4 file. +func AMp4(in []byte) bool { + return detect(in, amp4Sigs) +} + +// QuickTime matches a QuickTime File Format file. +func QuickTime(in []byte) bool { + return detect(in, qtSigs) +} + +// Mqv matches a Sony / Mobile QuickTime file. +func Mqv(in []byte) bool { + return detect(in, mqvSigs) +} + +// M4a matches an audio M4A file. +func M4a(in []byte) bool { + return detect(in, m4aSigs) +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/mime.go b/vendor/github.com/gabriel-vasile/mimetype/mime.go new file mode 100644 index 000000000..5992a2c4e --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/mime.go @@ -0,0 +1,55 @@ +// Package mimetype uses magic number signatures +// to detect the MIME type and extension of a file. +package mimetype + +import ( + "io" + "os" + + "github.com/gabriel-vasile/mimetype/internal/matchers" +) + +// Detect returns the MIME type and extension of the provided byte slice. +// +// mime is always a valid MIME type, with application/octet-stream as fallback. +// extension is empty string if detected file format does not have an extension. +func Detect(in []byte) (mime, extension string) { + if len(in) == 0 { + return "inode/x-empty", "" + } + n := root.match(in, root) + return n.mime, n.extension +} + +// DetectReader returns the MIME type and extension +// of the byte slice read from the provided reader. +// +// mime is always a valid MIME type, with application/octet-stream as fallback. +// extension is empty string if detection failed with an error or +// detected file format does not have an extension. +func DetectReader(r io.Reader) (mime, extension string, err error) { + in := make([]byte, matchers.ReadLimit) + n, err := r.Read(in) + if err != nil && err != io.EOF { + return root.mime, root.extension, err + } + in = in[:n] + + mime, extension = Detect(in) + return mime, extension, nil +} + +// DetectFile returns the MIME type and extension of the provided file. +// +// mime is always a valid MIME type, with application/octet-stream as fallback. +// extension is empty string if detection failed with an error or +// detected file format does not have an extension. +func DetectFile(file string) (mime, extension string, err error) { + f, err := os.Open(file) + if err != nil { + return root.mime, root.extension, err + } + defer f.Close() + + return DetectReader(f) +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/mimetype.gif b/vendor/github.com/gabriel-vasile/mimetype/mimetype.gif new file mode 100644 index 000000000..912aa5776 Binary files /dev/null and b/vendor/github.com/gabriel-vasile/mimetype/mimetype.gif differ diff --git a/vendor/github.com/gabriel-vasile/mimetype/node.go b/vendor/github.com/gabriel-vasile/mimetype/node.go new file mode 100644 index 000000000..721c76b8c --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/node.go @@ -0,0 +1,43 @@ +package mimetype + +type ( + // node represents a vertex in the matchers tree structure. + // It holds the mime type, the extension and the function + // to check whether a byte slice has the mime type. + node struct { + mime string + extension string + matchFunc func([]byte) bool + children []*node + } +) + +func newNode(mime, extension string, matchFunc func([]byte) bool, children ...*node) *node { + return &node{ + mime: mime, + extension: extension, + matchFunc: matchFunc, + children: children, + } +} + +// match does a depth-first search on the matchers tree. +// it returns the deepest successful matcher for which all the children fail. +func (n *node) match(in []byte, deepestMatch *node) *node { + for _, c := range n.children { + if c.matchFunc(in) { + return c.match(in, c) + } + } + + return deepestMatch +} + +func (n *node) flatten() []*node { + out := []*node{n} + for _, c := range n.children { + out = append(out, c.flatten()...) + } + + return out +} diff --git a/vendor/github.com/gabriel-vasile/mimetype/supported_mimes.md b/vendor/github.com/gabriel-vasile/mimetype/supported_mimes.md new file mode 100644 index 000000000..abe6cb36c --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/supported_mimes.md @@ -0,0 +1,119 @@ +## 114 Supported MIME types +This file is automatically generated when running tests. Do not edit manually. + +Extension | MIME type +--------- | -------- +**n/a** | application/octet-stream +**7z** | application/x-7z-compressed +**zip** | application/zip +**xlsx** | application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +**docx** | application/vnd.openxmlformats-officedocument.wordprocessingml.document +**pptx** | application/vnd.openxmlformats-officedocument.presentationml.presentation +**epub** | application/epub+zip +**jar** | application/jar +**odt** | application/vnd.oasis.opendocument.text +**ott** | application/vnd.oasis.opendocument.text-template +**ods** | application/vnd.oasis.opendocument.spreadsheet +**ots** | application/vnd.oasis.opendocument.spreadsheet-template +**odp** | application/vnd.oasis.opendocument.presentation +**otp** | application/vnd.oasis.opendocument.presentation-template +**odg** | application/vnd.oasis.opendocument.graphics +**otg** | application/vnd.oasis.opendocument.graphics-template +**odf** | application/vnd.oasis.opendocument.formula +**pdf** | application/pdf +**xls** | application/vnd.ms-excel +**ppt** | application/vnd.ms-powerpoint +**doc** | application/msword +**ps** | application/postscript +**psd** | application/x-photoshop +**ogg** | application/ogg +**png** | image/png +**jpg** | image/jpeg +**gif** | image/gif +**webp** | image/webp +**exe** | application/vnd.microsoft.portable-executable +**n/a** | application/x-elf +**n/a** | application/x-object +**n/a** | application/x-executable +**so** | application/x-sharedlib +**n/a** | application/x-coredump +**a** | application/x-archive +**deb** | application/vnd.debian.binary-package +**tar** | application/x-tar +**xar** | application/x-xar +**bz2** | application/x-bzip2 +**fits** | application/fits +**tiff** | image/tiff +**bmp** | image/bmp +**ico** | image/x-icon +**mp3** | audio/mpeg +**flac** | audio/flac +**midi** | audio/midi +**ape** | audio/ape +**mpc** | audio/musepack +**amr** | audio/amr +**wav** | audio/wav +**aiff** | audio/aiff +**au** | audio/basic +**mpeg** | video/mpeg +**mov** | video/quicktime +**mqv** | video/quicktime +**mp4** | video/mp4 +**webm** | video/webm +**3gp** | video/3gpp +**3g2** | video/3gpp2 +**avi** | video/x-msvideo +**flv** | video/x-flv +**mkv** | video/x-matroska +**asf** | video/x-ms-asf +**aac** | audio/aac +**voc** | audio/x-unknown +**mp4** | audio/mp4 +**m4a** | audio/x-m4a +**txt** | text/plain +**html** | text/html; charset=utf-8 +**svg** | image/svg+xml +**xml** | text/xml; charset=utf-8 +**rss** | application/rss+xml +**atom** | application/atom+xml +**x3d** | model/x3d+xml +**kml** | application/vnd.google-earth.kml+xml +**xlf** | application/x-xliff+xml +**dae** | model/vnd.collada+xml +**gml** | application/gml+xml +**gpx** | application/gpx+xml +**tcx** | application/vnd.garmin.tcx+xml +**amf** | application/x-amf +**3mf** | application/vnd.ms-package.3dmanufacturing-3dmodel+xml +**php** | text/x-php; charset=utf-8 +**js** | application/javascript +**lua** | text/x-lua +**pl** | text/x-perl +**py** | application/x-python +**json** | application/json +**geojson** | application/geo+json +**rtf** | text/rtf +**tcl** | text/x-tcl +**csv** | text/csv +**tsv** | text/tab-separated-values +**vcf** | text/vcard +**gz** | application/gzip +**class** | application/x-java-applet; charset=binary +**swf** | application/x-shockwave-flash +**crx** | application/x-chrome-extension +**woff** | font/woff +**woff2** | font/woff2 +**otf** | font/otf +**eot** | application/vnd.ms-fontobject +**wasm** | application/wasm +**shx** | application/octet-stream +**shp** | application/octet-stream +**dbf** | application/x-dbf +**dcm** | application/dicom +**rar** | application/x-rar-compressed +**djvu** | image/vnd.djvu +**mobi** | application/x-mobipocket-ebook +**lit** | application/x-ms-reader +**bpg** | image/bpg +**sqlite** | application/x-sqlite3 +**dwg** | image/vnd.dwg diff --git a/vendor/github.com/gabriel-vasile/mimetype/tree.go b/vendor/github.com/gabriel-vasile/mimetype/tree.go new file mode 100644 index 000000000..4543c3e32 --- /dev/null +++ b/vendor/github.com/gabriel-vasile/mimetype/tree.go @@ -0,0 +1,131 @@ +package mimetype + +import "github.com/gabriel-vasile/mimetype/internal/matchers" + +// root is a matcher which passes for any slice of bytes. +// When a matcher passes the check, the children matchers +// are tried in order to find a more accurate mime type. +var root = newNode("application/octet-stream", "", matchers.True, + sevenZ, zip, pdf, xls, ppt, doc, ps, psd, ogg, png, jpg, gif, webp, exe, elf, + ar, tar, xar, bz2, fits, tiff, bmp, ico, mp3, flac, midi, ape, musePack, amr, + wav, aiff, au, mpeg, quickTime, mqv, mp4, webM, threeGP, threeG2, avi, flv, + mkv, asf, aac, voc, aMp4, m4a, txt, gzip, class, swf, crx, woff, woff2, otf, + eot, wasm, shx, dbf, dcm, rar, djvu, mobi, lit, bpg, sqlite3, dwg, +) + +// The list of nodes appended to the root node +var ( + gzip = newNode("application/gzip", "gz", matchers.Gzip) + sevenZ = newNode("application/x-7z-compressed", "7z", matchers.SevenZ) + zip = newNode("application/zip", "zip", matchers.Zip, xlsx, docx, pptx, epub, jar, odt, ods, odp, odg, odf) + tar = newNode("application/x-tar", "tar", matchers.Tar) + xar = newNode("application/x-xar", "xar", matchers.Xar) + bz2 = newNode("application/x-bzip2", "bz2", matchers.Bz2) + pdf = newNode("application/pdf", "pdf", matchers.Pdf) + xlsx = newNode("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xlsx", matchers.Xlsx) + docx = newNode("application/vnd.openxmlformats-officedocument.wordprocessingml.document", "docx", matchers.Docx) + pptx = newNode("application/vnd.openxmlformats-officedocument.presentationml.presentation", "pptx", matchers.Pptx) + epub = newNode("application/epub+zip", "epub", matchers.Epub) + jar = newNode("application/jar", "jar", matchers.Jar) + doc = newNode("application/msword", "doc", matchers.Doc) + ppt = newNode("application/vnd.ms-powerpoint", "ppt", matchers.Ppt) + xls = newNode("application/vnd.ms-excel", "xls", matchers.Xls) + ps = newNode("application/postscript", "ps", matchers.Ps) + psd = newNode("application/x-photoshop", "psd", matchers.Psd) + fits = newNode("application/fits", "fits", matchers.Fits) + ogg = newNode("application/ogg", "ogg", matchers.Ogg) + txt = newNode("text/plain", "txt", matchers.Txt, html, svg, xml, php, js, lua, perl, python, json, rtf, tcl, csv, tsv, vCard) + xml = newNode("text/xml; charset=utf-8", "xml", matchers.Xml, rss, atom, x3d, kml, xliff, collada, gml, gpx, tcx, amf, threemf) + json = newNode("application/json", "json", matchers.Json, geoJson) + csv = newNode("text/csv", "csv", matchers.Csv) + tsv = newNode("text/tab-separated-values", "tsv", matchers.Tsv) + geoJson = newNode("application/geo+json", "geojson", matchers.GeoJson) + html = newNode("text/html; charset=utf-8", "html", matchers.Html) + php = newNode("text/x-php; charset=utf-8", "php", matchers.Php) + rtf = newNode("text/rtf", "rtf", matchers.Rtf) + js = newNode("application/javascript", "js", matchers.Js) + lua = newNode("text/x-lua", "lua", matchers.Lua) + perl = newNode("text/x-perl", "pl", matchers.Perl) + python = newNode("application/x-python", "py", matchers.Python) + tcl = newNode("text/x-tcl", "tcl", matchers.Tcl) + vCard = newNode("text/vcard", "vcf", matchers.VCard) + svg = newNode("image/svg+xml", "svg", matchers.Svg) + rss = newNode("application/rss+xml", "rss", matchers.Rss) + atom = newNode("application/atom+xml", "atom", matchers.Atom) + x3d = newNode("model/x3d+xml", "x3d", matchers.X3d) + kml = newNode("application/vnd.google-earth.kml+xml", "kml", matchers.Kml) + xliff = newNode("application/x-xliff+xml", "xlf", matchers.Xliff) + collada = newNode("model/vnd.collada+xml", "dae", matchers.Collada) + gml = newNode("application/gml+xml", "gml", matchers.Gml) + gpx = newNode("application/gpx+xml", "gpx", matchers.Gpx) + tcx = newNode("application/vnd.garmin.tcx+xml", "tcx", matchers.Tcx) + amf = newNode("application/x-amf", "amf", matchers.Amf) + threemf = newNode("application/vnd.ms-package.3dmanufacturing-3dmodel+xml", "3mf", matchers.Threemf) + png = newNode("image/png", "png", matchers.Png) + jpg = newNode("image/jpeg", "jpg", matchers.Jpg) + bpg = newNode("image/bpg", "bpg", matchers.Bpg) + gif = newNode("image/gif", "gif", matchers.Gif) + webp = newNode("image/webp", "webp", matchers.Webp) + tiff = newNode("image/tiff", "tiff", matchers.Tiff) + bmp = newNode("image/bmp", "bmp", matchers.Bmp) + ico = newNode("image/x-icon", "ico", matchers.Ico) + mp3 = newNode("audio/mpeg", "mp3", matchers.Mp3) + flac = newNode("audio/flac", "flac", matchers.Flac) + midi = newNode("audio/midi", "midi", matchers.Midi) + ape = newNode("audio/ape", "ape", matchers.Ape) + musePack = newNode("audio/musepack", "mpc", matchers.MusePack) + wav = newNode("audio/wav", "wav", matchers.Wav) + aiff = newNode("audio/aiff", "aiff", matchers.Aiff) + au = newNode("audio/basic", "au", matchers.Au) + amr = newNode("audio/amr", "amr", matchers.Amr) + aac = newNode("audio/aac", "aac", matchers.Aac) + voc = newNode("audio/x-unknown", "voc", matchers.Voc) + aMp4 = newNode("audio/mp4", "mp4", matchers.AMp4) + m4a = newNode("audio/x-m4a", "m4a", matchers.M4a) + mp4 = newNode("video/mp4", "mp4", matchers.Mp4) + webM = newNode("video/webm", "webm", matchers.WebM) + mpeg = newNode("video/mpeg", "mpeg", matchers.Mpeg) + quickTime = newNode("video/quicktime", "mov", matchers.QuickTime) + mqv = newNode("video/quicktime", "mqv", matchers.Mqv) + threeGP = newNode("video/3gpp", "3gp", matchers.ThreeGP) + threeG2 = newNode("video/3gpp2", "3g2", matchers.ThreeG2) + avi = newNode("video/x-msvideo", "avi", matchers.Avi) + flv = newNode("video/x-flv", "flv", matchers.Flv) + mkv = newNode("video/x-matroska", "mkv", matchers.Mkv) + asf = newNode("video/x-ms-asf", "asf", matchers.Asf) + class = newNode("application/x-java-applet; charset=binary", "class", matchers.Class) + swf = newNode("application/x-shockwave-flash", "swf", matchers.Swf) + crx = newNode("application/x-chrome-extension", "crx", matchers.Crx) + woff = newNode("font/woff", "woff", matchers.Woff) + woff2 = newNode("font/woff2", "woff2", matchers.Woff2) + otf = newNode("font/otf", "otf", matchers.Otf) + eot = newNode("application/vnd.ms-fontobject", "eot", matchers.Eot) + wasm = newNode("application/wasm", "wasm", matchers.Wasm) + shp = newNode("application/octet-stream", "shp", matchers.Shp) + shx = newNode("application/octet-stream", "shx", matchers.Shx, shp) + dbf = newNode("application/x-dbf", "dbf", matchers.Dbf) + exe = newNode("application/vnd.microsoft.portable-executable", "exe", matchers.Exe) + elf = newNode("application/x-elf", "", matchers.Elf, elfObj, elfExe, elfLib, elfDump) + elfObj = newNode("application/x-object", "", matchers.ElfObj) + elfExe = newNode("application/x-executable", "", matchers.ElfExe) + elfLib = newNode("application/x-sharedlib", "so", matchers.ElfLib) + elfDump = newNode("application/x-coredump", "", matchers.ElfDump) + ar = newNode("application/x-archive", "a", matchers.Ar, deb) + deb = newNode("application/vnd.debian.binary-package", "deb", matchers.Deb) + dcm = newNode("application/dicom", "dcm", matchers.Dcm) + odt = newNode("application/vnd.oasis.opendocument.text", "odt", matchers.Odt, ott) + ott = newNode("application/vnd.oasis.opendocument.text-template", "ott", matchers.Ott) + ods = newNode("application/vnd.oasis.opendocument.spreadsheet", "ods", matchers.Ods, ots) + ots = newNode("application/vnd.oasis.opendocument.spreadsheet-template", "ots", matchers.Ots) + odp = newNode("application/vnd.oasis.opendocument.presentation", "odp", matchers.Odp, otp) + otp = newNode("application/vnd.oasis.opendocument.presentation-template", "otp", matchers.Otp) + odg = newNode("application/vnd.oasis.opendocument.graphics", "odg", matchers.Odg, otg) + otg = newNode("application/vnd.oasis.opendocument.graphics-template", "otg", matchers.Otg) + odf = newNode("application/vnd.oasis.opendocument.formula", "odf", matchers.Odf) + rar = newNode("application/x-rar-compressed", "rar", matchers.Rar) + djvu = newNode("image/vnd.djvu", "djvu", matchers.DjVu) + mobi = newNode("application/x-mobipocket-ebook", "mobi", matchers.Mobi) + lit = newNode("application/x-ms-reader", "lit", matchers.Lit) + sqlite3 = newNode("application/x-sqlite3", "sqlite", matchers.Sqlite) + dwg = newNode("image/vnd.dwg", "dwg", matchers.Dwg) +)