DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Cleanup internal, vendors, cleanup cmd/*

Browse Source 
Introduces /pkg for non-intenral packages
This commit is contained in:
Denis Arh
2019-05-24 12:44:56 +02:00
parent b66ed81136
commit 5a9bce44e8
90 changed files with 4168 additions and 1525 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/api/cors.go
+18
View File
@@ -0,0 +1,18 @@
package api
import (
"net/http"
"github.com/go-chi/cors"
)
// Sets up default CORS rules to use as a middleware
func handleCORS(next http.Handler) http.Handler {
return cors.New(cors.Options{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"},
AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "X-CSRF-Token"},
AllowCredentials: true,
MaxAge: 300, // Maximum value not ignored by any of major browsers
}).Handler(next)
}
pkg/api/debug.go
+37
View File
@@ -0,0 +1,37 @@
package api
import (
"fmt"
"net/http"
"reflect"
"runtime"
"github.com/go-chi/chi"
"github.com/go-chi/chi/middleware"
)
func Debug(r chi.Router) {
r.Mount("/debug", middleware.Profiler())
DebugRoutes(r)
}
func DebugRoutes(r chi.Router) {
r.Get("/debug/routes", func(w http.ResponseWriter, req *http.Request) {
var printRoutes func(chi.Routes, string)
printRoutes = func(r chi.Routes, pfix string) {
routes := r.Routes()
for _, route := range routes {
if route.SubRoutes != nil && len(route.SubRoutes.Routes()) > 0 {
printRoutes(route.SubRoutes, pfix+route.Pattern[:len(route.Pattern)-2])
} else {
for method, fn := range route.Handlers {
fmt.Fprintf(w, "%-8s %-80s -> %s\n", method, pfix+route.Pattern, runtime.FuncForPC(reflect.ValueOf(fn).Pointer()).Name())
}
}
}
}
printRoutes(r, "")
})
}
pkg/api/logger.go
+79
View File
@@ -0,0 +1,79 @@
package api
import (
"net/http"
"strings"
"time"
"github.com/go-chi/chi/middleware"
"go.uber.org/zap"
"github.com/cortezaproject/corteza-server/internal/logger"
)
// contextLogger middleware binds logger to request's context.
//
// This allows us to use logger from context (with requestID)
// inside our (generated) handers and controllers
func contextLogger(log *zap.Logger) func(next http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
var requestID = middleware.GetReqID(req.Context())
w.Header().Add("X-Request-Id", requestID)
req = req.WithContext(logger.ContextWithValue(
req.Context(),
log.With(zap.String("requestID", requestID)).Named("rest"),
))
next.ServeHTTP(w, req)
})
}
}
// LogRequest middleware logs request details
//
// It uses logger from context, see contextLogger()
func LogRequest(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
var remote = req.RemoteAddr
if l := strings.LastIndex(remote, ":"); l > -1 {
remote = remote[:l]
}
logger.ContextValue(req.Context()).Info(
"HTTP request "+req.Method+" "+req.URL.Path,
zap.String("method", req.Method),
zap.String("path", req.URL.Path),
zap.Int64("size", req.ContentLength),
zap.String("remote", remote),
)
next.ServeHTTP(w, req)
})
}
// LogResponse middleware logs response details
//
// It uses logger from context, see contextLogger()
func LogResponse(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
wrapped := middleware.NewWrapResponseWriter(w, req.ProtoMajor)
t := time.Now()
defer func() {
logger.ContextValue(req.Context()).Info(
"HTTP response "+req.Method+" "+req.URL.Path,
zap.String("method", req.Method),
zap.String("path", req.URL.Path),
zap.Int("status", wrapped.Status()),
zap.Int("size", wrapped.BytesWritten()),
zap.Float64("duration", time.Since(t).Seconds()),
)
}()
next.ServeHTTP(wrapped, req)
})
}
pkg/api/metrics.go
+24
View File
@@ -0,0 +1,24 @@
package api
import (
"net/http"
"github.com/766b/chi-prometheus"
"github.com/99designs/basicauth-go"
"github.com/go-chi/chi"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
// Middleware is the request logger that provides metrics to prometheus
func Middleware(name string) func(http.Handler) http.Handler {
return chiprometheus.NewMiddleware(name)
}
func Mount(r chi.Router, username, password string) {
r.Group(func(r chi.Router) {
r.Use(basicauth.New("Metrics", map[string][]string{
username: {password},
}))
r.Handle("/metrics", promhttp.Handler())
})
}
pkg/api/middleware.go
+24
View File
@@ -0,0 +1,24 @@
package api
import (
"net/http"
"github.com/go-chi/chi/middleware"
"go.uber.org/zap"
)
func Base() []func(http.Handler) http.Handler {
return []func(http.Handler) http.Handler{
handleCORS,
middleware.RealIP,
middleware.RequestID,
}
}
func Logging(log *zap.Logger) []func(http.Handler) http.Handler {
return []func(http.Handler) http.Handler{
contextLogger(log),
LogRequest,
LogResponse,
}
}
pkg/api/monitor.go
+71
View File
@@ -0,0 +1,71 @@
package api
import (
"expvar"
"runtime"
"time"
"go.uber.org/zap"
"github.com/cortezaproject/corteza-server/internal/logger"
)
type Monitor struct {
Alloc,
TotalAlloc,
Sys,
Mallocs,
Frees,
LiveObjects,
PauseTotalNs uint64
NumGC uint32
NumGoroutine int
}
func NewMonitor(duration int) {
var (
m = Monitor{}
rtm runtime.MemStats
goroutines = expvar.NewInt("num_goroutine")
)
var interval = time.Duration(duration) * time.Second
for {
<-time.After(interval)
// Read full mem stats
runtime.ReadMemStats(&rtm)
// Number of goroutines
m.NumGoroutine = runtime.NumGoroutine()
goroutines.Set(int64(m.NumGoroutine))
// Misc memory stats
m.Alloc = rtm.Alloc
m.TotalAlloc = rtm.TotalAlloc
m.Sys = rtm.Sys
m.Mallocs = rtm.Mallocs
m.Frees = rtm.Frees
// Live objects = Mallocs - Frees
m.LiveObjects = m.Mallocs - m.Frees
// GC Stats
m.PauseTotalNs = rtm.PauseTotalNs
m.NumGC = rtm.NumGC
logger.Default().
With(
zap.Uint64("alloc", m.Alloc),
zap.Uint64("totalAlloc", m.TotalAlloc),
zap.Uint64("sys", m.Sys),
zap.Uint64("mallocs", m.Mallocs),
zap.Uint64("frees", m.Frees),
zap.Uint64("liveObjects", m.LiveObjects),
zap.Uint64("pauseTotalNs", m.PauseTotalNs),
zap.Uint32("numGC", m.NumGC),
zap.Int("numGoRoutines", m.NumGoroutine),
).
Debug("monitor")
}
}
pkg/api/server.go
+129
View File
@@ -0,0 +1,129 @@
package api
import (
"context"
"fmt"
"net"
"net/http"
"github.com/go-chi/chi"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/titpetric/factory/resputil"
"go.uber.org/zap"
"github.com/cortezaproject/corteza-server/internal/auth"
"github.com/cortezaproject/corteza-server/internal/version"
"github.com/cortezaproject/corteza-server/pkg/cli/flags"
)
type (
Server struct {
name string
log *zap.Logger
httpOpt *flags.HTTPOpt
monitorOpt *flags.MonitorOpt
endpoints []func(r chi.Router)
}
)
func NewServer(log *zap.Logger) *Server {
return &Server{
endpoints: make([]func(r chi.Router), 0),
log: log.Named("http"),
}
}
func (s *Server) Command(ctx context.Context, prefix string, preRun func(context.Context) error) (cmd *cobra.Command) {
cmd = &cobra.Command{
Use: "serve-api",
Short: "Start HTTP Server with REST API",
// Connect all the wires, prepare services, run watchers, bind endpoints
PreRun: func(cmd *cobra.Command, args []string) {
if s.monitorOpt.Interval > 0 {
go NewMonitor(s.monitorOpt.Interval)
}
preRun(ctx)
},
// Run the server
RunE: func(cmd *cobra.Command, args []string) error {
return s.Serve(ctx)
},
}
s.BindApiServerFlags(cmd, prefix)
return
}
func (s *Server) BindApiServerFlags(cmd *cobra.Command, prefix string) {
s.httpOpt = flags.HTTP(cmd, prefix)
s.monitorOpt = flags.Monitor(cmd, prefix)
}
func (s *Server) MountRoutes(mm ...func(chi.Router)) {
s.endpoints = append(s.endpoints, mm...)
}
func (s Server) Serve(ctx context.Context) error {
s.log.Info("Starting HTTP server with REST API", zap.String("address", s.httpOpt.Addr))
// configure resputil options
resputil.SetConfig(resputil.Options{
Pretty: s.httpOpt.Pretty,
Trace: s.httpOpt.Tracing,
Logger: func(err error) {
// @todo: error logging
},
})
listener, err := net.Listen("tcp", s.httpOpt.Addr)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Can't listen on addr %s", s.httpOpt.Addr))
}
router := chi.NewRouter()
router.Use(Base()...)
if s.httpOpt.Logging {
router.Use(Logging(s.log)...)
}
if s.httpOpt.EnableMetrics {
router.Use(Middleware(s.httpOpt.MetricsServiceLabel))
}
router.Group(func(r chi.Router) {
r.Use(
auth.DefaultJwtHandler.Verifier(),
auth.DefaultJwtHandler.Authenticator(),
)
for _, mountRoutes := range s.endpoints {
mountRoutes(r)
}
})
if s.httpOpt.EnableMetrics {
Mount(router, s.httpOpt.MetricsUsername, s.httpOpt.MetricsPassword)
}
if s.httpOpt.EnableDebugRoute {
Debug(router)
}
if s.httpOpt.EnableVersionRoute {
router.Get("/version", version.HttpHandler)
}
go http.Serve(listener, router)
<-ctx.Done()
return nil
}
pkg/cli/context.go
+16
View File
@@ -0,0 +1,16 @@
package cli
import (
"context"
ctxwrap "github.com/SentimensRG/ctx"
"github.com/SentimensRG/ctx/sigctx"
)
// Context is small wrapper that returns sig-term bound context
//
// This can be used as (proper) background context that properly terminates
// all subroutines.
func Context() context.Context {
return ctxwrap.AsContext(sigctx.New())
}
pkg/cli/flags/db.go
+26
View File
@@ -0,0 +1,26 @@
package flags
import (
"github.com/spf13/cobra"
)
type (
DBOpt struct {
DSN string
Profiler string
}
)
func DB(cmd *cobra.Command, pfix string) (o *DBOpt) {
o = &DBOpt{}
bindString(cmd, &o.DSN,
pFlag(pfix, "db-dsn"), "corteza:corteza@tcp(db:3306)/corteza?collation=utf8mb4_general_ci",
"DSN for database connection")
bindString(cmd, &o.Profiler,
pFlag(pfix, "db-profiler"), "none",
"Profiler for DB queries (none, stdout, logger)")
return
}
pkg/cli/flags/flags.go
+56
View File
@@ -0,0 +1,56 @@
package flags
import (
"os"
"strings"
"time"
"github.com/spf13/cast"
"github.com/spf13/cobra"
)
// Prefixes flag
func pFlag(pfix, name string) string {
if pfix != "" {
name = pfix + "-" + name
}
return name
}
// Converts input (flag-name) into ENVIRONMENTAL_VARIABLE_KEY
func envKey(s string) string {
return strings.ToUpper(strings.ReplaceAll(s, "-", "_"))
}
func bindString(cmd *cobra.Command, v *string, flag, def string, desc string) {
if env, has := os.LookupEnv(envKey(flag)); has {
def = cast.ToString(env)
}
cmd.Flags().StringVar(v, flag, def, desc)
}
func bindBool(cmd *cobra.Command, v *bool, flag string, def bool, desc string) {
if env, has := os.LookupEnv(envKey(flag)); has {
def = cast.ToBool(env)
}
cmd.Flags().BoolVar(v, flag, def, desc)
}
func bindInt(cmd *cobra.Command, v *int, flag string, def int, desc string) {
if env, has := os.LookupEnv(envKey(flag)); has {
def = cast.ToInt(env)
}
cmd.Flags().IntVar(v, flag, def, desc)
}
func bindDuration(cmd *cobra.Command, v *time.Duration, flag string, def time.Duration, desc string) {
if env, has := os.LookupEnv(envKey(flag)); has {
def = cast.ToDuration(env)
}
cmd.Flags().DurationVar(v, flag, def, desc)
}
pkg/cli/flags/http.go
+71
View File
@@ -0,0 +1,71 @@
package flags
import (
"github.com/spf13/cobra"
"github.com/cortezaproject/corteza-server/internal/rand"
)
type (
HTTPOpt struct {
Addr string
Logging bool
Pretty bool
Tracing bool
EnableVersionRoute bool
EnableDebugRoute bool
EnableMetrics bool
MetricsServiceLabel string
MetricsUsername string
MetricsPassword string
}
)
func HTTP(cmd *cobra.Command, pfix string) (o *HTTPOpt) {
o = &HTTPOpt{}
bindString(cmd, &o.Addr,
pFlag(pfix, "http-addr"), ":80",
"Listen address for HTTP server")
bindBool(cmd, &o.Logging,
pFlag(pfix, "http-log"), true,
"Enable/disable HTTP request log")
bindBool(cmd, &o.Pretty,
pFlag(pfix, "http-pretty-json"), false,
"Prettify returned JSON output")
bindBool(cmd, &o.Tracing,
pFlag(pfix, "http-error-tracing"), false,
"Return error stack frame")
bindBool(cmd, &o.EnableVersionRoute,
pFlag(pfix, "http-enable-version-route"), true,
"Enable /version route")
bindBool(cmd, &o.EnableDebugRoute,
pFlag(pfix, "http-enable-debug-route"), false,
"Enable /debug route with pprof data")
bindBool(cmd, &o.EnableMetrics,
pFlag(pfix, "http-metrics"), false,
"Enable metrics")
bindString(cmd, &o.MetricsServiceLabel,
pFlag(pfix, "http-metrics-name"), "corteza",
"Provide metrics service label for Prometheus")
bindString(cmd, &o.MetricsUsername,
pFlag(pfix, "http-metrics-username"), "metrics",
"Provide metrics username for Prometheus")
// Setting metrics password to random string to prevent security accidents...
bindString(cmd, &o.MetricsPassword,
pFlag(pfix, "http-metrics-password"), string(rand.Bytes(5)),
"Provide metrics password for Prometheus")
return
}
pkg/cli/flags/http_client.go
+28
View File
@@ -0,0 +1,28 @@
package flags
import (
"time"
"github.com/spf13/cobra"
)
type (
HttpClientOpt struct {
ClientTSLInsecure bool
HttpClientTimeout time.Duration
}
)
func HttpClient(cmd *cobra.Command) (o *HttpClientOpt) {
o = &HttpClientOpt{}
bindBool(cmd, &o.ClientTSLInsecure,
"http-client-tsl-insecure", false,
"Skip insecure TSL verification on outbound HTTP requests (allow invalid/self-signed certificates")
bindDuration(cmd, &o.HttpClientTimeout,
"http-client-timeout", 30*time.Second,
"Default HTTP client timeout")
return
}
pkg/cli/flags/jwt.go
+29
View File
@@ -0,0 +1,29 @@
package flags
import (
"github.com/spf13/cobra"
"github.com/cortezaproject/corteza-server/internal/rand"
)
type (
JWTOpt struct {
Secret string
Expiry int
}
)
func JWT(cmd *cobra.Command) (o *JWTOpt) {
o = &JWTOpt{}
// Setting JWT secret to random string to prevent security accidents...
bindString(cmd, &o.Secret,
"auth-jwt-secret", string(rand.Bytes(32)),
"JWT Secret")
bindInt(cmd, &o.Expiry,
"auth-jwt-expiry", 60*24*30,
"JWT Expiration in minutes")
return
}
pkg/cli/flags/log.go
+26
View File
@@ -0,0 +1,26 @@
package flags
import (
"github.com/spf13/cobra"
)
type (
LogOpt struct {
Level string
JSON bool
}
)
func Log(cmd *cobra.Command) (o *LogOpt) {
o = &LogOpt{}
bindString(cmd, &o.Level,
"log-level", "info",
"Log level (debug, info, warn, error, panic, fatal)")
bindBool(cmd, &o.JSON,
"log-json", true,
"Log in JSON format")
return
}
pkg/cli/flags/monitor.go
+21
View File
@@ -0,0 +1,21 @@
package flags
import (
"github.com/spf13/cobra"
)
type (
MonitorOpt struct {
Interval int
}
)
func Monitor(cmd *cobra.Command, pfix string) (o *MonitorOpt) {
o = &MonitorOpt{}
bindInt(cmd, &o.Interval,
pFlag(pfix, "monitor-interval"), 300,
"Monitor interval (seconds, 0 = disable)")
return
}
pkg/cli/flags/provision.go
+21
View File
@@ -0,0 +1,21 @@
package flags
import (
"github.com/spf13/cobra"
)
type (
ProvisionOpt struct {
Database bool
}
)
func Provision(cmd *cobra.Command, pfix string) (o *ProvisionOpt) {
o = &ProvisionOpt{}
bindBool(cmd, &o.Database,
pFlag(pfix, "provision-database"), true,
"Run database migration scripts")
return
}
pkg/cli/flags/pubsub.go
+58
View File
@@ -0,0 +1,58 @@
package flags
import (
"time"
"github.com/spf13/cobra"
)
type (
PubSubOpt struct {
Mode string
// Mode
PollingInterval time.Duration
// Redis
RedisAddr string
RedisTimeout time.Duration
RedisPingTimeout time.Duration
RedisPingPeriod time.Duration
}
)
func PubSub(cmd *cobra.Command, pfix string) (o *PubSubOpt) {
o = &PubSubOpt{}
const (
timeout = 15 * time.Second
pingTimeout = 120 * time.Second
pingPeriod = (pingTimeout * 9) / 10
)
bindString(cmd, &o.Mode,
pFlag(pfix, "pubsub-mode"), "poll",
"Pub/Sub mode (poll, redis")
bindDuration(cmd, &o.RedisPingTimeout,
pFlag(pfix, "pubsub-polling-interval"), timeout,
"Sub/Sub polling interval")
bindString(cmd, &o.RedisAddr,
pFlag(pfix, "pubsub-redis-addr"), "redis:6379",
"Pub/Sub mode (poll, redis")
bindDuration(cmd, &o.RedisTimeout,
pFlag(pfix, "pubsub-redis-timeout"), timeout,
"Websocket connection timeout")
bindDuration(cmd, &o.RedisPingTimeout,
pFlag(pfix, "pubsub-redis-ping-timeout"), pingTimeout,
"Pub/Sub connection ping timeout")
bindDuration(cmd, &o.RedisPingPeriod,
pFlag(pfix, "pubsub-redis-ping-period"), pingPeriod,
"Pub/Sub connection ping period (should be lower than timeout)")
return
}
pkg/cli/flags/smtp.go
+41
View File
@@ -0,0 +1,41 @@
package flags
import (
"github.com/spf13/cobra"
)
type (
SMTPOpt struct {
Host string
Port int
User string
Pass string
From string
}
)
func SMTP(cmd *cobra.Command) (o *SMTPOpt) {
o = &SMTPOpt{}
bindString(cmd, &o.Host,
"smtp-host", "localhost:25",
"SMTP hostname")
bindString(cmd, &o.User,
"smtp-username", "",
"SMTP server username")
bindString(cmd, &o.Pass,
"smtp-pass", "",
"SMTP server password")
bindString(cmd, &o.From,
"smtp-from", "",
"Sender's email address")
bindInt(cmd, &o.Port,
"smtp-port", 25,
"SMTP port number")
return
}
pkg/cli/flags/websocket.go
+39
View File
@@ -0,0 +1,39 @@
package flags
import (
"time"
"github.com/spf13/cobra"
)
type (
WebsocketOpt struct {
Timeout time.Duration
PingTimeout time.Duration
PingPeriod time.Duration
}
)
func Websocket(cmd *cobra.Command, pfix string) (o *WebsocketOpt) {
o = &WebsocketOpt{}
const (
timeout = 15 * time.Second
pingTimeout = 120 * time.Second
pingPeriod = (pingTimeout * 9) / 10
)
bindDuration(cmd, &o.Timeout,
pFlag(pfix, "websocket-timeout"), timeout,
"Websocket connection timeout")
bindDuration(cmd, &o.PingTimeout,
pFlag(pfix, "websocket-ping-timeout"), pingTimeout,
"Websocket connection ping timeout")
bindDuration(cmd, &o.PingPeriod,
pFlag(pfix, "websocket-ping-period"), pingPeriod,
"Websocket connection ping period (should be lower than timeout)")
return
}
pkg/cli/helpers.go
+107
View File
@@ -0,0 +1,107 @@
package cli
import (
"context"
"github.com/spf13/cobra"
"go.uber.org/zap"
"github.com/cortezaproject/corteza-server/internal/auth"
"github.com/cortezaproject/corteza-server/internal/http"
"github.com/cortezaproject/corteza-server/internal/logger"
"github.com/cortezaproject/corteza-server/internal/mail"
"github.com/cortezaproject/corteza-server/pkg/cli/flags"
)
// SetupProvisionCommands sets-up standard provision commands
// Deprecated: use SetupProvisionSubCommands
func SetupProvisionCommands(ac func() error, md func() error) *cobra.Command {
var (
cmd = &cobra.Command{
Use: "provision",
Short: "Provision tasks",
}
)
// Add only commands with defined callbacks
if ac != nil {
cmd.AddCommand(&cobra.Command{
Use: "access-control-rules",
Short: "Reset access control rules & roles",
RunE: func(cmd *cobra.Command, args []string) error {
return ac()
},
})
}
// Add only commands with defined callbacks
if md != nil {
cmd.AddCommand(&cobra.Command{
Use: "migrate-database",
Short: "Run database migration scripts",
RunE: func(cmd *cobra.Command, args []string) error {
return md()
},
})
}
return cmd
}
type (
provisioner interface {
ProvisionMigrateDatabase(ctx context.Context) error
ProvisionAccessControl(ctx context.Context) error
}
)
func SetupProvisionSubcommands(ctx context.Context, p provisioner) *cobra.Command {
var (
cmd = &cobra.Command{
Use: "provision",
Short: "Provision tasks",
}
)
// Add only commands with defined callbacks
cmd.AddCommand(&cobra.Command{
Use: "access-control-rules",
Short: "Reset access control rules & roles",
RunE: func(cmd *cobra.Command, args []string) error {
return p.ProvisionAccessControl(ctx)
},
})
// Add only commands with defined callbacks
cmd.AddCommand(&cobra.Command{
Use: "migrate-database",
Short: "Run database migration scripts",
RunE: func(cmd *cobra.Command, args []string) error {
return p.ProvisionMigrateDatabase(ctx)
},
})
return cmd
}
func InitGeneralServices(logOpt *flags.LogOpt, smtpOpt *flags.SMTPOpt, jwtOpt *flags.JWTOpt, httpClientOpt *flags.HttpClientOpt) {
var logLevel = zap.InfoLevel
_ = logLevel.Set(logOpt.Level)
if logger.Default() == nil {
logger.Init(logLevel)
} else {
logger.DefaultLevel.SetLevel(logLevel)
}
auth.SetupDefault(jwtOpt.Secret, jwtOpt.Expiry)
mail.SetupDialer(smtpOpt.Host, smtpOpt.Port, smtpOpt.User, smtpOpt.Pass, smtpOpt.From)
http.SetupDefaults(
httpClientOpt.HttpClientTimeout,
httpClientOpt.ClientTSLInsecure,
)
}