diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..445b8dd84 --- /dev/null +++ b/.env.example @@ -0,0 +1,183 @@ +# When running as monolith/bundle +# Note: default docker image without any extra command will +# force :80 via flag and override anything you set here +HTTP_ADDR=:80 + +# SMTP configuration +# For dev environment, run mailhog (`make mailhog.up`), set SMTP_HOST to `localhost:1025` +# and visit localhost:8025. +SMTP_HOST=localhost:1025 +SMTP_USER= +SMTP_PASS= +SMTP_FROM="Corteza" + +# JWT Secret, shared among all services. +# If not set, some random value will be set +AUTH_JWT_SECRET= + +# JWT expiration +AUTH_JWT_EXPIRY= + +# Debug level you want to use (anything equal or lower than that will be logged) +# Values: debug, info, warn, error, panic, fatal +LOG_LEVEL=info + +# Output log in JSON format +# Setting this to true will enable yoi to ship docker container logs directly to your ELK stack +LOG_JSON=false + +# Timeout for internal HTTP client +HTTP_CLIENT_TIMEOUT=30s + +# Allow usage of insecure hosts (self-signed, expired certificates) +HTTP_CLIENT_TSL_INSECURE=false + +######################################################################################################################## +# When running separate services (microservice-like architecture) +# +# Prefixed ENV variables allow you to keep the same .env file for all services + + +#----------------------------------------------------------------------------------------------------------------------- +# System + +# Database to use +SYSTEM_DB_DSN=corteza:corteza@tcp(localhost:3306)/corteza?collation=utf8mb4_general_ci + +# Database query profiler +SYSTEM_DB_PROFILER=stdout + +# Enable provisioning on startup: migrate database +SYSTEM_PROVISION_MIGRATE_DATABASE=true + +# Enable provisioning on startup: auto-setup +SYSTEM_PROVISION_AUTO_SETUP=true + +# System specific settings: +SYSTEM_HTTP_ADDR=:80 + +# Enable /debug route +SYSTEM_HTTP_ENABLE_DEBUG_ROUTE=true + +# Enable /version route +SYSTEM_HTTP_ENABLE_VERSION_ROUTE=true + +# Add stack trace on error to HTTP response +SYSTEM_HTTP_ENABLE_ERROR_TRACKING=true + +# Log HTTP requests +SYSTEM_HTTP_LOG=true + +# Serve prometheus metrics under /metrics +SYSTEM_HTTP_METRICS=true + +# Metrics service label +SYSTEM_HTTP_METRICS_NAME=corteza + +# Metrics access username +SYSTEM_HTTP_METRICS_USERNAME=metrics + +# Metrics access password +SYSTEM_HTTP_METRICS_PASSWORD=metrics + +# Log interval for monitoring +SYSTEM_MONITOR_INTERVAL=5min + +#----------------------------------------------------------------------------------------------------------------------- +# Messaging + +# Database to use +SYSTEM_DB_DSN=corteza:corteza@tcp(localhost:3306)/corteza?collation=utf8mb4_general_ci + +# Database query profiler +SYSTEM_DB_PROFILER=stdout + +# Enable provisioning on startup: migrate database +SYSTEM_PROVISION_MIGRATE_DATABASE=true + +# Enable provisioning on startup: auto-setup +SYSTEM_PROVISION_AUTO_SETUP=true + +# System specific settings: +SYSTEM_HTTP_ADDR=:80 + +# Enable /debug route +SYSTEM_HTTP_ENABLE_DEBUG_ROUTE=true + +# Enable /version route +SYSTEM_HTTP_ENABLE_VERSION_ROUTE=true + +# Add stack trace on error to HTTP response +SYSTEM_HTTP_ENABLE_ERROR_TRACKING=true + +# Log HTTP requests +SYSTEM_HTTP_LOG=true + +# Serve prometheus metrics under /metrics +SYSTEM_HTTP_METRICS=true + +# Metrics service label +SYSTEM_HTTP_METRICS_NAME=corteza + +# Metrics access username +SYSTEM_HTTP_METRICS_USERNAME=metrics + +# Metrics access password +SYSTEM_HTTP_METRICS_PASSWORD=metrics + +# Log interval for monitoring +SYSTEM_MONITOR_INTERVAL=5min + +# Websocket connection ping period +# MESSAGING_WEBSOCKET_PING_PERIOD= + +# Websocket connection ping timeout +# MESSAGING_WEBSOCKET_PING_TIMEOUT= + +# Websocket connection timeout +# MESSAGING_WEBSOCKET_TIMEOUT= + +#----------------------------------------------------------------------------------------------------------------------- +# System + +# Database to use +SYSTEM_DB_DSN=corteza:corteza@tcp(localhost:3306)/corteza?collation=utf8mb4_general_ci + +# Database query profiler +SYSTEM_DB_PROFILER=stdout + +# Enable provisioning on startup: migrate database +SYSTEM_PROVISION_MIGRATE_DATABASE=true + +# Enable provisioning on startup: auto-setup +SYSTEM_PROVISION_AUTO_SETUP=true + +# System specific settings: +SYSTEM_HTTP_ADDR=:80 + +# Enable /debug route +SYSTEM_HTTP_ENABLE_DEBUG_ROUTE=true + +# Enable /version route +SYSTEM_HTTP_ENABLE_VERSION_ROUTE=true + +# Add stack trace on error to HTTP response +SYSTEM_HTTP_ENABLE_ERROR_TRACKING=true + +# Log HTTP requests +SYSTEM_HTTP_LOG=true + +# Serve prometheus metrics under /metrics +SYSTEM_HTTP_METRICS=true + +# Metrics service label +SYSTEM_HTTP_METRICS_NAME=corteza + +# Metrics access username +SYSTEM_HTTP_METRICS_USERNAME=metrics + +# Metrics access password +SYSTEM_HTTP_METRICS_PASSWORD=metrics + +# Log interval for monitoring +SYSTEM_MONITOR_INTERVAL=5min diff --git a/.env.sample b/.env.sample deleted file mode 100644 index 4ab77182e..000000000 --- a/.env.sample +++ /dev/null @@ -1,20 +0,0 @@ -MESSAGING_HTTP_ADDR=:3000 -MESSAGING_DB_DSN=corteza:corteza@tcp(localhost:3306)/corteza?collation=utf8mb4_general_ci -MESSAGING_DB_PROFILER=stdout - -COMPOSE_HTTP_ADDR=:3001 -COMPOSE_DB_DSN=corteza:corteza@tcp(localhost:3306)/corteza?collation=utf8mb4_general_ci -COMPOSE_DB_PROFILER=stdout - -SYSTEM_HTTP_ADDR=:3002 -SYSTEM_DB_DSN=corteza:corteza@tcp(localhost:3306)/corteza?collation=utf8mb4_general_ci -SYSTEM_DB_PROFILER=stdout - -AUTH_JWT_SECRET=jwt-secret -AUTH_JWT_DEBUG=1 - -# Make sure you run `make mailhog.up` and visit localhost:8025. -SMTP_HOST=localhost:1025 -SMTP_USER= -SMTP_PASS= -SMTP_FROM="Corteza" diff --git a/Dockerfile.corteza-server b/Dockerfile.corteza-server index 279ba1e0c..0af33dc1c 100644 --- a/Dockerfile.corteza-server +++ b/Dockerfile.corteza-server @@ -17,4 +17,6 @@ COPY --from=builder /tmp/corteza-server /bin EXPOSE 80 ENTRYPOINT ["/bin/corteza-server"] -CMD ["serve-api"] + +# Forcing exposed port +CMD ["serve-api", "--http-addr=:80"] diff --git a/Dockerfile.corteza-server-compose b/Dockerfile.corteza-server-compose index 2a0cbb855..6d90d6db3 100644 --- a/Dockerfile.corteza-server-compose +++ b/Dockerfile.corteza-server-compose @@ -17,4 +17,7 @@ COPY --from=builder /tmp/corteza-server-compose /bin EXPOSE 80 ENTRYPOINT ["/bin/corteza-server-compose"] -CMD ["serve-api"] + +# Forcing exposed port +CMD ["serve-api", "--compose-http-addr=:80"] + diff --git a/Dockerfile.corteza-server-messaging b/Dockerfile.corteza-server-messaging index e9cc0d631..ba2d7dbc8 100644 --- a/Dockerfile.corteza-server-messaging +++ b/Dockerfile.corteza-server-messaging @@ -17,4 +17,7 @@ COPY --from=builder /tmp/corteza-server-messaging /bin EXPOSE 80 ENTRYPOINT ["/bin/corteza-server-messaging"] -CMD ["serve-api"] + +# Forcing exposed port +CMD ["serve-api", "--messaging-http-addr=:80"] + diff --git a/Dockerfile.corteza-server-system b/Dockerfile.corteza-server-system index 4cf6c12f0..e4a1a099c 100644 --- a/Dockerfile.corteza-server-system +++ b/Dockerfile.corteza-server-system @@ -17,4 +17,7 @@ COPY --from=builder /tmp/corteza-server-system /bin EXPOSE 80 ENTRYPOINT ["/bin/corteza-server-system"] -CMD ["serve-api"] + +# Forcing exposed port +CMD ["serve-api", "--system-http-addr=:80"] + diff --git a/docs/CLI.md b/docs/CLI.md new file mode 100644 index 000000000..58b930aa7 --- /dev/null +++ b/docs/CLI.md @@ -0,0 +1,80 @@ +# Corteza Command Line Interface + +## General + +You can choose from one of the four (4) app/service CLI entrypoints: + + - system (`corteza-server-system`) + - messaging (`corteza-server-messaging`) + - compose (`corteza-server-compose`) + - all-in-one monolith (`corteza-server`) + +When system is build into a all-in-one binary or image, +app-specific commands can be accessed under a sub-command with the +same name as the app. + +## Flags & environmental variables +Command and application behaviour can be adjusted using flags and/or environmental +variables. Flags have highest priority and override env. variables. Defaults that +are pre-set internally are optimized for production mode. + +Note that defaults shown under `--help` are overridden by value from the accompanying +environemntal variable. + +## How to run: + +### Compile source files and run with `go run` + +Source entrypoints can be found under `cmd/`: + +```sh +go run cmd/system/*.go [commands and flags] +``` + +### Docker or Docker Compose + +Docker images are built with handy presets for +`ENTRYPOINT` and `CMD`. By default, `CMD` is set to `serve-api`. + +See docker manual for more info. + +```sh +docker exec -it -rm corteza-server-system [commands and flags] +``` + +## Base commands: + +These commands are accessible from the base level (never +wrapped under an app-sub-command) and always affect all apps. + +### `provision` + +Command `provision` Wraps 2 sub-commands: + - `access-control-rules` + Reset access control rules for Everyone and Administrator roles. + - `migrate-database` + Run database migration scripts + +Both these commands are executed when you run `serve-api` command, +after database is connected and before initialization. This behaviour can +be changed with env variables. + +## System specific commands & sub-commands + +| Command | Subcommand | | +| --- | --- | --- | +| auth | auto-discovery | Auto discovers new OIDC client +| auth | jwt | Generates new JWT for a user +| auth | test-notifications | Sends samples of all authentication notification to receipient +| roles | useradd | Add user to role +| settings | auto-configure | Run autoconfiguration +| settings | delete | Set value (raw JSON) for a specific key +| settings | get | Get value (raw JSON) for a specific key +| settings | import | Import settings as JSON from stdin or file +| settings | list | List all +| settings | set | Set value (raw JSON) for a specific key +| users | add | Add new user +| users | list | List users +| users | password | Change user's password + +See help (`--help`) under each (sub) command for details about additional flags and arguments. diff --git a/docs/Authentication.md b/docs/ExternalAuth.md similarity index 86% rename from docs/Authentication.md rename to docs/ExternalAuth.md index 47ad0df6a..1710b8369 100644 --- a/docs/Authentication.md +++ b/docs/ExternalAuth.md @@ -1,5 +1,7 @@ # Authentication +If you are not already familiar with it, please read documentation about [Corteza Command Line Interface](CLI.md). + Corteza support a fixed set of standard OAuth 2 authentication providers (facebook, gplus, github and linkedin) and a arbitrary number of custom issuers (over OpenID Connect). @@ -12,9 +14,7 @@ Settings for external providers are stored under keys Prop is one of: `key`, `secret`, `enabled`. OIDC settings also have `issuer` prop. -Example settings (`system settings list --prefix=auth.external`): - - +Example settings (`settings list --prefix=auth.external`): ``` auth.external.callback-endpoint "https://your-corteza-system-api-backend/auth/external/%s/callback" auth.external.enabled true @@ -60,11 +60,11 @@ external authentication provider "openid-connect.corteza-iam" added Corteza CLI comes with auto-discovery tool: ```bash -system external-auth auto-discovery name url +external-auth auto-discovery name url ``` ```bash -system external-auth auto-discovery corteza-iam https://satosa.didmos.crust.example.tld +external-auth auto-discovery corteza-iam https://satosa.didmos.crust.example.tld ``` This will autodiscover and autoconfigure new OIDC provider. @@ -74,5 +74,7 @@ Please note that this provider is disabled by default. To enable it, run: ```bash -system settings key auth.external.providers.openid-connect.corteza-iam.enabled true +settings key auth.external.providers.openid-connect.corteza-iam.enabled true ``` + +Changing values requires system service restart. diff --git a/docs/Setup.md b/docs/Setup.md new file mode 100644 index 000000000..115fb2896 --- /dev/null +++ b/docs/Setup.md @@ -0,0 +1,50 @@ +# Corteza Setup + +If you are not already familiar with it, please read documentation about [Corteza Command Line Interface](CLI.md). + +## First steps + +Corteza will pre-initialize itself to allow you to access all of it's features as quickly as possible. +Internal authentication enabled, sign-up without email confirmation (in case you do not have your SMTP configured just +yet)... + +## Configuring system + +### Configuring authentication + +Review your current (auto-configure) settings with `settings list`: + +``` +auth.external.enabled false +auth.external.redirect-url "http://system.api.local.crust.tech/auth/external/%s/callback" +auth.external.session-store-secret "PBVta4xKfQ0LIQEOtycxXqZZrGbZdTCuF4hw1cxrly1YA2AY5uO8a0SyY4Tbd1bk" +auth.external.session-store-secure false +auth.internal.enabled true +auth.internal.password-reset.enabled true +auth.internal.signup-email-confirmation-required false +auth.internal.signup.enabled true +auth.mail.from-address "change-me@example.tld" +auth.mail.from-name "Corteza Team" +``` + +| Key | Description | +| ---- | ---- | +| auth.external.enabled | Enable external authentication, see [ExternalAuth.md](ExternalAuth.md) for details +| auth.external.redirect-url | Where to redirect after successful external authentication. This is the URL that you usually need to insert into your provider's auth app configuration page +| auth.external.session-store-secret | Keep session values secret +| auth.external.session-store-secure | Secure sessino store (set to false if not using TLS/HTTPS) +| auth.internal.enabled | Enable/disable internal authentication (will users be able to use Corteza username and password to login) +| auth.internal.password-reset.enabled | Enable password reset +| auth.internal.signup-email-confirmation-required | Is email confirmation required on sign-up. +| auth.internal.signup.enabled | Is sign-up enabled. +| auth.mail.from-address | Who (email) is sending auth emails (password reset, email confirmation) +| auth.mail.from-name | Who (name) is sending auth emails (password reset, email confirmation) + +## Configuring messaging + +_To be implemented_ + + +## Configuring corteza + +_To be implemented_ diff --git a/pkg/api/server.go b/pkg/api/server.go index fe69d69be..3f3794a85 100644 --- a/pkg/api/server.go +++ b/pkg/api/server.go @@ -74,8 +74,7 @@ func (s Server) Serve(ctx context.Context) { // configure resputil options resputil.SetConfig(resputil.Options{ - Pretty: s.httpOpt.Pretty, - Trace: s.httpOpt.Tracing, + Trace: s.httpOpt.Tracing, Logger: func(err error) { // @todo: error logging }, diff --git a/pkg/cli/flags/http.go b/pkg/cli/flags/http.go index a994db4db..6cffaae8f 100644 --- a/pkg/cli/flags/http.go +++ b/pkg/cli/flags/http.go @@ -10,7 +10,6 @@ type ( HTTPOpt struct { Addr string Logging bool - Pretty bool Tracing bool EnableVersionRoute bool diff --git a/pkg/cli/flags/jwt.go b/pkg/cli/flags/jwt.go index 74dbea86d..37dea3637 100644 --- a/pkg/cli/flags/jwt.go +++ b/pkg/cli/flags/jwt.go @@ -1,6 +1,8 @@ package flags import ( + "time" + "github.com/spf13/cobra" "github.com/cortezaproject/corteza-server/internal/rand" @@ -9,7 +11,7 @@ import ( type ( JWTOpt struct { Secret string - Expiry int + Expiry time.Duration } ) @@ -21,9 +23,9 @@ func JWT(cmd *cobra.Command) (o *JWTOpt) { "auth-jwt-secret", string(rand.Bytes(32)), "JWT Secret") - BindInt(cmd, &o.Expiry, - "auth-jwt-expiry", 60*24*30, - "JWT Expiration in minutes") + BindDuration(cmd, &o.Expiry, + "auth-jwt-expiry", time.Hour*24*30, + "JWT Expiration") return } diff --git a/pkg/cli/helpers.go b/pkg/cli/helpers.go index e9fe589e9..688675f31 100644 --- a/pkg/cli/helpers.go +++ b/pkg/cli/helpers.go @@ -3,6 +3,7 @@ package cli import ( "fmt" "os" + "time" "go.uber.org/zap" @@ -19,7 +20,7 @@ func InitGeneralServices(logOpt *flags.LogOpt, smtpOpt *flags.SMTPOpt, jwtOpt *f _ = logLevel.Set(logOpt.Level) logger.DefaultLevel.SetLevel(logLevel) - auth.SetupDefault(jwtOpt.Secret, jwtOpt.Expiry) + auth.SetupDefault(jwtOpt.Secret, int(jwtOpt.Expiry/time.Minute)) mail.SetupDialer(smtpOpt.Host, smtpOpt.Port, smtpOpt.User, smtpOpt.Pass, smtpOpt.From) http.SetupDefaults( httpClientOpt.HttpClientTimeout, diff --git a/system/commands/users.go b/system/commands/users.go index 2cf46a9bb..ae1c3f445 100644 --- a/system/commands/users.go +++ b/system/commands/users.go @@ -102,7 +102,7 @@ func Users(ctx context.Context) *cobra.Command { pwdCmd := &cobra.Command{ Use: "password [email]", - Short: "Add new user", + Short: "Change password for user", Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { var (