DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Refactor building and packing pipeline

Browse Source
This commit is contained in:
Denis Arh
2020-03-13 20:59:24 +01:00
parent 0c95128bdc
commit 612312c4ca
9 changed files with 97 additions and 248 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
.drone.yml
View File

@@ -1,88 +1,49 @@
workspace:
base: /go/src
path: github.com/cortezaproject/corteza-server
kind: pipeline
name: docker image build
name: Build
type: docker
steps:
- name: unit-tests
- name: "Quality Control"
image: cortezaproject/corteza-server-builder:latest
pull: always
environment:
CGO_ENABLED: 0
GOOS: linux
GOARCH: amd64
CI: circleci
commands:
- make test.unit
- name: corteza-server
image: plugins/docker
depends_on: [ "unit-tests" ]
settings:
repo: cortezaproject/corteza-server
auto_tag: true # generate tag names automatically based on git branch and git tag
dockerfile: Dockerfile.corteza-server
username: { from_secret: docker_hub_username }
password: { from_secret: docker_hub_password }
- name: corteza-server-system
image: plugins/docker
depends_on: [ "corteza-server" ]
settings:
repo: cortezaproject/corteza-server-system
auto_tag: true # generate tag names automatically based on git branch and git tag
dockerfile: Dockerfile.corteza-server-system
username: { from_secret: docker_hub_username }
password: { from_secret: docker_hub_password }
- name: "Build release (tag)"
image: cortezaproject/corteza-server-builder:latest
environment:
RELEASE_SFTP_KEY: { from_secret: RELEASE_SFTP_KEY }
RELEASE_SFTP_URI: { from_secret: RELEASE_SFTP_URI }
commands:
- make release-clean release -j4 BUILD_OS=linux BUILDARCH=amd64 BUILD_VERSION=${DRONE_TAG}
# - make release-clean release -j4 BUILD_OS=darwin BUILDARCH=amd64 BUILD_VERSION=${DRONE_TAG}
# - make release-clean release -j4 BUILD_OS=windows BUILDARCH=amd64 BUILD_VERSION=${DRONE_TAG}
- make upload
when:
ref: { include: [ "refs/tags/20??.*.*", "refs/heads/master" ] }
event: [ "push", "tag" ]
event: [ tag ]
ref:
- refs/tags/20??.3.*
- refs/tags/20??.6.*
- refs/tags/20??.9.*
- refs/tags/20??.12.*
- name: corteza-server-compose
image: plugins/docker
depends_on: [ "corteza-server" ]
settings:
repo: cortezaproject/corteza-server-compose
auto_tag: true # generate tag names automatically based on git branch and git tag
dockerfile: Dockerfile.corteza-server-compose
username: { from_secret: docker_hub_username }
password: { from_secret: docker_hub_password }
- name: "Build unstable release (develop)"
image: cortezaproject/corteza-server-builder:latest
environment:
RELEASE_SFTP_KEY: { from_secret: RELEASE_SFTP_KEY }
RELEASE_SFTP_URI: { from_secret: RELEASE_SFTP_URI }
commands:
- make release-clean release -j4 BUILD_OS=linux BUILDARCH=amd64 BUILD_VERSION=unstable
- make upload
when:
ref: { include: [ "refs/tags/20??.*.*", "refs/heads/master" ] }
event: [ "push", "tag" ]
- name: corteza-server-messaging
depends_on: [ "corteza-server" ]
image: plugins/docker
settings:
repo: cortezaproject/corteza-server-messaging
auto_tag: true # generate tag names automatically based on git branch and git tag
dockerfile: Dockerfile.corteza-server-messaging
username: { from_secret: docker_hub_username }
password: { from_secret: docker_hub_password }
when:
ref: { include: [ "refs/tags/20??.*.*", "refs/heads/master" ] }
event: [ "push", "tag" ]
trigger:
event:
# Pipeline will be executed from branch event,
# this ensures we do not run duplicates.
exclude: [ "pull_request" ]
event: [ push ]
branch: [ develop ]
---
workspace:
base: /go/src
path: github.com/cortezaproject/corteza-server
kind: pipeline
name: integration
name: Integration
type: docker
steps:
- name: test
@@ -125,7 +86,4 @@ services:
trigger:
event:
# Pipeline will be executed from branch event,
# this ensures we do not run duplicates.
exclude: [ "pull_request" ]
exclude: [ pull_request ]

1
.project
View File

@@ -1 +0,0 @@
cortezaproject/corteza

26
Dockerfile.corteza-server
View File

@@ -1,26 +0,0 @@
## builder image
FROM cortezaproject/corteza-server-builder:latest AS builder
WORKDIR /cortrza/src
COPY . .
RUN scripts/builder-make-bin.sh monolith /tmp/corteza-server
## == target image ==
FROM alpine:3.7
RUN apk add --no-cache ca-certificates
COPY --from=builder /tmp/corteza-server /bin
ENV STORAGE_PATH /data
ENV CORREDOR_ADDR corredor:80
VOLUME /data
EXPOSE 80
ENTRYPOINT ["/bin/corteza-server"]
CMD ["serve-api"]

27
Dockerfile.corteza-server-compose
View File

@@ -1,27 +0,0 @@
## builder image
FROM cortezaproject/corteza-server-builder:latest AS builder
WORKDIR /corteza/src
COPY . .
RUN scripts/builder-make-bin.sh compose /tmp/corteza-server-compose
## == target image ==
FROM alpine:3.7
RUN apk add --no-cache ca-certificates
COPY --from=builder /tmp/corteza-server-compose /bin
ENV STORAGE_PATH /data
ENV CORREDOR_ADDR corredor:80
VOLUME /data
EXPOSE 80
ENTRYPOINT ["/bin/corteza-server-compose"]
CMD ["serve-api"]

27
Dockerfile.corteza-server-messaging
View File

@@ -1,27 +0,0 @@
## builder image
FROM cortezaproject/corteza-server-builder:latest AS builder
WORKDIR /corteza/src
COPY . .
RUN scripts/builder-make-bin.sh messaging /tmp/corteza-server-messaging
## == target image ==
FROM alpine:3.7
RUN apk add --no-cache ca-certificates
COPY --from=builder /tmp/corteza-server-messaging /bin
ENV STORAGE_PATH /data
ENV CORREDOR_ADDR corredor:80
VOLUME /data
EXPOSE 80
ENTRYPOINT ["/bin/corteza-server-messaging"]
CMD ["serve-api"]

28
Dockerfile.corteza-server-system
View File

@@ -1,28 +0,0 @@
## builder image
FROM cortezaproject/corteza-server-builder:latest AS builder
WORKDIR /corteza/src
COPY . .
RUN scripts/builder-make-bin.sh system /tmp/corteza-server-system
## == target image ==
FROM alpine:3.7
RUN apk add --no-cache ca-certificates
COPY --from=builder /tmp/corteza-server-system /bin
ENV STORAGE_PATH /data
ENV CORREDOR_ADDR corredor:80
VOLUME /data
EXPOSE 80
ENTRYPOINT ["/bin/corteza-server-system"]
CMD ["serve-api"]

90
Makefile
View File

@@ -1,12 +1,34 @@
.PHONY: help docker docker-push realize qa critic vet codegen integration
.PHONY: pack build help realize qa critic vet codegen integration
GO = go
GOGET = $(GO) get -u
GO = go
GOGET = $(GO) get -u
GOTEST ?= go test
GOFLAGS ?= -mod=vendor
export GOFLAGS
BUILD_FLAVOUR ?= corteza
BUILD_APPS ?= system compose messaging monolith
BUILD_TIME ?= $(shell date +%FT%T%z)
BUILD_VERSION ?= $(shell git describe --tags --abbrev=0)
BUILD_ARCH ?= $(shell go env GOARCH)
BUILD_OS ?= $(shell go env GOOS)
BUILD_OS_is_windows = $(filter windows,$(BUILD_OS))
BUILD_DEST_DIR ?= build
BUILD_NAME = $(BUILD_FLAVOUR)-server-$*-$(BUILD_VERSION)-$(BUILD_OS)-$(BUILD_ARCH)
BUILD_BIN_NAME = $(BUILD_NAME)$(if $(BUILD_OS_is_windows),.exe,)
RELEASE_BASEDIR = $(BUILD_DEST_DIR)/pkg/$(BUILD_FLAVOUR)-server-$*
RELEASE_NAME = $(BUILD_NAME).tar.gz
RELEASE_EXTRA_FILES ?= README.md LICENSE CONTRIBUTING.md DCO .env.example
RELEASE_PKEY ?= .upload-rsa
LDFLAGS_BUILD_TIME = -X github.com/cortezaproject/corteza-server/pkg/version.BuildTime=$(BUILD_TIME)
LDFLAGS_VERSION = -X github.com/cortezaproject/corteza-server/pkg/version.Version=$(BUILD_VERSION)
LDFLAGS_EXTRA ?=
LDFLAGS = -ldflags "$(LDFLAGS_BUILD_TIME) $(LDFLAGS_GIT_TAG) $(LDFLAGS_EXTRA)"
BASEPKGS = system compose messaging
IMAGES = corteza-server-system corteza-server-compose corteza-server-messaging corteza-server
TESTABLE = messaging system compose pkg internal
# Run watcher with a different event-trigger delay, eg:
# $> WATCH_DELAY=5s make watch.test.integration
@@ -35,8 +57,12 @@ TEST_SUITE_unit = $(TEST_SUITE_pkg) $(TEST_SUITE_services)
TEST_SUITE_integration = ./tests/...
TEST_SUITE_all = $(TEST_SUITE_unit) $(TEST_SUITE_integration)
DEV_MINIO_PORT ?= 9000
# Dev Support apps settings
DEV_MINIO_PORT ?= 9000
DEV_MAILHOG_SMTP_ADDR ?= 1025
DEV_MAILHOG_HTTP_ADDR ?= 8025
DOCKER ?= docker
########################################################################################################################
# Tool bins
@@ -55,29 +81,43 @@ help:
@echo
@echo Usage: make [target]
@echo
@echo - docker-images: builds docker images locally
@echo - docker-push: push built images
@echo - build build all apps
@echo - build.<app> build a specific app
@echo - vet run go vet on all code
@echo - critic run go critic on all code
@echo - test.all run all tests
@echo - test.unit run all unit tests
@echo - test.integration run all integration tests
@echo
@echo - vet - run go vet on all code
@echo - critic - run go critic on all code
@echo - test.all - run all tests
@echo - test.unit - run all unit tests
@echo - test.integration - run all integration tests
@echo
@echo See tests/README.md for more info
@echo See tests/README.md for more info on running tests
@echo
########################################################################################################################
# Building & packing
docker-images: $(IMAGES:%=docker-image.%)
build: $(addprefix build., $(BUILD_APPS))
docker-image.%: Dockerfile.%
@ docker build --no-cache --rm -f Dockerfile.$* -t cortezaproject/$*:latest .
build.%: cmd/%
GOOS=$(BUILD_OS) GOARCH=$(BUILD_ARCH) go build $(LDFLAGS) -o $(BUILD_DEST_DIR)/$(BUILD_BIN_NAME) cmd/$*/main.go
docker-push: $(IMAGES:%=docker-push.%)
release.%: $(addprefix build., %)
@ mkdir -p $(RELEASE_BASEDIR) $(RELEASE_BASEDIR)/bin
@ cp $(RELEASE_EXTRA_FILES) $(RELEASE_BASEDIR)/
@ cp $(BUILD_DEST_DIR)/$(BUILD_BIN_NAME) $(RELEASE_BASEDIR)/bin/$(BUILD_FLAVOUR)-server-$*
@ tar -C $(dir $(RELEASE_BASEDIR)) -czf $(BUILD_DEST_DIR)/$(RELEASE_NAME) $(notdir $(RELEASE_BASEDIR))
docker-push.%: Dockerfile.%
@ docker push cortezaproject/$*:latest
release: $(addprefix release.,$(BUILD_APPS))
release-clean:
@ rm -rf $(RELEASE_BASEDIR)
upload: $(RELEASE_PKEY)
@ echo "put $(BUILD_DEST_DIR)/*.tar.gz" | sftp -q -i $(RELEASE_PKEY) $(RELEASE_SFTP_URI)
@ rm -f $(RELEASE_PKEY)
$(RELEASE_PKEY):
@ echo $(RELEASE_SFTP_KEY) | base64 -d > $(RELEASE_PKEY)
@ chmod 0400 $@
########################################################################################################################
# Development
@@ -89,12 +129,12 @@ codegen: $(PROTOGEN)
./codegen.sh
mailhog.up:
docker run --rm --publish 8025:8025 --publish 1025:1025 mailhog/mailhog
$(DOCKER) run --rm --publish $(DEV_MAILHOG_SMTP_ADDR):8025 --publish $(DEV_MAILHOG_HTTP_ADDR):1025 mailhog/mailhog
minio.up:
# Runs temp minio server
# No volume because we do not want the data to persist
docker run --rm --publish 9000:$(DEV_MINIO_PORT) --env-file .env minio/minio server /data
# No volume mounts because we do not want the data to persist
$(DOCKER) run --rm --publish $(DEV_MINIO_PORT):9000 --env-file .env minio/minio server /data
watch.test.%: $(NODEMON)
# Development helper - watches for file

23
build.sh
View File

@@ -1,23 +0,0 @@
#!/bin/bash
set -e
PROJECT=$(<.project)
NAMES=$(find cmd/* -type d | xargs -n1 basename)
if [ ! -z "$1" ]; then
NAMES="$1"
fi
for NAME in $NAMES; do
OSES=${OSS:-"linux"}
ARCHS=${ARCHS:-"amd64"}
for ARCH in $ARCHS; do
for OS in $OSES; do
echo $OS $ARCH $NAME
GOOS=${OS} GOARCH=${ARCH} CGO_ENABLED=0 GOARM=7 go build -o build/${NAME}-${OS}-${ARCH} cmd/${NAME}/*.go
if [ $? -eq 0 ]; then
echo OK
fi
if [ "$OS" == "windows" ]; then
mv build/${NAME}-${OS}-${ARCH} build/${NAME}-${OS}-${ARCH}.exe
fi
done
done
done

17
scripts/builder-make-bin.sh
View File

@@ -1,17 +0,0 @@
#!/usr/bin/env sh
# This file is intended to be used from Dockerfile,
# presumably from cortezaproject/corteza-server-builder.
set -eu
BUILD_TIME=${BUILD_TIME:-$(date +%FT%T%z)}
GIT_TAG=${GIT_TAG:-$(git describe --always --tags)}
APP=${1}
DST=${2:-"/bin/corteza-server-${APP}"}
LDFLAGS=""
LDFLAGS="${LDFLAGS} -X github.com/cortezaproject/corteza-server/pkg/version.BuildTime=${BUILD_TIME}"
LDFLAGS="${LDFLAGS} -X github.com/cortezaproject/corteza-server/pkg/version.Version=${GIT_TAG}"
go build -ldflags "${LDFLAGS}" -o $DST ./cmd/$APP/*.go