From 19f53dbb8efd09e870913ff1015712c648745613 Mon Sep 17 00:00:00 2001 From: Santiago Lo Coco Date: Wed, 1 Nov 2023 16:50:11 -0300 Subject: [PATCH] Improve pipeline Implement advanced filters for optimized resource usage and faster execution --- .gitlab-ci.yml | 132 +++++++++++++++++++++++++------ gateway/docker-compose.yml | 4 - observability/docker-compose.yml | 3 +- 3 files changed, 107 insertions(+), 32 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 995943a..1f816c2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ image: docker:latest variables: IMAGE_BASE: "$CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME" DOCKER_BUILDKIT: 1 + GIT_DEPTH: 1 stages: - prep @@ -172,6 +173,9 @@ test-auth-api: stage: test tags: - dev + rules: + - changes: + - auth-domain/**/* script: - export $(cat context.env | xargs) @@ -193,14 +197,17 @@ test-auth-api: reports: junit: report.xml needs: - - job: preparation - job: build-auth-api + - job: preparation artifacts: true test-subscription-api: stage: test tags: - dev + rules: + - changes: + - subscription-domain/**/* script: - export $(cat context.env | xargs) @@ -222,14 +229,17 @@ test-subscription-api: reports: junit: report.xml needs: - - job: preparation - job: build-subscription-api + - job: preparation artifacts: true test-flights-api: stage: test tags: - dev + rules: + - changes: + - flights-domain/**/* script: - export $(cat context.env | xargs) @@ -251,14 +261,17 @@ test-flights-api: reports: junit: report.xml needs: - - job: preparation - job: build-flights-api + - job: preparation artifacts: true test-gateway: stage: test tags: - dev + rules: + - changes: + - gateway/**/* script: - export $(cat context.env | xargs) @@ -266,8 +279,6 @@ test-gateway: - docker login -u $CI_REGISTRY_USER --password $CI_JOB_TOKEN $CI_REGISTRY - - docker network inspect observability_elk > /dev/null 2>&1 || docker network create -d bridge observability_elk - - docker compose -f gateway/docker-compose.yml --env-file $ENV_DEV_FILE down - docker compose -f gateway/docker-compose.yml --env-file $ENV_DEV_FILE pull - docker compose -f gateway/docker-compose.yml --env-file $ENV_DEV_FILE up --abort-on-container-exit --renew-anon-volumes @@ -281,8 +292,8 @@ test-gateway: reports: junit: report.xml needs: - - job: preparation - job: build-gateway + - job: preparation artifacts: true test-browser-client: @@ -290,6 +301,9 @@ test-browser-client: coverage: /All files[^|]*\|[^|]*\s+([\d\.]+)/ tags: - dev + rules: + - changes: + - browser-domain/**/* script: - export $(cat context.env | xargs) @@ -311,8 +325,8 @@ test-browser-client: coverage_format: cobertura path: cobertura-coverage.xml needs: - - job: preparation - job: build-browser-client + - job: preparation artifacts: true test-screen-client: @@ -320,6 +334,9 @@ test-screen-client: coverage: /All files[^|]*\|[^|]*\s+([\d\.]+)/ tags: - dev + rules: + - changes: + - screen-domain/**/* script: - export $(cat context.env | xargs) @@ -341,14 +358,21 @@ test-screen-client: coverage_format: cobertura path: cobertura-coverage.xml needs: - - job: preparation - job: build-screen-client + - job: preparation artifacts: true test-browser-integration: stage: test tags: - dev + rules: + - changes: + - auth-domain/**/* + - browser-domain/**/* + - flights-domain/**/* + - subscription-domain/**/* + - gateway/**/* script: - export $(cat context.env | xargs) - docker login -u $CI_REGISTRY_USER --password $CI_JOB_TOKEN $CI_REGISTRY @@ -383,8 +407,15 @@ test-browser-integration: - docker compose -f browser-domain/docker-compose.yml --env-file $ENV_DEV_FILE up --abort-on-container-exit needs: - job: test-flights-api + optional: true - job: test-auth-api + optional: true + - job: test-subscription-api + optional: true + - job: test-gateway + optional: true - job: test-browser-client + optional: true - job: preparation artifacts: true @@ -392,6 +423,13 @@ test-screen-integration: stage: test tags: - dev + rules: + - changes: + - auth-domain/**/* + - screen-domain/**/* + - flights-domain/**/* + - subscription-domain/**/* + - gateway/**/* script: - export $(cat context.env | xargs) - docker login -u $CI_REGISTRY_USER --password $CI_JOB_TOKEN $CI_REGISTRY @@ -426,21 +464,33 @@ test-screen-integration: - docker compose -f screen-domain/docker-compose.yml --env-file $ENV_DEV_FILE up --abort-on-container-exit needs: - job: test-flights-api + optional: true - job: test-auth-api + optional: true + - job: test-subscription-api + optional: true + - job: test-gateway + optional: true - job: test-screen-client + optional: true - job: preparation artifacts: true -deliver-dockerhub: - stage: deliver +test-clean: + stage: test tags: - dev + rules: + - changes: + - auth-domain/**/* + - screen-domain/**/* + - browser-domain/**/* + - flights-domain/**/* + - subscription-domain/**/* + - gateway/**/* script: - export $(cat context.env | xargs) - - docker login -u $CI_REGISTRY_USER --password $CI_JOB_TOKEN $CI_REGISTRY - - docker login -u $DOCKER_HUB_USER --password $DOCKER_HUB_PASS - - export API_IMAGE=$FLIGHTS_INFO_TEST_IMAGE_NAME - docker compose -f flights-domain/docker-compose.yml --env-file $ENV_DEV_FILE down - export API_IMAGE=$USER_MANAGER_TEST_IMAGE_NAME @@ -449,6 +499,31 @@ deliver-dockerhub: - docker compose -f subscription-domain/docker-compose.yml --env-file $ENV_DEV_FILE down - export API_IMAGE=$GATEWAY_TEST_IMAGE_NAME - docker compose -f gateway/docker-compose.yml --env-file $ENV_DEV_FILE down + needs: + - job: test-screen-integration + optional: true + - job: test-browser-integration + optional: true + - job: preparation + artifacts: true + +deliver-dockerhub: + stage: deliver + tags: + - dev + rules: + - changes: + - auth-domain/**/* + - screen-domain/**/* + - browser-domain/**/* + - flights-domain/**/* + - subscription-domain/**/* + - gateway/**/* + script: + - export $(cat context.env | xargs) + + - docker login -u $CI_REGISTRY_USER --password $CI_JOB_TOKEN $CI_REGISTRY + - docker login -u $DOCKER_HUB_USER --password $DOCKER_HUB_PASS - docker tag $FLIGHTS_INFO_PROD_IMAGE_NAME $DOCKER_HUB_FLIGHT_INFO_IMAGE - docker tag $USER_MANAGER_PROD_IMAGE_NAME $DOCKER_HUB_USER_MANAGER_IMAGE @@ -463,18 +538,17 @@ deliver-dockerhub: - docker push $DOCKER_HUB_GATEWAY_IMAGE - docker push $DOCKER_HUB_BROWSER_CLIENT_IMAGE - docker push $DOCKER_HUB_SCREEN_CLIENT_IMAGE - needs: - - job: test-screen-integration - - job: test-browser-integration - job: preparation artifacts: true - build-and-run-elk: stage: build tags: - dev + rules: + - changes: + - observability/**/* script: - export $(cat context.env | xargs) @@ -485,15 +559,16 @@ build-and-run-elk: - export CURATOR_IMAGE=${CURATOR_PROD_IMAGE_NAME} - export HEARTBEAT_IMAGE=${HEARTBEAT_PROD_IMAGE_NAME} - - docker build observability/elk/setup -f observability/elk/setup/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${ELK_SETUP_PROD_IMAGE_NAME} - - docker build observability/elk/elasticsearch -f observability/elk/elasticsearch/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${ELASTIC_PROD_IMAGE_NAME} - - docker build observability/elk/logstash -f observability/elk/logstash/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${LOGSTASH_PROD_IMAGE_NAME} - - docker build observability/elk/kibana -f observability/elk/kibana/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${KIBANA_PROD_IMAGE_NAME} - - docker build observability/elk/extensions/curator -f observability/elk/extensions/curator/Dockerfile -t ${CURATOR_PROD_IMAGE_NAME} - - docker build observability/elk/extensions/heartbeat -f observability/elk/extensions/heartbeat/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${HEARTBEAT_PROD_IMAGE_NAME} + - export ELK=observability/elk + - docker build $ELK/setup -f $ELK/setup/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${ELK_SETUP_PROD_IMAGE_NAME} + - docker build $ELK/elasticsearch -f $ELK/elasticsearch/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${ELASTIC_PROD_IMAGE_NAME} + - docker build $ELK/logstash -f $ELK/logstash/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${LOGSTASH_PROD_IMAGE_NAME} + - docker build $ELK/kibana -f $ELK/kibana/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${KIBANA_PROD_IMAGE_NAME} + - docker build $ELK/extensions/curator -f $ELK/extensions/curator/Dockerfile -t ${CURATOR_PROD_IMAGE_NAME} + - docker build $ELK/extensions/heartbeat -f $ELK/extensions/heartbeat/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${HEARTBEAT_PROD_IMAGE_NAME} - docker compose -f observability/docker-compose.yml --env-file $ENV_DEV_FILE --profile setup up -d - - docker compose -f observability/docker-compose.yml -f observability/elk/extensions/curator/curator-compose.yml -f observability/elk/extensions/heartbeat/heartbeat-compose.yml --env-file $ENV_DEV_FILE up -d + - docker compose -f observability/docker-compose.yml -f $ELK/extensions/curator/curator-compose.yml -f $ELK/extensions/heartbeat/heartbeat-compose.yml --env-file $ENV_DEV_FILE up -d needs: - job: preparation artifacts: true @@ -505,6 +580,13 @@ deploy-prod: rules: - if: $CI_COMMIT_REF_NAME == "master" when: on_success + - changes: + - auth-domain/**/* + - screen-domain/**/* + - browser-domain/**/* + - flights-domain/**/* + - subscription-domain/**/* + - gateway/**/* script: - export $(cat context.env | xargs) @@ -547,7 +629,5 @@ deploy-prod: - docker compose -f browser-domain/docker-compose.yml pull - docker compose -f browser-domain/docker-compose.yml up -d needs: - - job: deliver-dockerhub - job: preparation - - job: build-and-run-elk artifacts: true diff --git a/gateway/docker-compose.yml b/gateway/docker-compose.yml index ddd369b..c2db139 100644 --- a/gateway/docker-compose.yml +++ b/gateway/docker-compose.yml @@ -22,7 +22,6 @@ services: - flights - gateway - subscriptions - - elk logging: driver: gelf options: @@ -39,8 +38,5 @@ networks: subscriptions: name: subscription-domain_subscriptions external: true - elk: - name: observability_elk - external: true gateway: driver: bridge diff --git a/observability/docker-compose.yml b/observability/docker-compose.yml index ebabcfd..bd83435 100644 --- a/observability/docker-compose.yml +++ b/observability/docker-compose.yml @@ -70,8 +70,7 @@ services: networks: elk: - name: observability_elk - external: true + driver: bridge volumes: elasticsearch: