Improve pipeline

Implement advanced filters for optimized resource usage and faster execution
2023-11-01 16:50:11 -03:00 · 2023-11-01 16:50:11 -03:00 · 19f53dbb8e
parent cad88bf24e
commit 19f53dbb8e
3 changed files with 107 additions and 32 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -3,6 +3,7 @@ image: docker:latest
 variables:
  IMAGE_BASE: "$CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME"
  DOCKER_BUILDKIT: 1
+  GIT_DEPTH: 1

 stages:
  - prep
@ -172,6 +173,9 @@ test-auth-api:
  stage: test
  tags:
    - dev
+  rules:
+    - changes:
+      - auth-domain/**/*
  script:
    - export $(cat context.env | xargs)

@ -193,14 +197,17 @@ test-auth-api:
    reports:
      junit: report.xml
  needs:
-    - job: preparation
    - job: build-auth-api
+    - job: preparation
      artifacts: true

 test-subscription-api:
  stage: test
  tags:
    - dev
+  rules:
+    - changes:
+      - subscription-domain/**/*
  script:
    - export $(cat context.env | xargs)

@ -222,14 +229,17 @@ test-subscription-api:
    reports:
      junit: report.xml
  needs:
-    - job: preparation
    - job: build-subscription-api
+    - job: preparation
      artifacts: true

 test-flights-api:
  stage: test
  tags:
    - dev
+  rules:
+    - changes:
+      - flights-domain/**/*
  script:
    - export $(cat context.env | xargs)

@ -251,14 +261,17 @@ test-flights-api:
    reports:
      junit: report.xml
  needs:
-    - job: preparation
    - job: build-flights-api
+    - job: preparation
      artifacts: true

 test-gateway:
  stage: test
  tags:
    - dev
+  rules:
+    - changes:
+      - gateway/**/*
  script:
    - export $(cat context.env | xargs)

@ -266,8 +279,6 @@ test-gateway:

    - docker login -u $CI_REGISTRY_USER --password $CI_JOB_TOKEN $CI_REGISTRY

-    - docker network inspect observability_elk > /dev/null 2>&1 || docker network create -d bridge observability_elk
-
    - docker compose -f gateway/docker-compose.yml --env-file $ENV_DEV_FILE down
    - docker compose -f gateway/docker-compose.yml --env-file $ENV_DEV_FILE pull
    - docker compose -f gateway/docker-compose.yml --env-file $ENV_DEV_FILE up --abort-on-container-exit --renew-anon-volumes
@ -281,8 +292,8 @@ test-gateway:
    reports:
      junit: report.xml
  needs:
-    - job: preparation
    - job: build-gateway
+    - job: preparation
      artifacts: true

 test-browser-client:
@ -290,6 +301,9 @@ test-browser-client:
  coverage: /All files[^|]*\|[^|]*\s+([\d\.]+)/
  tags:
    - dev
+  rules:
+    - changes:
+      - browser-domain/**/*
  script:
    - export $(cat context.env | xargs)

@ -311,8 +325,8 @@ test-browser-client:
        coverage_format: cobertura
        path: cobertura-coverage.xml
  needs:
-    - job: preparation
    - job: build-browser-client
+    - job: preparation
      artifacts: true

 test-screen-client:
@ -320,6 +334,9 @@ test-screen-client:
  coverage: /All files[^|]*\|[^|]*\s+([\d\.]+)/
  tags:
    - dev
+  rules:
+    - changes:
+      - screen-domain/**/*
  script:
    - export $(cat context.env | xargs)

@ -341,14 +358,21 @@ test-screen-client:
        coverage_format: cobertura
        path: cobertura-coverage.xml
  needs:
-    - job: preparation
    - job: build-screen-client
+    - job: preparation
      artifacts: true

 test-browser-integration:
  stage: test
  tags:
    - dev
+  rules:
+    - changes:
+      - auth-domain/**/*
+      - browser-domain/**/*
+      - flights-domain/**/*
+      - subscription-domain/**/*
+      - gateway/**/*
  script:
    - export $(cat context.env | xargs)
    - docker login -u $CI_REGISTRY_USER --password $CI_JOB_TOKEN $CI_REGISTRY
@ -383,8 +407,15 @@ test-browser-integration:
    - docker compose -f browser-domain/docker-compose.yml --env-file $ENV_DEV_FILE up --abort-on-container-exit
  needs:
    - job: test-flights-api
+      optional: true
    - job: test-auth-api
+      optional: true
+    - job: test-subscription-api
+      optional: true
+    - job: test-gateway
+      optional: true
    - job: test-browser-client
+      optional: true
    - job: preparation
      artifacts: true

@ -392,6 +423,13 @@ test-screen-integration:
  stage: test
  tags:
    - dev
+  rules:
+    - changes:
+      - auth-domain/**/*
+      - screen-domain/**/*
+      - flights-domain/**/*
+      - subscription-domain/**/*
+      - gateway/**/*
  script:
    - export $(cat context.env | xargs)
    - docker login -u $CI_REGISTRY_USER --password $CI_JOB_TOKEN $CI_REGISTRY
@ -426,21 +464,33 @@ test-screen-integration:
    - docker compose -f screen-domain/docker-compose.yml --env-file $ENV_DEV_FILE up --abort-on-container-exit
  needs:
    - job: test-flights-api
+      optional: true
    - job: test-auth-api
+      optional: true
+    - job: test-subscription-api
+      optional: true
+    - job: test-gateway
+      optional: true
    - job: test-screen-client
+      optional: true
    - job: preparation
      artifacts: true

-deliver-dockerhub:
-  stage: deliver
+test-clean:
+  stage: test
  tags:
    - dev
+  rules:
+    - changes:
+      - auth-domain/**/*
+      - screen-domain/**/*
+      - browser-domain/**/*
+      - flights-domain/**/*
+      - subscription-domain/**/*
+      - gateway/**/*
  script:
    - export $(cat context.env | xargs)

-    - docker login -u $CI_REGISTRY_USER --password $CI_JOB_TOKEN $CI_REGISTRY
-    - docker login -u $DOCKER_HUB_USER --password $DOCKER_HUB_PASS
-
    - export API_IMAGE=$FLIGHTS_INFO_TEST_IMAGE_NAME
    - docker compose -f flights-domain/docker-compose.yml --env-file $ENV_DEV_FILE down
    - export API_IMAGE=$USER_MANAGER_TEST_IMAGE_NAME
@ -449,6 +499,31 @@ deliver-dockerhub:
    - docker compose -f subscription-domain/docker-compose.yml --env-file $ENV_DEV_FILE down
    - export API_IMAGE=$GATEWAY_TEST_IMAGE_NAME
    - docker compose -f gateway/docker-compose.yml --env-file $ENV_DEV_FILE down
+  needs:
+    - job: test-screen-integration
+      optional: true
+    - job: test-browser-integration
+      optional: true
+    - job: preparation
+      artifacts: true
+
+deliver-dockerhub:
+  stage: deliver
+  tags:
+    - dev
+  rules:
+    - changes:
+      - auth-domain/**/*
+      - screen-domain/**/*
+      - browser-domain/**/*
+      - flights-domain/**/*
+      - subscription-domain/**/*
+      - gateway/**/*
+  script:
+    - export $(cat context.env | xargs)
+
+    - docker login -u $CI_REGISTRY_USER --password $CI_JOB_TOKEN $CI_REGISTRY
+    - docker login -u $DOCKER_HUB_USER --password $DOCKER_HUB_PASS

    - docker tag $FLIGHTS_INFO_PROD_IMAGE_NAME $DOCKER_HUB_FLIGHT_INFO_IMAGE
    - docker tag $USER_MANAGER_PROD_IMAGE_NAME $DOCKER_HUB_USER_MANAGER_IMAGE
@ -463,18 +538,17 @@ deliver-dockerhub:
    - docker push $DOCKER_HUB_GATEWAY_IMAGE
    - docker push $DOCKER_HUB_BROWSER_CLIENT_IMAGE
    - docker push $DOCKER_HUB_SCREEN_CLIENT_IMAGE
-
  needs:
-    - job: test-screen-integration
-    - job: test-browser-integration
    - job: preparation
      artifacts: true

-
 build-and-run-elk:
  stage: build
  tags:
    - dev
+  rules:
+    - changes:
+      - observability/**/*
  script:
    - export $(cat context.env | xargs)

@ -485,15 +559,16 @@ build-and-run-elk:
    - export CURATOR_IMAGE=${CURATOR_PROD_IMAGE_NAME}
    - export HEARTBEAT_IMAGE=${HEARTBEAT_PROD_IMAGE_NAME}

-    - docker build observability/elk/setup -f observability/elk/setup/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${ELK_SETUP_PROD_IMAGE_NAME}
-    - docker build observability/elk/elasticsearch -f observability/elk/elasticsearch/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${ELASTIC_PROD_IMAGE_NAME}
-    - docker build observability/elk/logstash -f observability/elk/logstash/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${LOGSTASH_PROD_IMAGE_NAME}
-    - docker build observability/elk/kibana -f observability/elk/kibana/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${KIBANA_PROD_IMAGE_NAME}
-    - docker build observability/elk/extensions/curator -f observability/elk/extensions/curator/Dockerfile -t ${CURATOR_PROD_IMAGE_NAME}
-    - docker build observability/elk/extensions/heartbeat -f observability/elk/extensions/heartbeat/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${HEARTBEAT_PROD_IMAGE_NAME}
+    - export ELK=observability/elk
+    - docker build $ELK/setup -f $ELK/setup/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${ELK_SETUP_PROD_IMAGE_NAME}
+    - docker build $ELK/elasticsearch -f $ELK/elasticsearch/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${ELASTIC_PROD_IMAGE_NAME}
+    - docker build $ELK/logstash -f $ELK/logstash/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${LOGSTASH_PROD_IMAGE_NAME}
+    - docker build $ELK/kibana -f $ELK/kibana/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${KIBANA_PROD_IMAGE_NAME}
+    - docker build $ELK/extensions/curator -f $ELK/extensions/curator/Dockerfile -t ${CURATOR_PROD_IMAGE_NAME}
+    - docker build $ELK/extensions/heartbeat -f $ELK/extensions/heartbeat/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${HEARTBEAT_PROD_IMAGE_NAME}

    - docker compose -f observability/docker-compose.yml --env-file $ENV_DEV_FILE --profile setup up -d
-    - docker compose -f observability/docker-compose.yml -f observability/elk/extensions/curator/curator-compose.yml -f observability/elk/extensions/heartbeat/heartbeat-compose.yml --env-file $ENV_DEV_FILE up -d
+    - docker compose -f observability/docker-compose.yml -f $ELK/extensions/curator/curator-compose.yml -f $ELK/extensions/heartbeat/heartbeat-compose.yml --env-file $ENV_DEV_FILE up -d
  needs:
    - job: preparation
      artifacts: true
@ -505,6 +580,13 @@ deploy-prod:
  rules:
    - if: $CI_COMMIT_REF_NAME == "master"
      when: on_success
+    - changes:
+      - auth-domain/**/*
+      - screen-domain/**/*
+      - browser-domain/**/*
+      - flights-domain/**/*
+      - subscription-domain/**/*
+      - gateway/**/*
  script:
    - export $(cat context.env | xargs)

@ -547,7 +629,5 @@ deploy-prod:
    - docker compose -f browser-domain/docker-compose.yml pull
    - docker compose -f browser-domain/docker-compose.yml up -d
  needs:
-    - job: deliver-dockerhub
    - job: preparation
-    - job: build-and-run-elk
      artifacts: true
--- a/gateway/docker-compose.yml
+++ b/gateway/docker-compose.yml
@ -22,7 +22,6 @@ services:
      - flights
      - gateway
      - subscriptions
-      - elk
    logging:
      driver: gelf
      options:
@ -39,8 +38,5 @@ networks:
  subscriptions:
    name: subscription-domain_subscriptions
    external: true
-  elk:
-    name: observability_elk
-    external: true
  gateway:
    driver: bridge
--- a/observability/docker-compose.yml
+++ b/observability/docker-compose.yml
@ -70,8 +70,7 @@ services:

 networks:
  elk:
-    name: observability_elk
-    external: true
+    driver: bridge

 volumes:
  elasticsearch: