Update pre-commit hook and reformat files

.gitlab-ci.yml

@@ -50,11 +50,11 @@ preparation:
     - echo "DOCKER_HUB_SUBSCRIPTION_IMAGE=$DOCKER_HUB_USER/subs-manager:${BUILD_ID}" >> context.env
     - echo "DOCKER_HUB_USER_MANAGER_IMAGE=$DOCKER_HUB_USER/user-manager:${BUILD_ID}" >> context.env
     - echo "DOCKER_HUB_FLIGHT_INFO_IMAGE=$DOCKER_HUB_USER/flights-information:${BUILD_ID}" >> context.env
-    
+
     - echo "ENV_DEV_FILE=$(echo $ENV_DEV)"     >> context.env
     - echo "ENV_PROD_FILE=$(echo $ENV_PROD)"   >> context.env
-  artifacts: 
-    paths: 
+  artifacts:
+    paths:
       - context.env
 
 build-auth-api:
@@ -352,8 +352,8 @@ test-browser-integration:
   script:
     - export $(cat context.env | xargs)
     - docker login -u $CI_REGISTRY_USER --password $CI_JOB_TOKEN $CI_REGISTRY
-    
-    - export TEST_TARGET=INTEGRATION 
+
+    - export TEST_TARGET=INTEGRATION
 
     - export API_IMAGE=$FLIGHTS_INFO_TEST_IMAGE_NAME
     - docker compose -f flights-domain/docker-compose.yml --env-file $ENV_DEV_FILE down
@@ -395,8 +395,8 @@ test-screen-integration:
   script:
     - export $(cat context.env | xargs)
     - docker login -u $CI_REGISTRY_USER --password $CI_JOB_TOKEN $CI_REGISTRY
-    
-    - export TEST_TARGET=INTEGRATION 
+
+    - export TEST_TARGET=INTEGRATION
 
     - export API_IMAGE=$FLIGHTS_INFO_TEST_IMAGE_NAME
     - docker compose -f flights-domain/docker-compose.yml --env-file $ENV_DEV_FILE down
@@ -463,12 +463,12 @@ deliver-dockerhub:
     - docker push $DOCKER_HUB_GATEWAY_IMAGE
     - docker push $DOCKER_HUB_BROWSER_CLIENT_IMAGE
     - docker push $DOCKER_HUB_SCREEN_CLIENT_IMAGE
-    
+
   needs:
     - job: test-screen-integration
     - job: test-browser-integration
     - job: preparation
-      artifacts: true  
+      artifacts: true
 
 
 build-and-run-elk:
@@ -491,7 +491,7 @@ build-and-run-elk:
     - docker build observability/elk/kibana -f observability/elk/kibana/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${KIBANA_PROD_IMAGE_NAME}
     - docker build observability/elk/extensions/curator -f observability/elk/extensions/curator/Dockerfile -t ${CURATOR_PROD_IMAGE_NAME}
     - docker build observability/elk/extensions/heartbeat -f observability/elk/extensions/heartbeat/Dockerfile --build-arg "ELASTIC_VERSION=8.7.1" -t ${HEARTBEAT_PROD_IMAGE_NAME}
-    
+
     - docker compose -f observability/docker-compose.yml --env-file $ENV_DEV_FILE --profile setup up -d
     - docker compose -f observability/docker-compose.yml -f observability/elk/extensions/curator/curator-compose.yml -f observability/elk/extensions/heartbeat/heartbeat-compose.yml --env-file $ENV_DEV_FILE up -d
   needs:
@@ -500,7 +500,7 @@ build-and-run-elk:
 
 deploy-prod:
   stage: deploy
-  tags: 
+  tags:
     - prod
   rules:
     - if: $CI_COMMIT_REF_NAME == "master"

.pre-commit-config.yaml

@@ -1,11 +1,28 @@
-files: ^flights-domain/
 repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-executables-have-shebangs
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-yaml
+        exclude: observability/
+      - id: debug-statements
+        exclude: tests/
+      - id: destroyed-symlinks
+      - id: end-of-file-fixer
+        files: \.(py|sh|rst|yml|yaml)$
+      - id: mixed-line-ending
+      - id: trailing-whitespace
+        files: \.(py|sh|rst|yml|yaml)$
   - repo: https://github.com/ambv/black
     rev: 23.10.0
     hooks:
       - id: black
   - repo: https://github.com/pycqa/flake8
-    rev: 6.1.0
+    rev: 6.0.0
     hooks:
       - id: flake8
         args: [--config, flights-domain/flights-information/setup.cfg]
@@ -13,4 +30,4 @@ repos:
     rev: 5.12.0
     hooks:
       - id: isort
-        args: ['--src-path', 'flights-domain/flights-information/src', 'auth-domain/user-manager/src', 'gateway/src', 'subscription-domain/subscription-manager/src']
+        args: ['--profile', 'black', '--src-path', 'flights-domain/flights-information/src', 'auth-domain/user-manager/src', 'gateway/src', 'subscription-domain/subscription-manager/src']

auth-domain/docker-compose.yml

@@ -21,7 +21,7 @@ services:
     networks:
       - auth
 
-  usermanager-db: 
+  usermanager-db:
     container_name: fids_usermanager_db
     build:
       context: ./db
@@ -42,4 +42,4 @@ services:
 
 networks:
   auth:
-    driver: bridge
+    driver: bridge

auth-domain/user-manager/.bandit.yml

@@ -1,5 +1,5 @@
 
-exclude_dirs: 
+exclude_dirs:
   - src/tests
 #tests: ['B201', 'B301']
-#skips: ['B101', 'B601']
+#skips: ['B101', 'B601']

auth-domain/user-manager/entrypoint.sh

@@ -8,4 +8,4 @@ done
 
 echo "PostgreSQL started"
 
-python manage.py run -h 0.0.0.0
+python manage.py run -h 0.0.0.0

auth-domain/user-manager/manage.py

@@ -4,7 +4,7 @@ from src import create_app, db
 from src.api.models.users import User
 
 app = create_app()
-cli = FlaskGroup(create_app=create_app) 
+cli = FlaskGroup(create_app=create_app)
 
 
 @cli.command("recreate_db")
@@ -16,9 +16,25 @@ def recreate_db():
 
 @cli.command("seed_db")
 def seed_db():
-    db.session.add(User(username="lufthansa", email="info@lufthansa.com", password="password1234", airline=True))
-    db.session.add(User(username="ryanair", email="info@ryanair.com", password="password1234", airline=True))
-    db.session.add(User(username="messi", email="messi@gmail.com", password="password1234"))
+    db.session.add(
+        User(
+            username="lufthansa",
+            email="info@lufthansa.com",
+            password="password1234",
+            airline=True,
+        )
+    )
+    db.session.add(
+        User(
+            username="ryanair",
+            email="info@ryanair.com",
+            password="password1234",
+            airline=True,
+        )
+    )
+    db.session.add(
+        User(username="messi", email="messi@gmail.com", password="password1234")
+    )
     db.session.commit()
 
 

auth-domain/user-manager/src/.cicd/test.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -e 
+#!/bin/bash -e
 
 
 if [ "${TEST_TARGET:-}" = "INTEGRATION" ]; then
@@ -13,11 +13,10 @@ else
 
 
     ## Linting
-    flake8 src --extend-ignore E221 --extend-ignore E501 
+    flake8 src --extend-ignore E221 --extend-ignore E501
     # black src --check
     # isort src --check
 
     ## Security
     # bandit -c .bandit.yml -r .
 fi
-

auth-domain/user-manager/src/api/auth.py

@@ -55,9 +55,11 @@ class Login(Resource):
         access_token = user.encode_token(user.id, "access", user.airline)
         refresh_token = user.encode_token(user.id, "refresh")
 
-        response_object = {"access_token": access_token,
-                           "refresh_token": refresh_token,
-                           "user_id": user.id}
+        response_object = {
+            "access_token": access_token,
+            "refresh_token": refresh_token,
+            "user_id": user.id,
+        }
         return response_object, 200
 
 
@@ -84,7 +86,7 @@ class Refresh(Resource):
             response_object = {
                 "access_token": access_token,
                 "refresh_token": refresh_token,
-                "user_id": user.id
+                "user_id": user.id,
             }
             return response_object, 200
         except jwt.ExpiredSignatureError:

auth-domain/user-manager/src/api/models/users.py

@@ -39,7 +39,7 @@ class User(db.Model):
             "exp": datetime.datetime.utcnow() + datetime.timedelta(seconds=seconds),
             "iat": datetime.datetime.utcnow(),
             "sub": user_id,
-            "airline": airline
+            "airline": airline,
         }
         return jwt.encode(
             payload, current_app.config.get("SECRET_KEY"), algorithm="HS256"
@@ -61,7 +61,7 @@ class User(db.Model):
                 "username": fields.String(required=True),
                 "email": fields.String(required=True),
                 "created_date": fields.DateTime,
-                "airline": fields.Boolean(readOnly=True)
+                "airline": fields.Boolean(readOnly=True),
             },
         )
 
@@ -82,7 +82,7 @@ class User(db.Model):
             {
                 "username": fields.String(required=True),
                 "email": fields.String(required=True),
-                "id": fields.Integer(required=True)
+                "id": fields.Integer(required=True),
             },
         )
 
@@ -119,5 +119,5 @@ class User(db.Model):
             "Access and Refresh Token",
             cls.get_api_auth_refresh_model(namespace),
             {"access_token": fields.String(required=True)},
-            {"user_id": fields.Integer(required=True)}
+            {"user_id": fields.Integer(required=True)},
         )

auth-domain/user-manager/src/tests/unit/test_generic.py

@@ -1,5 +1,4 @@
-from src.api.models.generic import (get_model_create_response,
-                                    get_model_error_response)
+from src.api.models.generic import get_model_create_response, get_model_error_response
 
 
 def test_model_create_response(test_namespace):

flights-domain/docker-compose.yml

@@ -23,7 +23,7 @@ services:
       - flights
       - subscriptions
 
-  flights-api-db: 
+  flights-api-db:
     container_name: fids_flights_db
     build:
       context: ./db
@@ -47,4 +47,4 @@ networks:
     name: subscription-domain_subscriptions
     external: true
   flights:
-    driver: bridge
+    driver: bridge

flights-domain/flights-information/.bandit.yml

@@ -1,5 +1,5 @@
 
-exclude_dirs: 
+exclude_dirs:
   - src/tests
 #tests: ['B201', 'B301']
-#skips: ['B101', 'B601']
+#skips: ['B101', 'B601']

flights-domain/flights-information/src/.cicd/test.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -e 
+#!/bin/bash -e
 
 
 if [ "${TEST_TARGET:-}" = "INTEGRATION" ]; then
@@ -12,7 +12,7 @@ else
 
 
     ## Linting
-    flake8 src --extend-ignore E221 --extend-ignore E501 
+    flake8 src --extend-ignore E221 --extend-ignore E501
     # black src --check
     # isort . --src-path src --check
 

flights-domain/flights-information/src/api/main.py

@@ -6,6 +6,7 @@ from src.api.routes import flights, health
 
 Base.metadata.create_all(bind=engine)
 
+
 app = FastAPI(title="Flights Information API")
 app.include_router(flights.router, prefix="/flights")
 app.include_router(health.router, prefix="/health")

gateway/.bandit.yml

@@ -1,5 +1,5 @@
 
-exclude_dirs: 
+exclude_dirs:
   - src/tests
 #tests: ['B201', 'B301']
-#skips: ['B101', 'B601']
+#skips: ['B101', 'B601']

gateway/docker-compose.yml

@@ -43,4 +43,4 @@ networks:
     name: observability_elk
     external: true
   gateway:
-    driver: bridge
+    driver: bridge

gateway/sleep.sh

@@ -1,7 +0,0 @@
-#!/bin/sh
-
-# while ! curl -f 'fids_logstash:9600/_node/pipelines'; do
-#     sleep 1
-# done
-
-nc -vz -w1 localhost 5004 # && curl -f fids_logstash:9600/_node/pipelines

gateway/src/.cicd/test.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -e 
+#!/bin/bash -e
 
 
 if [ "${TEST_TARGET:-}" = "INTEGRATION" ]; then
@@ -14,7 +14,7 @@ else
 
 
     ## Linting
-    flake8 src --extend-ignore E221 --extend-ignore E501 
+    flake8 src --extend-ignore E221 --extend-ignore E501
     # black src --check
     # isort . --src-path src --check
 

gateway/src/api/main.py

@@ -3,10 +3,10 @@ import logging
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
-from src.api.routes import (auth, flights, health, notifications,
-                            subscriptions, users)
+from src.api.routes import auth, flights, health, notifications, subscriptions, users
+
+logging.config.fileConfig("logging.conf", disable_existing_loggers=False)
 
-logging.config.fileConfig('logging.conf', disable_existing_loggers=False)
 
 app = FastAPI(title="Flights Information API")
 app.include_router(flights.router, prefix="/flights")

gateway/src/api/routes/flights.py

@@ -25,9 +25,7 @@ async def create_flight(
     auth = await checkAuth(authorization)
     flight_data = flight.model_dump()
     flight_data["user_id"] = auth["id"]
-    (response, status, _) = await request(
-        f"{API_FLIGHTS}", "POST", json=flight_data
-    )
+    (response, status, _) = await request(f"{API_FLIGHTS}", "POST", json=flight_data)
     if status < 200 or status > 204:
         raise HTTPException(status_code=status, detail=response)
     return response
@@ -42,16 +40,18 @@ async def update_flight(
     auth = await checkAuth(authorization)
     status = status_update.model_dump()
     status["user_id"] = auth["id"]
-    (response, status, _) = await request(
-        f"{API_FLIGHTS}/{id}", "PATCH", json=status
-    )
+    (response, status, _) = await request(f"{API_FLIGHTS}/{id}", "PATCH", json=status)
     if status < 200 or status > 204:
         raise HTTPException(status_code=status, detail=response)
     return response
 
 
 @router.get("", response_model=list[Flight])
-async def get_flights(origin: Optional[str] = None, destination: Optional[str] = None, lastUpdated: Optional[str] = None):
+async def get_flights(
+    origin: Optional[str] = None,
+    destination: Optional[str] = None,
+    lastUpdated: Optional[str] = None,
+):
     query = {}
     if origin:
         query["origin"] = origin

gateway/src/api/routes/health.py

@@ -1,21 +1,13 @@
-# from src.api.main import my_logger
 import logging
 
 from fastapi import APIRouter
 
-# from sys import stdout
-
 my_logger = logging.getLogger(__name__)
-# my_logger.setLevel(logging.INFO)
-# consoleHandler = logging.StreamHandler(stdout)
-# my_logger.addHandler(consoleHandler)
-
-# handler = graypy.GELFUDPHandler("fids_logstash", 12201)
-# my_logger.addHandler(handler)
 
 router = APIRouter()
 
+
 @router.get("", status_code=200)
 async def get_health():
-    my_logger.info("{\"health\":\"OK\"}")
+    my_logger.info('{"health":"OK"}')
     return {"status": "OK"}

gateway/src/api/routes/subscriptions.py

@@ -12,8 +12,7 @@ router = APIRouter()
 
 @router.post("")
 async def create_subscription(
-    subscription: Subscription,
-    authorization: Annotated[str | None, Header()] = None
+    subscription: Subscription, authorization: Annotated[str | None, Header()] = None
 ):
     await checkAuth(authorization)
     (response, status, _) = await request(

observability/docker-compose.yml

@@ -5,7 +5,7 @@ services:
   setup:
     profiles:
       - setup
-    image: ${ELK_SETUP_IMAGE} 
+    image: ${ELK_SETUP_IMAGE}
     init: true
     environment:
       ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}

run.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-usage() { 
+usage() {
     printf "$0 usage: \n -x: down\n -d \$DOMAIN: domain\n -t: tests\n" $0
     exit 0
 }

subscription-domain/docker-compose.yml

@@ -25,7 +25,7 @@ services:
     networks:
       - subscriptions
 
-  subscriptions-api-db: 
+  subscriptions-api-db:
     container_name: fids_subscriptions_db
     build:
       context: ./db
@@ -46,4 +46,4 @@ services:
 
 networks:
   subscriptions:
-    driver: bridge
+    driver: bridge

subscription-domain/subscription-manager/.bandit.yml

@@ -1,5 +1,5 @@
 
-exclude_dirs: 
+exclude_dirs:
   - src/tests
 #tests: ['B201', 'B301']
-#skips: ['B101', 'B601']
+#skips: ['B101', 'B601']

subscription-domain/subscription-manager/src/.cicd/test.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -e 
+#!/bin/bash -e
 
 
 if [ "${TEST_TARGET:-}" = "INTEGRATION" ]; then
@@ -13,7 +13,7 @@ else
     touch coverage.xml
 
     ## Linting
-    flake8 src --extend-ignore E221 --extend-ignore E501 
+    flake8 src --extend-ignore E221 --extend-ignore E501
     # black src --check
     # isort . --src-path src --check
 

subscription-domain/subscription-manager/src/api/cruds/subscription.py

@@ -21,8 +21,9 @@ def create_subscription(db: Session, subscription: SubscriptionPydantic):
 
 
 def remove_subscription(db: Session, user_id: int, flight_id: int):
-    db.query(Subscription).filter(Subscription.user_id == user_id
-                                  and Subscription.flight_id == flight_id).delete()
+    db.query(Subscription).filter(
+        Subscription.user_id == user_id and Subscription.flight_id == flight_id
+    ).delete()
     db.commit()
 
 

subscription-domain/subscription-manager/src/api/main.py

@@ -6,6 +6,7 @@ from src.api.routes import health, messages, notifications, subscriptions
 
 Base.metadata.create_all(bind=engine)
 
+
 app = FastAPI(title="Subscription Information API")
 app.include_router(subscriptions.router, prefix="/subscriptions")
 app.include_router(notifications.router, prefix="/notifications")

subscription-domain/subscription-manager/src/api/routes/messages.py

@@ -12,7 +12,9 @@ router = APIRouter()
 
 
 @router.post("")
-async def send_notification(flight: FlightData, background_tasks: BackgroundTasks, db: Session = Depends(get_db)):
+async def send_notification(
+    flight: FlightData, background_tasks: BackgroundTasks, db: Session = Depends(get_db)
+):
     db_subscriptions = sub_crud.send_subscriptions(db=db, flight=flight)
     for subscription in db_subscriptions:
         db_chat = notif_crud.get_chat_id(db=db, user_id=subscription.user_id)

subscription-domain/subscription-manager/src/api/routes/notifications.py

@@ -14,14 +14,12 @@ from src.api.utils.messages import get_flight_message, get_invalid_message
 
 router = APIRouter()
 
-msg_options = re.compile(r'^/(flight \d+|stop|start)$')
+msg_options = re.compile(r"^/(flight \d+|stop|start)$")
 
 
 @router.post("")
 async def create_chat(
-    chat: Update,
-    background_tasks: BackgroundTasks,
-    db: Session = Depends(get_db)
+    chat: Update, background_tasks: BackgroundTasks, db: Session = Depends(get_db)
 ):
     print(chat.model_dump())
     message = chat.message
@@ -32,19 +30,19 @@ async def create_chat(
         background_tasks.add_task(telegram.send_message, chat_id, msg)
         return Response(status_code=204)
 
-    action = text.partition(' ')[0]
-    if action == '/start':
-        user_id = int(message["text"].partition(' ')[2])
+    action = text.partition(" ")[0]
+    if action == "/start":
+        user_id = int(message["text"].partition(" ")[2])
         new_chat = Chat(chat_id=str(message["chat"]["id"]), user_id=user_id)
         notif_crud.create_chat(db=db, chat=new_chat)
-    elif action == '/stop':
+    elif action == "/stop":
         chat_id = str(message["chat"]["id"])
         user_id = notif_crud.get_user_from_chat(db=db, chat_id=chat_id).user_id
         subs_crud.remove_subscriptions(user_id)
         notif_crud.remove_chat(db=db, chat_id=chat_id)
-    elif action == '/flight':
+    elif action == "/flight":
         chat_id = str(message["chat"]["id"])
-        flight_id = int(message["text"].partition(' ')[2])
+        flight_id = int(message["text"].partition(" ")[2])
         (response, status, _) = await request(f"{API_FLIGHTS}/{flight_id}", "GET")
         if status < 200 or status > 204:
             msg = f"Could not get flight '{flight_id}'. Sorry!"

subscription-domain/subscription-manager/src/api/routes/subscriptions.py

@@ -22,6 +22,10 @@ def get_subscriptions(user_id: int, db: Session = Depends(get_db)):
 
 
 @router.delete("/{user_id}")
-def delete_subscription(user_id: int, subscription: SubscriptionRemove, db: Session = Depends(get_db)):
-    sub_crud.remove_subscription(db=db, user_id=user_id, flight_id=subscription.flight_id)
+def delete_subscription(
+    user_id: int, subscription: SubscriptionRemove, db: Session = Depends(get_db)
+):
+    sub_crud.remove_subscription(
+        db=db, user_id=user_id, flight_id=subscription.flight_id
+    )
     return Response(status_code=204)