Merge pull request #9 from aljazceru/codex/add-docker-infrastructure-files

Add docker infrastructure

docker-compose.yml

@@ -0,0 +1,80 @@
+version: '3.8'
+services:
+  redis:
+    image: redis:7
+    ports:
+      - "6379:6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+  mongodb:
+    image: mongo:7
+    ports:
+      - "27017:27017"
+    volumes:
+      - mongo-data:/data/db
+    healthcheck:
+      test: ["CMD", "mongo", "--eval", "db.adminCommand('ping')"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+  minio:
+    image: minio/minio
+    command: server /data --console-address ":9001"
+    environment:
+      MINIO_ACCESS_KEY: minioadmin
+      MINIO_SECRET_KEY: minioadmin
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    volumes:
+      - minio-data:/data
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 5
+  worker:
+    build:
+      context: .
+      dockerfile: docker/worker/Dockerfile
+    depends_on:
+      redis:
+        condition: service_healthy
+      mongodb:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
+    environment:
+      REDIS_URL: redis://redis:6379
+      REDIS_QUEUE_NAME: rss-feed-queue
+      MONGODB_URL: mongodb://mongodb:27017
+      MONGODB_DB_NAME: ingestrss
+      MONGODB_COLLECTION_NAME: rss_feeds
+      MINIO_ENDPOINT: http://minio:9000
+      MINIO_ACCESS_KEY: minioadmin
+      MINIO_SECRET_KEY: minioadmin
+      MINIO_BUCKET: ingestrss
+      STORAGE_STRATEGY: s3
+      LOG_LEVEL: INFO
+  scheduler:
+    build:
+      context: .
+      dockerfile: docker/scheduler/Dockerfile
+    depends_on:
+      redis:
+        condition: service_healthy
+      mongodb:
+        condition: service_healthy
+    environment:
+      REDIS_URL: redis://redis:6379
+      REDIS_QUEUE_NAME: rss-feed-queue
+      MONGODB_URL: mongodb://mongodb:27017
+      MONGODB_DB_NAME: ingestrss
+      MONGODB_COLLECTION_NAME: rss_feeds
+      LOG_LEVEL: INFO
+volumes:
+  mongo-data:
+  minio-data:

docker/scheduler/Dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+COPY . .
+CMD ["python", "local_services/scheduler.py"]

docker/worker/Dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+COPY . .
+CMD ["python", "local_services/worker.py"]

launch.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import json
+import subprocess
 import boto3
 from dotenv import load_dotenv
 import logging
@@ -70,31 +71,31 @@ lambda_client = boto3.client("lambda")
 current_dir = os.path.dirname(os.path.abspath(__file__))
 sys.path.append(current_dir)
 
-from src.infra.deploy_infrastructure import deploy_infrastructure
+from src.infra.import deploy_infrastructure
 from src.infra.lambdas.RSSFeedProcessorLambda.deploy_rss_feed_lambda import deploy_lambda
 from src.infra.lambdas.lambda_utils.update_lambda_env_vars import update_env_vars
 from src.feed_management.upload_rss_feeds import upload_rss_feeds
 
+def main():
+    if "--local" in sys.argv:
+        subprocess.run(["docker", "compose", "up", "-d"], check=False)
+        return
 
-def main(local: bool = False) -> None:
-    if local:
-        start_docker_containers()
-    else:
-        # Deploy infrastructure
-        deploy_infrastructure()
-        logging.info("Finished Deploying Infrastructure")
+    # Deploy infrastructure
+    deploy_infrastructure()
+    logging.info("Finished Deploying Infrastructure")
+   
+    # Deploy Lambda function
+    deploy_lambda()
+    logging.info("Finished Deploying Lambda")
 
-        # Deploy Lambda function
-        deploy_lambda()
-        logging.info("Finished Deploying Lambda")
+    deploy_sqs_filler()
+    logging.info("Finished Deploying Queue Filler Lambda")
 
-        deploy_sqs_filler()
-        logging.info("Finished Deploying Queue Filler Lambda")
+    # Update Lambda environment variables
+    update_env_vars(os.getenv("LAMBDA_FUNCTION_NAME"))
+    print("Finished Environment Variable Updates")
 
-        # Update Lambda environment variables
-        update_env_vars(os.getenv("LAMBDA_FUNCTION_NAME"))
-        print("Finished Environment Variable Updates")
-    
     # Upload RSS feeds
     rss_feeds_file = os.path.join(current_dir, "rss_feeds.json")
     if os.path.exists(rss_feeds_file):

local_services/scheduler.py

@@ -0,0 +1,27 @@
+import os
+import time
+import logging
+import importlib
+
+handler = importlib.import_module(
+    'src.infra.lambdas.RSSQueueFiller.lambda.lambda_function'
+).handler
+
+logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"))
+logger = logging.getLogger(__name__)
+
+INTERVAL_MINUTES = int(os.getenv("SCHEDULER_INTERVAL_MINUTES", "240"))
+
+
+def main():
+    logger.info("Starting scheduler loop")
+    while True:
+        try:
+            handler({}, None)
+        except Exception as exc:
+            logger.exception("Scheduler job failed: %s", exc)
+        time.sleep(INTERVAL_MINUTES * 60)
+
+
+if __name__ == "__main__":
+    main()

local_services/worker.py

@@ -0,0 +1,22 @@
+import os
+import time
+import logging
+
+from src.infra.lambdas.RSSFeedProcessorLambda.src.lambda_function import lambda_handler
+
+logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"))
+logger = logging.getLogger(__name__)
+
+SLEEP_SECONDS = int(os.getenv("WORKER_SLEEP_SECONDS", "5"))
+
+def main():
+    logger.info("Starting worker loop")
+    while True:
+        try:
+            lambda_handler({}, None)
+        except Exception as exc:
+            logger.exception("Worker iteration failed: %s", exc)
+        time.sleep(SLEEP_SECONDS)
+
+if __name__ == "__main__":
+    main()