Merge branch 'main' into codex/add-docker-infrastructure-files

launch.py

@@ -5,17 +5,65 @@ import subprocess
 import boto3
 from dotenv import load_dotenv
 import logging
+import argparse
+import subprocess
 from src.infra.lambdas.RSSQueueFiller.deploy_sqs_filler_lambda import deploy_sqs_filler
 
 from src.utils.check_env import check_env
 
+
+def check_local_env() -> None:
+    """Ensure required environment variables for local mode are set."""
+    required_vars = [
+        "MONGODB_URL",
+        "MONGODB_DB_NAME",
+        "MONGODB_COLLECTION_NAME",
+        "REDIS_URL",
+        "REDIS_QUEUE_NAME",
+        "MINIO_ENDPOINT",
+        "MINIO_ACCESS_KEY",
+        "MINIO_SECRET_KEY",
+        "MINIO_BUCKET",
+    ]
+
+    missing = [var for var in required_vars if not os.getenv(var)]
+    if missing:
+        raise EnvironmentError(
+            f"Missing required environment variables for local mode: {', '.join(missing)}"
+        )
+
+
+def start_docker_containers() -> None:
+    """Start required Docker containers for local development."""
+    try:
+        subprocess.check_call(["docker-compose", "up", "-d"])
+    except FileNotFoundError:
+        # fallback to `docker compose` if docker-compose command not found
+        subprocess.check_call(["docker", "compose", "up", "-d"])
+    except Exception as exc:
+        logging.error(f"Failed to start Docker containers: {exc}")
+        raise
+
+
 print("🗞️  💵 ⚖️  IngestRSS⚖️  💵 🗞️".center(100, "-"))
 
+parser = argparse.ArgumentParser(description="Launch IngestRSS")
+parser.add_argument(
+    "--local",
+    action="store_true",
+    help="Run locally using Docker instead of deploying to AWS",
+)
+args = parser.parse_args()
+
 load_dotenv(override=True)
-check_env()
+
+if args.local:
+    check_local_env()
+else:
+    check_env()
 
 # Set up logging
-logging.basicConfig(level=os.getenv('LOG_LEVEL'))
+logging.basicConfig(level=os.getenv("LOG_LEVEL"))
 
 lambda_client = boto3.client("lambda")
 
@@ -23,7 +71,7 @@ lambda_client = boto3.client("lambda")
 current_dir = os.path.dirname(os.path.abspath(__file__))
 sys.path.append(current_dir)
 
-from src.infra.deploy_infrastructure import deploy_infrastructure
+from src.infra.import deploy_infrastructure
 from src.infra.lambdas.RSSFeedProcessorLambda.deploy_rss_feed_lambda import deploy_lambda
 from src.infra.lambdas.lambda_utils.update_lambda_env_vars import update_env_vars
 from src.feed_management.upload_rss_feeds import upload_rss_feeds
@@ -47,7 +95,7 @@ def main():
     # Update Lambda environment variables
     update_env_vars(os.getenv("LAMBDA_FUNCTION_NAME"))
     print("Finished Environment Variable Updates")
-    
+
     # Upload RSS feeds
     rss_feeds_file = os.path.join(current_dir, "rss_feeds.json")
     if os.path.exists(rss_feeds_file):
@@ -66,4 +114,4 @@ def main():
     print("RSS Feed Processor launched successfully!")
 
 if __name__ == "__main__":
-    main()
+    main(args.local)

local.env.template

@@ -0,0 +1,25 @@
+# Local environment configuration for IngestRSS
+
+# Redis configuration
+REDIS_URL=redis://localhost:6379
+REDIS_QUEUE_NAME=rss-feed-queue
+
+# MinIO configuration
+MINIO_ENDPOINT=***
+MINIO_ACCESS_KEY=***
+MINIO_SECRET_KEY=***
+MINIO_BUCKET=***
+
+# MongoDB settings
+MONGODB_URL=mongodb://localhost:27017
+MONGODB_DB_NAME=ingestrss
+MONGODB_COLLECTION_NAME=rss_feeds
+
+# Logging Configuration
+LOG_LEVEL=INFO
+
+# Other Application Settings
+APP_NAME=RSS Feed Processor
+VERSION=1.0.0
+
+STORAGE_STRATEGY=s3

requirements.txt

@@ -10,3 +10,4 @@ tqdm
 prometheus-clien
 redis
 minio
+schedule==1.*

src/local/scheduler.py

@@ -0,0 +1,17 @@
+import schedule
+import time
+from src.infra.lambdas.RSSQueueFiller.lambda.lambda_function import handler
+
+
+def run_queue_filler():
+    """Invoke the queue filler lambda logic."""
+    handler(None, None)
+
+
+schedule.every(4).hours.do(run_queue_filler)
+
+
+if __name__ == "__main__":
+    while True:
+        schedule.run_pending()
+        time.sleep(1)

src/local/worker.py

@@ -0,0 +1,30 @@
+import os
+import sys
+import time
+import logging
+
+# Ensure project root is in the Python path so imports work when executed
+CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+PROJECT_ROOT = os.path.abspath(os.path.join(CURRENT_DIR, "..", ".."))
+if PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, PROJECT_ROOT)
+
+from src.infra.lambdas.RSSFeedProcessorLambda.src.lambda_function import lambda_handler
+
+logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"))
+logger = logging.getLogger(__name__)
+
+
+def main() -> None:
+    """Continuously run the existing Lambda handler as a local worker."""
+    logger.info("Starting local RSS worker")
+    while True:
+        try:
+            lambda_handler({}, None)
+        except Exception as exc:
+            logger.error("Worker iteration failed", exc_info=exc)
+        time.sleep(1)
+
+
+if __name__ == "__main__":
+    main()