diff --git a/launch.py b/launch.py index 0c22299..668857f 100644 --- a/launch.py +++ b/launch.py @@ -5,17 +5,65 @@ import subprocess import boto3 from dotenv import load_dotenv import logging +import argparse +import subprocess from src.infra.lambdas.RSSQueueFiller.deploy_sqs_filler_lambda import deploy_sqs_filler from src.utils.check_env import check_env + +def check_local_env() -> None: + """Ensure required environment variables for local mode are set.""" + required_vars = [ + "MONGODB_URL", + "MONGODB_DB_NAME", + "MONGODB_COLLECTION_NAME", + "REDIS_URL", + "REDIS_QUEUE_NAME", + "MINIO_ENDPOINT", + "MINIO_ACCESS_KEY", + "MINIO_SECRET_KEY", + "MINIO_BUCKET", + ] + + missing = [var for var in required_vars if not os.getenv(var)] + if missing: + raise EnvironmentError( + f"Missing required environment variables for local mode: {', '.join(missing)}" + ) + + +def start_docker_containers() -> None: + """Start required Docker containers for local development.""" + try: + subprocess.check_call(["docker-compose", "up", "-d"]) + except FileNotFoundError: + # fallback to `docker compose` if docker-compose command not found + subprocess.check_call(["docker", "compose", "up", "-d"]) + except Exception as exc: + logging.error(f"Failed to start Docker containers: {exc}") + raise + + print("🗞️ 💵 ⚖️ IngestRSS⚖️ 💵 🗞️".center(100, "-")) +parser = argparse.ArgumentParser(description="Launch IngestRSS") +parser.add_argument( + "--local", + action="store_true", + help="Run locally using Docker instead of deploying to AWS", +) +args = parser.parse_args() + load_dotenv(override=True) -check_env() + +if args.local: + check_local_env() +else: + check_env() # Set up logging -logging.basicConfig(level=os.getenv('LOG_LEVEL')) +logging.basicConfig(level=os.getenv("LOG_LEVEL")) lambda_client = boto3.client("lambda") @@ -23,7 +71,7 @@ lambda_client = boto3.client("lambda") current_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(current_dir) -from src.infra.deploy_infrastructure import deploy_infrastructure +from src.infra.import deploy_infrastructure from src.infra.lambdas.RSSFeedProcessorLambda.deploy_rss_feed_lambda import deploy_lambda from src.infra.lambdas.lambda_utils.update_lambda_env_vars import update_env_vars from src.feed_management.upload_rss_feeds import upload_rss_feeds @@ -47,7 +95,7 @@ def main(): # Update Lambda environment variables update_env_vars(os.getenv("LAMBDA_FUNCTION_NAME")) print("Finished Environment Variable Updates") - + # Upload RSS feeds rss_feeds_file = os.path.join(current_dir, "rss_feeds.json") if os.path.exists(rss_feeds_file): @@ -66,4 +114,4 @@ def main(): print("RSS Feed Processor launched successfully!") if __name__ == "__main__": - main() \ No newline at end of file + main(args.local) diff --git a/local.env.template b/local.env.template new file mode 100644 index 0000000..6e1b88c --- /dev/null +++ b/local.env.template @@ -0,0 +1,25 @@ +# Local environment configuration for IngestRSS + +# Redis configuration +REDIS_URL=redis://localhost:6379 +REDIS_QUEUE_NAME=rss-feed-queue + +# MinIO configuration +MINIO_ENDPOINT=*** +MINIO_ACCESS_KEY=*** +MINIO_SECRET_KEY=*** +MINIO_BUCKET=*** + +# MongoDB settings +MONGODB_URL=mongodb://localhost:27017 +MONGODB_DB_NAME=ingestrss +MONGODB_COLLECTION_NAME=rss_feeds + +# Logging Configuration +LOG_LEVEL=INFO + +# Other Application Settings +APP_NAME=RSS Feed Processor +VERSION=1.0.0 + +STORAGE_STRATEGY=s3 diff --git a/requirements.txt b/requirements.txt index c46cb06..8520887 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ tqdm prometheus-clien redis minio +schedule==1.* diff --git a/src/local/__init__.py b/src/local/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/local/scheduler.py b/src/local/scheduler.py new file mode 100644 index 0000000..bf3a3ec --- /dev/null +++ b/src/local/scheduler.py @@ -0,0 +1,17 @@ +import schedule +import time +from src.infra.lambdas.RSSQueueFiller.lambda.lambda_function import handler + + +def run_queue_filler(): + """Invoke the queue filler lambda logic.""" + handler(None, None) + + +schedule.every(4).hours.do(run_queue_filler) + + +if __name__ == "__main__": + while True: + schedule.run_pending() + time.sleep(1) diff --git a/src/local/worker.py b/src/local/worker.py new file mode 100644 index 0000000..1ab4ea9 --- /dev/null +++ b/src/local/worker.py @@ -0,0 +1,30 @@ +import os +import sys +import time +import logging + +# Ensure project root is in the Python path so imports work when executed +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) +PROJECT_ROOT = os.path.abspath(os.path.join(CURRENT_DIR, "..", "..")) +if PROJECT_ROOT not in sys.path: + sys.path.insert(0, PROJECT_ROOT) + +from src.infra.lambdas.RSSFeedProcessorLambda.src.lambda_function import lambda_handler + +logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO")) +logger = logging.getLogger(__name__) + + +def main() -> None: + """Continuously run the existing Lambda handler as a local worker.""" + logger.info("Starting local RSS worker") + while True: + try: + lambda_handler({}, None) + except Exception as exc: + logger.error("Worker iteration failed", exc_info=exc) + time.sleep(1) + + +if __name__ == "__main__": + main()