diff --git a/delete.json b/delete.json deleted file mode 100644 index f7582f5..0000000 --- a/delete.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "Records": [ - { - "messageId": "3214ab69-abb0-40c0-9065-2614c8da45b5", - "receiptHandle": "AQEBweJAoLGD9NDhm/bPl+LVdWp262pEnpVop1R+e8k//NeQ4T14AQsv2WTq/iwQt0/3bh5KGrBaqZHNwiUUXkd0b40NEO4lt/YWbZrSTVxdC1wcdYO8v2w5imd+rQ/+J6AlEXHHhnyqKkutwEKCA0QiMt6TfYON0sxZy83JRZmIWX6uE9zmejS92EYAwvMKGwKf45ASgiP4cwOptka4dj6NhAOP/cC8ygt7cvZtpyC7WcwBAak/XuAI1JeObUtOXKSdglUo4o46uGiNIiwwnXkuuw0A0sYcCxPFIO/8GpgWHmCQdgsPYFsZBjFNf0/pdldplOJhjjzxqdadEEqlKoxEtvmouS5SQURubUxP3X/tq3+Waljqmf1Hre97ASNG4+6pYbMyXP/uuv+NMSnoJujiuA==", - "body": "{"u": "https://www.sciencedaily.com/rss/all.xml", "dt": 0}", - "attributes": { - "ApproximateReceiveCount": "1", - "AWSTraceHeader": "Root=1-66d7944f-3d6dc8822638689228df26cf;Parent=3e6b55a229989a9a;Sampled=0;Lineage=11d5b652:0", - "SentTimestamp": "1725404240031", - "SenderId": "AROA6B6PHGPNYVX6EREKP:RSSQueueFiller", - "ApproximateFirstReceiveTimestamp": "1725404240032" - }, - "messageAttributes": {}, - "md5OfBody": "5cc09387bc0d19c9bfead740d1aba6cf", - "eventSource": "aws:sqs", - "eventSourceARN": "arn:aws:sqs:us-east-1:966265353179:rss-feed-queue", - "awsRegion": "us-east-1" - } - ] -} \ No newline at end of file diff --git a/launch.py b/launch.py index 506e156..22f65f1 100644 --- a/launch.py +++ b/launch.py @@ -5,41 +5,34 @@ import boto3 from dotenv import load_dotenv import logging from src.infra.lambdas.RSSQueueFiller.deploy_sqs_filler_lambda import deploy_sqs_filler +from src.infra.deploy_infrastructure import deploy_infrastructure +from src.infra.lambdas.RSSFeedProcessorLambda.deploy_rss_feed_lambda import deploy_lambda +from src.infra.lambdas.lambda_utils.update_lambda_env_vars import update_env_vars +from src.feed_management.upload_rss_feeds import upload_rss_feeds # Load environment variables -load_dotenv() +load_dotenv(override=True) # Set up logging logging.basicConfig(level=os.getenv('LOG_LEVEL')) +lambda_client = boto3.client("lambda") + # Set AWS credentials from environment variables -os.environ['AWS_ACCESS_KEY_ID'] = os.getenv('AWS_ACCESS_KEY_ID') -os.environ['AWS_SECRET_ACCESS_KEY'] = os.getenv('AWS_SECRET_ACCESS_KEY') -os.environ['AWS_DEFAULT_REGION'] = os.getenv('AWS_REGION') TABLE_NAME = os.getenv('DYNAMODB_TABLE_NAME') ACCOUNT_NUM = os.getenv("AWS_ACCOUNT_ID") SQS_QUEUE_NAME = os.getenv("SQS_QUEUE_NAME") REGION = os.getenv("AWS_REGION") -os.environ["SQS_QUEUE_URL"] = f"https://sqs.{REGION}.amazonaws.com/{ACCOUNT_NUM}/{SQS_QUEUE_NAME}" - - -lambda_client = boto3.client("lambda") -LAMBDA_FUNCTION_NAME = os.getenv("LAMBDA_FUNCTION_NAME") - # Add the src directory to the Python path current_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(current_dir) -from src.infra.deploy_infrastructure import deploy_infrastructure -from src.infra.lambdas.RSSFeedProcessorLambda.deploy_rss_feed_lambda import deploy_lambda -from src.infra.lambdas.lambda_utils.update_lambda_env_vars import update_env_vars -from src.feed_management.upload_rss_feeds import upload_rss_feeds + def main(): # Deploy infrastructure deploy_infrastructure() logging.info("Finished Deploying Infrastructure") - # Deploy Lambda function deploy_lambda() @@ -49,7 +42,7 @@ def main(): logging.info("Finished Deploying SQS Filler Lambda") # Update Lambda environment variables - update_env_vars(LAMBDA_FUNCTION_NAME) + update_env_vars(os.getenv("LAMBDA_FUNCTION_NAME")) print("Finished Environment Variable Updates") diff --git a/src/infra/deploy_infrastructure.py b/src/infra/deploy_infrastructure.py index 5b14ec2..eb8d991 100644 --- a/src/infra/deploy_infrastructure.py +++ b/src/infra/deploy_infrastructure.py @@ -16,7 +16,7 @@ stack_base = os.getenv("STACK_BASE") def deploy_cloudformation(template_file, stack_suffix, force_recreate=False, parameters=[]): cf_client = boto3.client('cloudformation') stack_name = f"{stack_base}-{stack_suffix}" - print(stack_name) + with open(f'src/infra/cloudformation/{template_file}', 'r') as file: template_body = file.read() @@ -141,14 +141,7 @@ def deploy_infrastructure(): key_info = kms_client.describe_key(KeyId=kms_key_id) kms_key_arn = key_info['KeyMetadata']['Arn'] - - deploy_cloudformation('s3.yaml', 'S3', - parameters=[ - { - 'ParameterKey': 'BucketName', - 'ParameterValue': os.getenv('S3_BUCKET_NAME') - } - ]) + deploy_cloudformation('dynamo.yaml', 'DynamoDB', parameters=[ { @@ -156,6 +149,16 @@ def deploy_infrastructure(): 'ParameterValue': os.environ.get('DYNAMODB_TABLE_NAME', 'default-table-name') } ]) + + + deploy_cloudformation('s3.yaml', 'S3', + parameters=[ + { + 'ParameterKey': 'BucketName', + 'ParameterValue': os.getenv('S3_BUCKET_NAME') + } + ]) + deploy_cloudformation('sqs.yaml', 'SQS', parameters=[ { diff --git a/src/infra/lambdas/RSSFeedProcessorLambda/deploy_rss_feed_lambda.py b/src/infra/lambdas/RSSFeedProcessorLambda/deploy_rss_feed_lambda.py index 825f41f..1c05da2 100644 --- a/src/infra/lambdas/RSSFeedProcessorLambda/deploy_rss_feed_lambda.py +++ b/src/infra/lambdas/RSSFeedProcessorLambda/deploy_rss_feed_lambda.py @@ -118,7 +118,7 @@ def create_function(lambda_client, function_name, runtime, role, handler, zip_fi return lambda_client.create_function(**config) def get_pillow_layer_arn(): - url = "https://api.klayers.cloud/api/v2/p3.11/layers/latest/us-east-1/json" + url = f"https://api.klayers.cloud/api/v2/p3.11/layers/latest/{os.getenv('AWS_REGION')}/json" try: response = requests.get(url) response.raise_for_status() diff --git a/todo.md b/todo.md index b2fda23..a70643e 100644 --- a/todo.md +++ b/todo.md @@ -2,11 +2,14 @@ * Test Creation * Add in console setup python script for new project into launch.py * Better Readme.md -* Change name +* Update Lambda Layer Creation Script to be more comprehensive. +** Make a layer for every region in existance. +** Save this script. +** Update the layer references with os.getenv("AWS_REGION") # Misc TODO: * More RSS Feed Module - Easy -* Update Lambda Layer Creation Script to be more comprehensive. + # Next Modules