aljaz/IngestRSS
1
0
Fork 0
mirror of https://github.com/aljazceru/IngestRSS.git synced 2025-12-17 22:14:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

testing.

Browse Source
This commit is contained in:
Charles E. Gormley
2024-09-08 01:36:30 -04:00
parent 85094f3ee1
commit 3cd0bcc412
5 changed files with 27 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
delete.json
View File

@@ -1,21 +0,0 @@
{
"Records": [
{
"messageId": "3214ab69-abb0-40c0-9065-2614c8da45b5",
"receiptHandle": "AQEBweJAoLGD9NDhm/bPl+LVdWp262pEnpVop1R+e8k//NeQ4T14AQsv2WTq/iwQt0/3bh5KGrBaqZHNwiUUXkd0b40NEO4lt/YWbZrSTVxdC1wcdYO8v2w5imd+rQ/+J6AlEXHHhnyqKkutwEKCA0QiMt6TfYON0sxZy83JRZmIWX6uE9zmejS92EYAwvMKGwKf45ASgiP4cwOptka4dj6NhAOP/cC8ygt7cvZtpyC7WcwBAak/XuAI1JeObUtOXKSdglUo4o46uGiNIiwwnXkuuw0A0sYcCxPFIO/8GpgWHmCQdgsPYFsZBjFNf0/pdldplOJhjjzxqdadEEqlKoxEtvmouS5SQURubUxP3X/tq3+Waljqmf1Hre97ASNG4+6pYbMyXP/uuv+NMSnoJujiuA==",
"body": "{"u": "https://www.sciencedaily.com/rss/all.xml", "dt": 0}",
"attributes": {
"ApproximateReceiveCount": "1",
"AWSTraceHeader": "Root=1-66d7944f-3d6dc8822638689228df26cf;Parent=3e6b55a229989a9a;Sampled=0;Lineage=11d5b652:0",
"SentTimestamp": "1725404240031",
"SenderId": "AROA6B6PHGPNYVX6EREKP:RSSQueueFiller",
"ApproximateFirstReceiveTimestamp": "1725404240032"
},
"messageAttributes": {},
"md5OfBody": "5cc09387bc0d19c9bfead740d1aba6cf",
"eventSource": "aws:sqs",
"eventSourceARN": "arn:aws:sqs:us-east-1:966265353179:rss-feed-queue",
"awsRegion": "us-east-1"
}
]
}

25
launch.py
View File

@@ -5,42 +5,35 @@ import boto3
from dotenv import load_dotenv
import logging
from src.infra.lambdas.RSSQueueFiller.deploy_sqs_filler_lambda import deploy_sqs_filler
from src.infra.deploy_infrastructure import deploy_infrastructure
from src.infra.lambdas.RSSFeedProcessorLambda.deploy_rss_feed_lambda import deploy_lambda
from src.infra.lambdas.lambda_utils.update_lambda_env_vars import update_env_vars
from src.feed_management.upload_rss_feeds import upload_rss_feeds
# Load environment variables
load_dotenv()
load_dotenv(override=True)
# Set up logging
logging.basicConfig(level=os.getenv('LOG_LEVEL'))
lambda_client = boto3.client("lambda")
# Set AWS credentials from environment variables
os.environ['AWS_ACCESS_KEY_ID'] = os.getenv('AWS_ACCESS_KEY_ID')
os.environ['AWS_SECRET_ACCESS_KEY'] = os.getenv('AWS_SECRET_ACCESS_KEY')
os.environ['AWS_DEFAULT_REGION'] = os.getenv('AWS_REGION')
TABLE_NAME = os.getenv('DYNAMODB_TABLE_NAME')
ACCOUNT_NUM = os.getenv("AWS_ACCOUNT_ID")
SQS_QUEUE_NAME = os.getenv("SQS_QUEUE_NAME")
REGION = os.getenv("AWS_REGION")
os.environ["SQS_QUEUE_URL"] = f"https://sqs.{REGION}.amazonaws.com/{ACCOUNT_NUM}/{SQS_QUEUE_NAME}"
lambda_client = boto3.client("lambda")
LAMBDA_FUNCTION_NAME = os.getenv("LAMBDA_FUNCTION_NAME")
# Add the src directory to the Python path
current_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(current_dir)
from src.infra.deploy_infrastructure import deploy_infrastructure
from src.infra.lambdas.RSSFeedProcessorLambda.deploy_rss_feed_lambda import deploy_lambda
from src.infra.lambdas.lambda_utils.update_lambda_env_vars import update_env_vars
from src.feed_management.upload_rss_feeds import upload_rss_feeds
def main():
# Deploy infrastructure
deploy_infrastructure()
logging.info("Finished Deploying Infrastructure")
# Deploy Lambda function
deploy_lambda()
logging.info("Finished Deploying Lambda")
@@ -49,7 +42,7 @@ def main():
logging.info("Finished Deploying SQS Filler Lambda")
# Update Lambda environment variables
update_env_vars(LAMBDA_FUNCTION_NAME)
update_env_vars(os.getenv("LAMBDA_FUNCTION_NAME"))
print("Finished Environment Variable Updates")

19
src/infra/deploy_infrastructure.py
View File

@@ -16,7 +16,7 @@ stack_base = os.getenv("STACK_BASE")
def deploy_cloudformation(template_file, stack_suffix, force_recreate=False, parameters=[]):
cf_client = boto3.client('cloudformation')
stack_name = f"{stack_base}-{stack_suffix}"
print(stack_name)
with open(f'src/infra/cloudformation/{template_file}', 'r') as file:
template_body = file.read()
@@ -142,13 +142,6 @@ def deploy_infrastructure():
key_info = kms_client.describe_key(KeyId=kms_key_id)
kms_key_arn = key_info['KeyMetadata']['Arn']
deploy_cloudformation('s3.yaml', 'S3',
parameters=[
{
'ParameterKey': 'BucketName',
'ParameterValue': os.getenv('S3_BUCKET_NAME')
}
])
deploy_cloudformation('dynamo.yaml', 'DynamoDB',
parameters=[
{
@@ -156,6 +149,16 @@ def deploy_infrastructure():
'ParameterValue': os.environ.get('DYNAMODB_TABLE_NAME', 'default-table-name')
}
])
deploy_cloudformation('s3.yaml', 'S3',
parameters=[
{
'ParameterKey': 'BucketName',
'ParameterValue': os.getenv('S3_BUCKET_NAME')
}
])
deploy_cloudformation('sqs.yaml', 'SQS',
parameters=[
{

2
src/infra/lambdas/RSSFeedProcessorLambda/deploy_rss_feed_lambda.py
View File

@@ -118,7 +118,7 @@ def create_function(lambda_client, function_name, runtime, role, handler, zip_fi
return lambda_client.create_function(**config)
def get_pillow_layer_arn():
url = "https://api.klayers.cloud/api/v2/p3.11/layers/latest/us-east-1/json"
url = f"https://api.klayers.cloud/api/v2/p3.11/layers/latest/{os.getenv('AWS_REGION')}/json"
try:
response = requests.get(url)
response.raise_for_status()

7
todo.md
View File

@@ -2,11 +2,14 @@
* Test Creation
* Add in console setup python script for new project into launch.py
* Better Readme.md
* Change name
* Update Lambda Layer Creation Script to be more comprehensive.
** Make a layer for every region in existance.
** Save this script.
** Update the layer references with os.getenv("AWS_REGION")
# Misc TODO:
* More RSS Feed Module - Easy
* Update Lambda Layer Creation Script to be more comprehensive.
# Next Modules