mirror of
https://github.com/aljazceru/IngestRSS.git
synced 2025-12-17 05:54:22 +01:00
236 lines
8.2 KiB
Python
236 lines
8.2 KiB
Python
import boto3
|
|
import os
|
|
import zipfile
|
|
import io
|
|
import requests
|
|
import json
|
|
from botocore.exceptions import ClientError
|
|
from src.utils.retry_logic import retry_with_backoff
|
|
import time
|
|
import sys
|
|
from src.infra.deploy_infrastructure import get_or_create_kms_key
|
|
from dotenv import load_dotenv
|
|
load_dotenv(override=True)
|
|
|
|
import logging
|
|
logging.basicConfig(level=os.getenv('LOG_LEVEL', 'INFO'))
|
|
|
|
# Set variables
|
|
|
|
LAMBDA_NAME = os.getenv('LAMBDA_FUNCTION_NAME')
|
|
|
|
ACCOUNT_NUM = os.getenv('AWS_ACCOUNT_ID')
|
|
REGION = os.getenv("AWS_REGION")
|
|
LAMBDA_ROLE_ARN = os.getenv("LAMBDA_ROLE_ARN")
|
|
LAMBDA_TIMEOUT = int(os.getenv('LAMBDA_TIMEOUT'))
|
|
LAMBDA_MEMORY = int(os.getenv('LAMBDA_MEMORY'))
|
|
LAMBDA_RUNTIME = os.getenv('LAMBDA_RUNTIME')
|
|
LAMBDA_STACK_NAME = os.getenv("STACK_BASE") + f"-{LAMBDA_NAME}"
|
|
LAMBDA_HANDLER = "lambda_function.lambda_handler"
|
|
LAMBDA_LAYER_NAME = LAMBDA_NAME + "Layer"
|
|
S3_LAYER_KEY = os.getenv('S3_LAYER_KEY_NAME')+'.zip'
|
|
|
|
def zip_directory(path):
|
|
print(f"Creating deployment package from {path}...")
|
|
zip_buffer = io.BytesIO()
|
|
with zipfile.ZipFile(zip_buffer, 'a', zipfile.ZIP_DEFLATED, False) as zip_file:
|
|
for root, _, files in os.walk(path):
|
|
for file in files:
|
|
file_path = os.path.join(root, file)
|
|
arcname = os.path.relpath(file_path, path)
|
|
zip_file.write(file_path, arcname)
|
|
return zip_buffer.getvalue()
|
|
|
|
@retry_with_backoff()
|
|
def update_function_code(lambda_client, function_name, zip_file):
|
|
return lambda_client.update_function_code(
|
|
FunctionName=function_name,
|
|
ZipFile=zip_file
|
|
)
|
|
|
|
def get_or_create_lambda_layer():
|
|
layer_arn = os.getenv('LAMBDA_LAYER_ARN')
|
|
|
|
return layer_arn
|
|
|
|
@retry_with_backoff(max_retries=50, initial_backoff=5, backoff_multiplier=2) # Note: This function usually takes a long time to be successful.
|
|
def update_function_configuration(lambda_client, function_name, handler, role, timeout, memory, layers, kms_key_id):
|
|
|
|
config = {
|
|
'FunctionName': function_name,
|
|
'Handler': handler,
|
|
'Role': role,
|
|
'Timeout': timeout,
|
|
'MemorySize': memory,
|
|
'Layers': layers
|
|
}
|
|
|
|
|
|
if kms_key_id:
|
|
config['KMSKeyArn'] = f"arn:aws:kms:{REGION}:{ACCOUNT_NUM}:key/{kms_key_id}"
|
|
|
|
try:
|
|
response = lambda_client.update_function_configuration(**config)
|
|
print(f"Update request sent successfully for {function_name}.")
|
|
|
|
except ClientError as e:
|
|
if e.response['Error']['Code'] == 'ResourceConflictException':
|
|
logging.info(f"Function {function_name} is currently being updated. Retrying...")
|
|
raise e
|
|
|
|
@retry_with_backoff()
|
|
def configure_sqs_trigger(lambda_client, function_name, queue_arn):
|
|
event_source_mapping = {
|
|
'FunctionName': function_name,
|
|
'EventSourceArn': queue_arn,
|
|
'BatchSize': 1,
|
|
'MaximumBatchingWindowInSeconds': 0,
|
|
'ScalingConfig': {
|
|
'MaximumConcurrency': 50
|
|
}
|
|
}
|
|
|
|
try:
|
|
response = lambda_client.create_event_source_mapping(**event_source_mapping)
|
|
print(f"SQS trigger configured successfully for {function_name}")
|
|
except ClientError as e:
|
|
if e.response['Error']['Code'] == 'ResourceConflictException':
|
|
print(f"SQS trigger already exists for {function_name}. Updating configuration...")
|
|
# If you want to update existing trigger, you'd need to list existing mappings and update them
|
|
# This is left as an exercise as it requires additional error handling and logic
|
|
else:
|
|
raise e
|
|
|
|
@retry_with_backoff()
|
|
def create_function(lambda_client, function_name, runtime, role, handler, zip_file, timeout, memory, layers, kms_key_id, policy):
|
|
config = {
|
|
'FunctionName': function_name,
|
|
'Runtime': runtime,
|
|
'Role': role,
|
|
'Handler': handler,
|
|
'Code': {'ZipFile': zip_file},
|
|
'Timeout': timeout,
|
|
'MemorySize': memory,
|
|
'Layers': layers
|
|
}
|
|
print(policy)
|
|
|
|
if kms_key_id:
|
|
config['KMSKeyArn'] = f"arn:aws:kms:{REGION}:{ACCOUNT_NUM}:key/{kms_key_id}"
|
|
|
|
try:
|
|
return lambda_client.create_function(**config)
|
|
except ClientError as e:
|
|
if e.response['Error']['Code'] == 'InvalidParameterValueException':
|
|
print(f"Error creating function: {e}")
|
|
print("Ensure that the IAM role has the correct trust relationship and permissions.")
|
|
print("There might be a delay in role propagation. Please wait a few minutes and try again.")
|
|
raise
|
|
|
|
def get_pillow_layer_arn():
|
|
url = f"https://api.klayers.cloud/api/v2/p3.11/layers/latest/{os.getenv('AWS_REGION')}/json"
|
|
try:
|
|
response = requests.get(url)
|
|
response.raise_for_status()
|
|
layers_data = response.json()
|
|
|
|
pillow_layer = next((layer for layer in layers_data if layer['package'] == 'Pillow'), None)
|
|
|
|
if pillow_layer:
|
|
return pillow_layer['arn']
|
|
else:
|
|
print("Pillow layer not found in the API response.")
|
|
return None
|
|
except requests.RequestException as e:
|
|
print(f"Error fetching Pillow layer ARN: {e}")
|
|
return None
|
|
|
|
def get_lambda_policy():
|
|
policy = {
|
|
"Version": "2012-10-17",
|
|
"Statement": [
|
|
{
|
|
"Effect": "Allow",
|
|
"Action": [
|
|
"logs:CreateLogGroup",
|
|
"logs:CreateLogStream",
|
|
"logs:PutLogEvents"
|
|
],
|
|
"Resource": "arn:aws:logs:*:*:*"
|
|
},
|
|
{
|
|
"Effect": "Allow",
|
|
"Action": [
|
|
"s3:GetObject",
|
|
"s3:PutObject"
|
|
],
|
|
"Resource": "arn:aws:s3:::your-bucket-name/*"
|
|
}
|
|
]
|
|
}
|
|
|
|
def deploy_lambda():
|
|
lambda_client = boto3.client('lambda', region_name=REGION)
|
|
|
|
print(f"Starting deployment of Lambda function: {LAMBDA_NAME}")
|
|
deployment_package = zip_directory('src/infra/lambdas/RSSFeedProcessorLambda/src')
|
|
|
|
layer_arn = get_or_create_lambda_layer()
|
|
if layer_arn:
|
|
print(f"Using Lambda Layer ARN: {layer_arn}")
|
|
else:
|
|
print("Warning: Lambda Layer not found or created. Proceeding without Layer.")
|
|
|
|
pillow_layer_arn = get_pillow_layer_arn()
|
|
if pillow_layer_arn:
|
|
print(f"Using Pillow Layer ARN: {pillow_layer_arn}")
|
|
else:
|
|
print("Warning: Pillow Layer not found. Proceeding without Pillow Layer.")
|
|
|
|
kms_key_id = get_or_create_kms_key()
|
|
if kms_key_id:
|
|
print(f"Using KMS Key ID: {kms_key_id}")
|
|
else:
|
|
print("Warning: KMS Key not found or created. Proceeding without KMS Key.")
|
|
sys.exit(1)
|
|
|
|
try:
|
|
# Check if the function exists
|
|
try:
|
|
lambda_client.get_function(FunctionName=LAMBDA_NAME)
|
|
function_exists = True
|
|
except ClientError as e:
|
|
if e.response['Error']['Code'] == 'ResourceNotFoundException':
|
|
function_exists = False
|
|
else:
|
|
raise e
|
|
|
|
# Combine the layers
|
|
layers = [layer_arn] if layer_arn else []
|
|
if pillow_layer_arn:
|
|
layers.append(pillow_layer_arn)
|
|
|
|
if function_exists:
|
|
print("Updating existing Lambda function...")
|
|
update_function_configuration(lambda_client, LAMBDA_NAME, LAMBDA_HANDLER, LAMBDA_ROLE_ARN, LAMBDA_TIMEOUT, LAMBDA_MEMORY, layers, kms_key_id)
|
|
update_function_code(lambda_client, LAMBDA_NAME, deployment_package)
|
|
else:
|
|
print(f"Lambda function '{LAMBDA_NAME}' not found. Creating new function...")
|
|
policy = get_lambda_policy()
|
|
create_function(lambda_client, LAMBDA_NAME, LAMBDA_RUNTIME, LAMBDA_ROLE_ARN, LAMBDA_HANDLER, deployment_package, LAMBDA_TIMEOUT, LAMBDA_MEMORY, layers, kms_key_id, policy)
|
|
|
|
# Configure SQS trigger
|
|
queue_arn = os.getenv('SQS_QUEUE_ARN') # Make sure to set this environment variable
|
|
if queue_arn:
|
|
configure_sqs_trigger(lambda_client, LAMBDA_NAME, queue_arn)
|
|
else:
|
|
print("Warning: SQS_QUEUE_ARN not set. Skipping SQS trigger configuration.")
|
|
|
|
print("Lambda deployment completed successfully!")
|
|
|
|
except Exception as e:
|
|
print(f"Error during Lambda deployment: {str(e)}")
|
|
raise
|
|
|
|
if __name__ == "__main__":
|
|
deploy_lambda() |