mirror of
https://github.com/aljazceru/IngestRSS.git
synced 2025-12-17 14:04:19 +01:00
finalized product.
This commit is contained in:
@@ -5,10 +5,6 @@ import boto3
|
|||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
import logging
|
import logging
|
||||||
from src.infra.lambdas.RSSQueueFiller.deploy_sqs_filler_lambda import deploy_sqs_filler
|
from src.infra.lambdas.RSSQueueFiller.deploy_sqs_filler_lambda import deploy_sqs_filler
|
||||||
from src.infra.deploy_infrastructure import deploy_infrastructure
|
|
||||||
from src.infra.lambdas.RSSFeedProcessorLambda.deploy_rss_feed_lambda import deploy_lambda
|
|
||||||
from src.infra.lambdas.lambda_utils.update_lambda_env_vars import update_env_vars
|
|
||||||
from src.feed_management.upload_rss_feeds import upload_rss_feeds
|
|
||||||
# Load environment variables
|
# Load environment variables
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
@@ -27,7 +23,10 @@ REGION = os.getenv("AWS_REGION")
|
|||||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||||
sys.path.append(current_dir)
|
sys.path.append(current_dir)
|
||||||
|
|
||||||
|
from src.infra.deploy_infrastructure import deploy_infrastructure
|
||||||
|
from src.infra.lambdas.RSSFeedProcessorLambda.deploy_rss_feed_lambda import deploy_lambda
|
||||||
|
from src.infra.lambdas.lambda_utils.update_lambda_env_vars import update_env_vars
|
||||||
|
from src.feed_management.upload_rss_feeds import upload_rss_feeds
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
# Deploy infrastructure
|
# Deploy infrastructure
|
||||||
|
|||||||
@@ -8,6 +8,10 @@ Parameters:
|
|||||||
LambdaKMSKeyArn:
|
LambdaKMSKeyArn:
|
||||||
Type: String
|
Type: String
|
||||||
Description: "ARN of the KMS Key for Lambda environment variable encryption"
|
Description: "ARN of the KMS Key for Lambda environment variable encryption"
|
||||||
|
Region:
|
||||||
|
Type: String
|
||||||
|
Description: "AWS Region for deployment"
|
||||||
|
Default: "us-east-1"
|
||||||
|
|
||||||
Resources:
|
Resources:
|
||||||
LambdaExecutionRole:
|
LambdaExecutionRole:
|
||||||
@@ -57,4 +61,9 @@ Outputs:
|
|||||||
Description: 'ARN of the KMS Key for Lambda'
|
Description: 'ARN of the KMS Key for Lambda'
|
||||||
Value: !Ref LambdaKMSKeyArn
|
Value: !Ref LambdaKMSKeyArn
|
||||||
Export:
|
Export:
|
||||||
Name: !Sub '${AWS::StackName}-LambdaKMSKeyArn'
|
Name: !Sub '${AWS::StackName}-LambdaKMSKeyArn'
|
||||||
|
Region:
|
||||||
|
Description: 'AWS Region for deployment'
|
||||||
|
Value: !Ref Region
|
||||||
|
Export:
|
||||||
|
Name: !Sub '${AWS::StackName}-Region'
|
||||||
@@ -9,6 +9,8 @@ from src.utils.retry_logic import retry_with_backoff
|
|||||||
import time
|
import time
|
||||||
import sys
|
import sys
|
||||||
from src.infra.deploy_infrastructure import get_or_create_kms_key
|
from src.infra.deploy_infrastructure import get_or_create_kms_key
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
load_dotenv()
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
logging.basicConfig(level=os.getenv('LOG_LEVEL', 'INFO'))
|
logging.basicConfig(level=os.getenv('LOG_LEVEL', 'INFO'))
|
||||||
@@ -18,6 +20,7 @@ logging.basicConfig(level=os.getenv('LOG_LEVEL', 'INFO'))
|
|||||||
LAMBDA_NAME = os.getenv('LAMBDA_FUNCTION_NAME')
|
LAMBDA_NAME = os.getenv('LAMBDA_FUNCTION_NAME')
|
||||||
|
|
||||||
ACCOUNT_NUM = os.getenv('AWS_ACCOUNT_ID')
|
ACCOUNT_NUM = os.getenv('AWS_ACCOUNT_ID')
|
||||||
|
REGION = os.getenv("AWS_REGION")
|
||||||
LAMBDA_ROLE_ARN = os.getenv("LAMBDA_ROLE_ARN")
|
LAMBDA_ROLE_ARN = os.getenv("LAMBDA_ROLE_ARN")
|
||||||
LAMBDA_TIMEOUT = int(os.getenv('LAMBDA_TIMEOUT'))
|
LAMBDA_TIMEOUT = int(os.getenv('LAMBDA_TIMEOUT'))
|
||||||
LAMBDA_MEMORY = int(os.getenv('LAMBDA_MEMORY'))
|
LAMBDA_MEMORY = int(os.getenv('LAMBDA_MEMORY'))
|
||||||
@@ -63,18 +66,18 @@ def update_function_configuration(lambda_client, function_name, handler, role, t
|
|||||||
'Layers': layers
|
'Layers': layers
|
||||||
}
|
}
|
||||||
|
|
||||||
if kms_key_id:
|
|
||||||
config['KMSKeyArn'] = f"arn:aws:kms:{os.environ['AWS_REGION']}:{ACCOUNT_NUM}:key/{kms_key_id}"
|
|
||||||
|
|
||||||
|
if kms_key_id:
|
||||||
|
config['KMSKeyArn'] = f"arn:aws:kms:{REGION}:{ACCOUNT_NUM}:key/{kms_key_id}"
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = lambda_client.update_function_configuration(**config)
|
response = lambda_client.update_function_configuration(**config)
|
||||||
print(f"Update request sent successfully for {function_name}.")
|
print(f"Update request sent successfully for {function_name}.")
|
||||||
|
|
||||||
except ClientError as e:
|
except ClientError as e:
|
||||||
if e.response['Error']['Code'] == 'ResourceConflictException':
|
if e.response['Error']['Code'] == 'ResourceConflictException':
|
||||||
logging.info(f"Function {function_name} is currently being updated. Retrying...")
|
logging.info(f"Function {function_name} is currently being updated. Retrying...")
|
||||||
raise e
|
raise e
|
||||||
|
|
||||||
@retry_with_backoff()
|
@retry_with_backoff()
|
||||||
def configure_sqs_trigger(lambda_client, function_name, queue_arn):
|
def configure_sqs_trigger(lambda_client, function_name, queue_arn):
|
||||||
@@ -100,7 +103,7 @@ def configure_sqs_trigger(lambda_client, function_name, queue_arn):
|
|||||||
raise e
|
raise e
|
||||||
|
|
||||||
@retry_with_backoff()
|
@retry_with_backoff()
|
||||||
def create_function(lambda_client, function_name, runtime, role, handler, zip_file, timeout, memory, layers, kms_key_id):
|
def create_function(lambda_client, function_name, runtime, role, handler, zip_file, timeout, memory, layers, kms_key_id, policy):
|
||||||
config = {
|
config = {
|
||||||
'FunctionName': function_name,
|
'FunctionName': function_name,
|
||||||
'Runtime': runtime,
|
'Runtime': runtime,
|
||||||
@@ -111,11 +114,19 @@ def create_function(lambda_client, function_name, runtime, role, handler, zip_fi
|
|||||||
'MemorySize': memory,
|
'MemorySize': memory,
|
||||||
'Layers': layers
|
'Layers': layers
|
||||||
}
|
}
|
||||||
|
print(policy)
|
||||||
|
|
||||||
if kms_key_id:
|
if kms_key_id:
|
||||||
config['KMSKeyArn'] = f"arn:aws:kms:{os.environ['AWS_DEFAULT_REGION']}:{ACCOUNT_NUM}:key/{kms_key_id}"
|
config['KMSKeyArn'] = f"arn:aws:kms:{REGION}:{ACCOUNT_NUM}:key/{kms_key_id}"
|
||||||
|
|
||||||
return lambda_client.create_function(**config)
|
try:
|
||||||
|
return lambda_client.create_function(**config)
|
||||||
|
except ClientError as e:
|
||||||
|
if e.response['Error']['Code'] == 'InvalidParameterValueException':
|
||||||
|
print(f"Error creating function: {e}")
|
||||||
|
print("Ensure that the IAM role has the correct trust relationship and permissions.")
|
||||||
|
print("There might be a delay in role propagation. Please wait a few minutes and try again.")
|
||||||
|
raise
|
||||||
|
|
||||||
def get_pillow_layer_arn():
|
def get_pillow_layer_arn():
|
||||||
url = f"https://api.klayers.cloud/api/v2/p3.11/layers/latest/{os.getenv('AWS_REGION')}/json"
|
url = f"https://api.klayers.cloud/api/v2/p3.11/layers/latest/{os.getenv('AWS_REGION')}/json"
|
||||||
@@ -134,9 +145,33 @@ def get_pillow_layer_arn():
|
|||||||
except requests.RequestException as e:
|
except requests.RequestException as e:
|
||||||
print(f"Error fetching Pillow layer ARN: {e}")
|
print(f"Error fetching Pillow layer ARN: {e}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def get_lambda_policy():
|
||||||
|
policy = {
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": [
|
||||||
|
"logs:CreateLogGroup",
|
||||||
|
"logs:CreateLogStream",
|
||||||
|
"logs:PutLogEvents"
|
||||||
|
],
|
||||||
|
"Resource": "arn:aws:logs:*:*:*"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": [
|
||||||
|
"s3:GetObject",
|
||||||
|
"s3:PutObject"
|
||||||
|
],
|
||||||
|
"Resource": "arn:aws:s3:::your-bucket-name/*"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
def deploy_lambda():
|
def deploy_lambda():
|
||||||
lambda_client = boto3.client('lambda')
|
lambda_client = boto3.client('lambda', region_name=REGION)
|
||||||
|
|
||||||
print(f"Starting deployment of Lambda function: {LAMBDA_NAME}")
|
print(f"Starting deployment of Lambda function: {LAMBDA_NAME}")
|
||||||
deployment_package = zip_directory('src/infra/lambdas/RSSFeedProcessorLambda/src')
|
deployment_package = zip_directory('src/infra/lambdas/RSSFeedProcessorLambda/src')
|
||||||
@@ -182,7 +217,8 @@ def deploy_lambda():
|
|||||||
update_function_code(lambda_client, LAMBDA_NAME, deployment_package)
|
update_function_code(lambda_client, LAMBDA_NAME, deployment_package)
|
||||||
else:
|
else:
|
||||||
print(f"Lambda function '{LAMBDA_NAME}' not found. Creating new function...")
|
print(f"Lambda function '{LAMBDA_NAME}' not found. Creating new function...")
|
||||||
create_function(lambda_client, LAMBDA_NAME, LAMBDA_RUNTIME, LAMBDA_ROLE_ARN, LAMBDA_HANDLER, deployment_package, LAMBDA_TIMEOUT, LAMBDA_MEMORY, layers, kms_key_id)
|
policy = get_lambda_policy()
|
||||||
|
create_function(lambda_client, LAMBDA_NAME, LAMBDA_RUNTIME, LAMBDA_ROLE_ARN, LAMBDA_HANDLER, deployment_package, LAMBDA_TIMEOUT, LAMBDA_MEMORY, layers, kms_key_id, policy)
|
||||||
|
|
||||||
# Configure SQS trigger
|
# Configure SQS trigger
|
||||||
queue_arn = os.getenv('SQS_QUEUE_ARN') # Make sure to set this environment variable
|
queue_arn = os.getenv('SQS_QUEUE_ARN') # Make sure to set this environment variable
|
||||||
|
|||||||
@@ -1,98 +1,117 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# TODO: This needs to be completely overhauled
|
|
||||||
# Update system packages
|
|
||||||
echo "Updating system packages..."
|
|
||||||
sudo yum update -y
|
|
||||||
|
|
||||||
# Install development tools
|
set -e
|
||||||
echo "Installing development tools..."
|
|
||||||
sudo yum groupinstall "Development Tools" -y
|
|
||||||
|
|
||||||
# Install Python 3.11
|
####### Section 1: Checking Python Existence ########
|
||||||
echo "Installing Python 3.11..."
|
echo "Section 1: Checking Python Existence"
|
||||||
sudo amazon-linux-extras enable python3.11
|
|
||||||
sudo yum install python3.11 -y
|
|
||||||
|
|
||||||
# Verify Python 3.11 installation
|
# Ensure python3.12 is installed
|
||||||
if command -v python3.11 &>/dev/null; then
|
if ! command -v python3.12 &> /dev/null; then
|
||||||
echo "Python 3.11 installed successfully:"
|
echo "Python 3.12 is not installed. Please install it before running this script."
|
||||||
python3.11 --version
|
|
||||||
else
|
|
||||||
echo "Failed to install Python 3.11. Exiting."
|
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
echo "Python 3.12 found. Proceeding..."
|
||||||
|
|
||||||
# Install pip for Python 3.11
|
####### Section 2: Installing Dependencies ########
|
||||||
echo "Installing pip for Python 3.11..."
|
echo "Section 2: Installing Dependencies"
|
||||||
sudo python3.11 -m ensurepip --upgrade
|
|
||||||
|
|
||||||
# Verify pip installation
|
# Install dependencies
|
||||||
if command -v pip3.11 &>/dev/null; then
|
python3.12 -m pip install --upgrade Pillow feedfinder2==0.0.4 python-dateutil newspaper3k==0.2.8 feedparser lxml[html5lib] lxml_html_clean lxml[html_clean] -t python/
|
||||||
echo "pip installed successfully:"
|
echo "Dependencies installed successfully."
|
||||||
pip3.11 --version
|
|
||||||
else
|
|
||||||
echo "Failed to install pip. Exiting."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Create directory for Lambda layer
|
####### Section 3: Creating ZIP File ########
|
||||||
echo "Creating directory for Lambda layer..."
|
echo "Section 3: Creating ZIP File"
|
||||||
mkdir -p OpenRSSLambdaLayer/python
|
|
||||||
cd OpenRSSLambdaLayer
|
|
||||||
|
|
||||||
# Install packages
|
|
||||||
echo "Installing packages..."
|
|
||||||
pip3.11 install newspaper3k feedparser python-dateutil-t python/
|
|
||||||
|
|
||||||
# Create ZIP file
|
# Create ZIP file
|
||||||
echo "Creating ZIP file..."
|
|
||||||
zip -r OpenRSSLambdaLayer.zip python/
|
zip -r OpenRSSLambdaLayer.zip python/
|
||||||
|
echo "ZIP file created."
|
||||||
|
|
||||||
# Upload to S3
|
# Check if ZIP file was created and is not empty
|
||||||
echo "Uploading to S3..."
|
if [ ! -s OpenRSSLambdaLayer.zip ]; then
|
||||||
aws s3 cp OpenRSSLambdaLayer.zip s3://rss-feed-processor-layers/OpenRSSLambdaLayer.zip
|
echo "Error: ZIP file is empty or was not created."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "ZIP file check passed."
|
||||||
|
|
||||||
# Create Lambda layer
|
####### Section 4: Getting AWS Regions ########
|
||||||
echo "Creating Lambda layer..."
|
echo "Section 4: Getting AWS Regions"
|
||||||
LAYER_VERSION=$(aws lambda publish-layer-version \
|
|
||||||
--layer-name OpenRSSLambdaLayer \
|
|
||||||
--description "Layer with dependencies for RSS processing" \
|
|
||||||
--license-info "MIT" \
|
|
||||||
--content S3Bucket=rss-feed-processor-layers,S3Key=OpenRSSLambdaLayer.zip \
|
|
||||||
--compatible-runtimes python3.11 \
|
|
||||||
--query 'Version' \
|
|
||||||
--output text)
|
|
||||||
|
|
||||||
# Make layer public
|
# Get list of all AWS regions
|
||||||
echo "Making layer public..."
|
REGIONS=$(aws ec2 describe-regions --query 'Regions[].RegionName' --output text)
|
||||||
aws lambda add-layer-version-permission \
|
echo "Retrieved AWS regions: $REGIONS"
|
||||||
--layer-name OpenRSSLambdaLayer \
|
|
||||||
--version-number $LAYER_VERSION \
|
|
||||||
--statement-id public \
|
|
||||||
--action lambda:GetLayerVersion \
|
|
||||||
--principal '*'
|
|
||||||
|
|
||||||
# Calculate and print the ARN
|
####### Section 5: Creating Buckets, Uploading, and Publishing Layer ########
|
||||||
REGION=$(aws configure get region)
|
echo "Section 5: Creating Buckets, Uploading, and Publishing Layer"
|
||||||
ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
|
|
||||||
ARN="arn:aws:lambda:${REGION}:${ACCOUNT_ID}:layer:OpenRSSLambdaLayer:${LAYER_VERSION}"
|
|
||||||
|
|
||||||
echo "Setup complete! OpenRSSLambdaLayer is now available to anyone on the internet."
|
create_bucket_upload_and_publish_layer() {
|
||||||
echo "Layer ARN: $ARN"
|
local region=$1
|
||||||
echo ""
|
local bucket_name="rss-feed-processor-layers-$region"
|
||||||
echo "Copy the ARN below:"
|
local layer_name="ingest-rss-lambda-layer-$region"
|
||||||
echo "$ARN"
|
|
||||||
|
echo "Processing region: $region"
|
||||||
|
|
||||||
|
# Create bucket if it doesn't exist
|
||||||
|
if ! aws s3api head-bucket --bucket "$bucket_name" --region "$region" 2>/dev/null; then
|
||||||
|
echo "Creating bucket $bucket_name in $region"
|
||||||
|
if [ "$region" == "us-east-1" ]; then
|
||||||
|
aws s3api create-bucket --bucket "$bucket_name" --region "$region"
|
||||||
|
else
|
||||||
|
aws s3api create-bucket --bucket "$bucket_name" --region "$region" --create-bucket-configuration LocationConstraint=$region
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "Bucket $bucket_name already exists in $region"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Upload ZIP to the region-specific bucket
|
||||||
|
echo "Uploading ZIP to $bucket_name"
|
||||||
|
aws s3 cp OpenRSSLambdaLayer.zip "s3://$bucket_name/" --region "$region"
|
||||||
|
|
||||||
|
# Create and publish Lambda layer
|
||||||
|
echo "Creating Lambda layer in region: $region"
|
||||||
|
LAYER_VERSION=$(aws lambda publish-layer-version \
|
||||||
|
--region "$region" \
|
||||||
|
--layer-name $layer_name \
|
||||||
|
--description "Layer with dependencies for RSS processing" \
|
||||||
|
--license-info "MIT" \
|
||||||
|
--content "S3Bucket=$bucket_name,S3Key=OpenRSSLambdaLayer.zip" \
|
||||||
|
--compatible-runtimes python3.12 \
|
||||||
|
--query 'Version' \
|
||||||
|
--output text
|
||||||
|
)
|
||||||
|
|
||||||
# Double-check and verify
|
if [ -z "$LAYER_VERSION" ]; then
|
||||||
echo ""
|
echo "Failed to create Lambda layer in region $region."
|
||||||
echo "Verification steps:"
|
return 1
|
||||||
echo "1. Verifying S3 upload..."
|
fi
|
||||||
aws s3 ls s3://rss-feed-processor-layers/OpenRSSLambdaLayer.zip
|
|
||||||
|
|
||||||
echo "2. Verifying Lambda layer..."
|
echo "Making layer public in region: $region"
|
||||||
aws lambda get-layer-version --layer-name OpenRSSLambdaLayer --version-number $LAYER_VERSION
|
aws lambda add-layer-version-permission \
|
||||||
|
--region "$region" \
|
||||||
|
--layer-name $layer_name \
|
||||||
|
--version-number "$LAYER_VERSION" \
|
||||||
|
--statement-id public \
|
||||||
|
--action lambda:GetLayerVersion \
|
||||||
|
--principal '*'
|
||||||
|
|
||||||
echo "3. Verifying public access..."
|
ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
|
||||||
aws lambda get-layer-version-policy --layer-name OpenRSSLambdaLayer --version-number $LAYER_VERSION
|
ARN="arn:aws:lambda:${region}:${ACCOUNT_ID}:layer:$layer_name:${LAYER_VERSION}"
|
||||||
|
echo "Layer ARN for region $region: $ARN"
|
||||||
|
echo "$region:$ARN" >> layer_arns.txt
|
||||||
|
}
|
||||||
|
|
||||||
echo "Script execution completed. Please review the output above for any errors."
|
# Process all regions
|
||||||
|
for region in $REGIONS; do
|
||||||
|
if create_bucket_upload_and_publish_layer "$region"; then
|
||||||
|
echo "Successfully processed region: $region"
|
||||||
|
else
|
||||||
|
echo "Failed to process region: $region. Continuing with next region..."
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
####### Section 6: Completion ########
|
||||||
|
echo "Section 6: Completion"
|
||||||
|
|
||||||
|
echo "Setup complete! OpenRSSLambdaLayer is now available in all processed regions."
|
||||||
|
echo "Layer ARNs have been saved to layer_arns.txt"
|
||||||
|
|
||||||
|
echo "Script execution completed successfully."
|
||||||
@@ -1,45 +1,98 @@
|
|||||||
import os
|
import os
|
||||||
from prompt_toolkit import prompt
|
|
||||||
from prompt_toolkit.completion import WordCompleter
|
|
||||||
from rich.console import Console
|
from rich.console import Console
|
||||||
|
from rich.panel import Panel
|
||||||
from rich.text import Text
|
from rich.text import Text
|
||||||
from time import sleep
|
from rich.prompt import Confirm
|
||||||
|
from utils import animate_text, get_env_value, display_summary, save_env_file, emojis, get_aws_regions
|
||||||
|
|
||||||
console = Console()
|
console = Console()
|
||||||
|
|
||||||
def animate_text(text):
|
def check_aws_credentials():
|
||||||
for char in text:
|
return os.environ.get('AWS_ACCESS_KEY_ID') and os.environ.get('AWS_SECRET_ACCESS_KEY')
|
||||||
console.print(char, end='', style="bold green")
|
|
||||||
sleep(0.05)
|
|
||||||
print()
|
|
||||||
|
|
||||||
def set_env_var(name, value):
|
def check_aws_region():
|
||||||
os.environ[name] = value
|
return os.environ.get('AWS_REGION')
|
||||||
animate_text(f"Environment variable {name} set to {value}")
|
|
||||||
|
|
||||||
def list_env_vars():
|
|
||||||
animate_text("Current environment variables:")
|
|
||||||
for key, value in os.environ.items():
|
|
||||||
console.print(f"{key}: {value}")
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
while True:
|
animate_text("Welcome to the Ingest RSS Environment Setup!", emojis)
|
||||||
action = prompt(
|
console.print(Panel(Text("Welcome to the Ingest RSS Environment Setup! 🌴🌻🦍", style="bold yellow")))
|
||||||
"Choose an action (set/list/quit): ",
|
|
||||||
completer=WordCompleter(['set', 'list', 'quit'])
|
|
||||||
)
|
console.print(Panel(Text("Let's configure your environment variables", style="bold yellow")))
|
||||||
|
|
||||||
|
env_vars = {}
|
||||||
|
|
||||||
|
# Determine if we're in advanced mode
|
||||||
|
advanced_mode = not Confirm.ask("Do you want to use basic mode? \n( We recommend basic for your first time ) ")
|
||||||
|
|
||||||
|
# AWS Configuration
|
||||||
|
|
||||||
|
|
||||||
|
env_vars["AWS_ACCOUNT_ID"] = get_env_value("AWS_ACCOUNT_ID", "Enter AWS Account ID:")
|
||||||
|
|
||||||
|
# AWS Credentials
|
||||||
|
if not check_aws_region():
|
||||||
|
console.print("AWS region not found in environment variables.")
|
||||||
|
env_vars["AWS_REGION"] = get_env_value("AWS_REGION", "Enter AWS Region:", options=get_aws_regions())
|
||||||
|
|
||||||
if action == 'set':
|
if not check_aws_credentials():
|
||||||
name = prompt("Enter variable name: ")
|
console.print("AWS credentials not found in environment variables.")
|
||||||
value = prompt("Enter variable value: ")
|
if Confirm.ask("Do you want to set AWS credentials?"):
|
||||||
set_env_var(name, value)
|
env_vars["AWS_ACCESS_KEY_ID"] = get_env_value("AWS_ACCESS_KEY_ID", "Enter AWS Access Key ID:")
|
||||||
elif action == 'list':
|
env_vars["AWS_SECRET_ACCESS_KEY"] = get_env_value("AWS_SECRET_ACCESS_KEY", "Enter AWS Secret Access Key:")
|
||||||
list_env_vars()
|
else:
|
||||||
elif action == 'quit':
|
console.print("AWS credentials found in environment variables.")
|
||||||
animate_text("Goodbye!")
|
|
||||||
break
|
# Resource Names
|
||||||
else:
|
env_vars["LAMBDA_FUNCTION_NAME"] = get_env_value("LAMBDA_FUNCTION_NAME", "Enter Lambda Function Name:", options=["RSSFeedProcessor", "CustomRSSProcessor"], advanced=advanced_mode)
|
||||||
console.print("Invalid action. Please try again.", style="bold red")
|
env_vars["STACK_BASE"] = env_vars["LAMBDA_FUNCTION_NAME"]
|
||||||
|
env_vars["LAMBDA_EXECUTION_ROLE_NAME"] = f"rss-feed-processor-role-{env_vars['AWS_REGION']}"
|
||||||
|
env_vars["LAMBDA_ROLE_ARN"] = f"arn:aws:iam::{env_vars['AWS_ACCOUNT_ID']}:role/{env_vars['LAMBDA_EXECUTION_ROLE_NAME']}"
|
||||||
|
env_vars["S3_BUCKET_NAME"] = f"open-rss-articles-{env_vars['AWS_REGION']}"
|
||||||
|
env_vars["DYNAMODB_TABLE_NAME"] = get_env_value("DYNAMODB_TABLE_NAME", "Enter DynamoDB Table Name:", options=["rss-feeds-table", "custom-rss-table"], advanced=advanced_mode)
|
||||||
|
env_vars["SQS_QUEUE_NAME"] = get_env_value("SQS_QUEUE_NAME", "Enter SQS Queue Name:", options=["rss-feed-queue", "custom-rss-queue"], advanced=advanced_mode)
|
||||||
|
|
||||||
|
# Advanced Configuration
|
||||||
|
env_vars["LAMBDA_LAYER_VERSION"] = get_env_value("LAMBDA_LAYER_VERSION", "Enter Lambda Layer Version:", options=["1", "2", "3"], advanced=advanced_mode)
|
||||||
|
env_vars["LAMBDA_LAYER_NAME"] = f"ingest-rss-lambda-layer-{env_vars['AWS_REGION']}"
|
||||||
|
env_vars["LAMBDA_LAYER_ARN"] = f"arn:aws:lambda:{env_vars['AWS_REGION']}:{env_vars['AWS_ACCOUNT_ID']}:layer:{env_vars['LAMBDA_LAYER_NAME']}:{env_vars['LAMBDA_LAYER_VERSION']}"
|
||||||
|
env_vars["S3_LAYER_BUCKET_NAME"] = f"rss-feed-processor-layers-{env_vars['AWS_REGION']}"
|
||||||
|
env_vars["S3_LAYER_KEY_NAME"] = get_env_value("S3_LAYER_KEY_NAME", "Enter S3 Layer Key Name:", options=["RSSFeedProcessorDependencies", "CustomDependencies"], advanced=advanced_mode)
|
||||||
|
env_vars["SQS_QUEUE_URL"] = f"https://sqs.{env_vars['AWS_REGION']}.amazonaws.com/{env_vars['AWS_ACCOUNT_ID']}/{env_vars['SQS_QUEUE_NAME']}"
|
||||||
|
env_vars["SQS_QUEUE_ARN"] = f"arn:aws:sqs:{env_vars['AWS_REGION']}:{env_vars['AWS_ACCOUNT_ID']}:{env_vars['SQS_QUEUE_NAME']}"
|
||||||
|
env_vars["DYNAMODB_TABLE_ARN"] = f"arn:aws:dynamodb:{env_vars['AWS_REGION']}:{env_vars['AWS_ACCOUNT_ID']}:table/{env_vars['DYNAMODB_TABLE_NAME']}"
|
||||||
|
|
||||||
|
env_vars["PYTHON_VERSION"] = get_env_value("PYTHON_VERSION", "Enter Python Version:", options=["3.8", "3.9", "3.10", "3.11", "3.12"], advanced=advanced_mode)
|
||||||
|
env_vars["LAMBDA_RUNTIME"] = f"python{env_vars['PYTHON_VERSION']}"
|
||||||
|
env_vars["LAMBDA_TIMEOUT"] = get_env_value("LAMBDA_TIMEOUT", "Enter Lambda Timeout (in seconds):", options=["60", "120", "300"], advanced=advanced_mode)
|
||||||
|
env_vars["LAMBDA_MEMORY"] = get_env_value("LAMBDA_MEMORY", "Enter Lambda Memory (in MB):", options=["128", "256", "512", "1024"], advanced=advanced_mode)
|
||||||
|
|
||||||
|
env_vars["QUEUE_FILLER_LAMBDA_NAME"] = get_env_value("QUEUE_FILLER_LAMBDA_NAME", "Enter Queue Filler Lambda Name:", options=["RSSQueueFiller", "CustomQueueFiller"], advanced=advanced_mode)
|
||||||
|
env_vars["QUEUE_FILLER_LAMBDA_S3_KEY"] = get_env_value("QUEUE_FILLER_LAMBDA_S3_KEY", "Enter Queue Filler Lambda S3 Key:", options=["RSSQueueFiller.zip", "CustomQueueFiller.zip"], advanced=advanced_mode)
|
||||||
|
|
||||||
|
# Logging Configuration
|
||||||
|
env_vars["LOG_LEVEL"] = get_env_value("LOG_LEVEL", "Enter Log Level:", options=["DEBUG", "INFO", "WARNING", "ERROR"], advanced=advanced_mode)
|
||||||
|
|
||||||
|
# Other Application Settings
|
||||||
|
env_vars["APP_NAME"] = get_env_value("APP_NAME", "Enter Application Name:", options=["RSS Feed Processor", "Custom RSS Processor"], advanced=advanced_mode)
|
||||||
|
env_vars["VERSION"] = get_env_value("VERSION", "Enter Version:", options=["1.0.0", "1.1.0", "2.0.0"], advanced=advanced_mode)
|
||||||
|
env_vars["TEST"] = get_env_value("TEST", "Enter Test Value:", options=["0", "1"], advanced=advanced_mode)
|
||||||
|
|
||||||
|
# Storage Strategy
|
||||||
|
env_vars["STORAGE_STRATEGY"] = get_env_value("STORAGE_STRATEGY", "Choose Storage Strategy:", options=["s3", "pinecone"], advanced=advanced_mode)
|
||||||
|
|
||||||
|
# Pinecone Configuration (only if pinecone is selected)
|
||||||
|
if env_vars["STORAGE_STRATEGY"] == "pinecone":
|
||||||
|
env_vars["PINECONE_API_KEY"] = get_env_value("PINECONE_API_KEY", "Enter Pinecone API Key:", advanced=advanced_mode)
|
||||||
|
env_vars["PINECONE_DB_NAME"] = get_env_value("PINECONE_DB_NAME", "Enter Pinecone DB Name:", options=["open-rss-articles", "custom-rss-db"], advanced=advanced_mode)
|
||||||
|
|
||||||
|
# Display summary
|
||||||
|
display_summary(env_vars)
|
||||||
|
|
||||||
|
# Save to .env file
|
||||||
|
save_env_file(env_vars)
|
||||||
|
|
||||||
|
animate_text("Environment setup complete! Happy RSS ingesting! 🎉", emojis)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
73
src/launch/utils.py
Normal file
73
src/launch/utils.py
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
import os
|
||||||
|
from typing import List, Dict, Any
|
||||||
|
from rich.console import Console
|
||||||
|
from rich.prompt import Prompt, Confirm
|
||||||
|
from rich.panel import Panel
|
||||||
|
from rich.text import Text
|
||||||
|
from rich.table import Table
|
||||||
|
from rich.progress import Progress, SpinnerColumn, TextColumn
|
||||||
|
import time
|
||||||
|
import random
|
||||||
|
import boto3
|
||||||
|
|
||||||
|
console = Console()
|
||||||
|
|
||||||
|
def animate_text(text: str, emoji_list: List[str], duration: float = 0.05):
|
||||||
|
with Progress(
|
||||||
|
SpinnerColumn(),
|
||||||
|
TextColumn("[progress.description]{task.description}"),
|
||||||
|
transient=True,
|
||||||
|
) as progress:
|
||||||
|
task = progress.add_task(description="", total=len(text))
|
||||||
|
for char in text:
|
||||||
|
time.sleep(duration)
|
||||||
|
emoji = random.choice(emoji_list)
|
||||||
|
progress.update(task, advance=1, description=f"{emoji} {text[:progress.tasks[0].completed + 1]}")
|
||||||
|
console.print()
|
||||||
|
|
||||||
|
def create_dropdown(options: List[str], prompt: str) -> str:
|
||||||
|
table = Table(show_header=False, box=None)
|
||||||
|
for i, option in enumerate(options, 1):
|
||||||
|
table.add_row(f"{i}. {option}")
|
||||||
|
console.print(table)
|
||||||
|
return Prompt.ask(prompt, choices=[str(i) for i in range(1, len(options) + 1)])
|
||||||
|
|
||||||
|
def get_env_value(key: str, prompt: str, options: List[str] = None, advanced: bool = False) -> str:
|
||||||
|
if advanced and not Confirm.ask("Do you want to configure advanced settings?"):
|
||||||
|
return os.environ.get(key, "")
|
||||||
|
|
||||||
|
if options:
|
||||||
|
choice = create_dropdown(options, prompt)
|
||||||
|
return options[int(choice) - 1]
|
||||||
|
else:
|
||||||
|
return Prompt.ask(prompt)
|
||||||
|
|
||||||
|
def display_summary(env_vars: Dict[str, Any]):
|
||||||
|
table = Table(title="Environment Variables Summary")
|
||||||
|
table.add_column("Variable", style="cyan")
|
||||||
|
table.add_column("Value", style="magenta")
|
||||||
|
|
||||||
|
for key, value in env_vars.items():
|
||||||
|
if key in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"]:
|
||||||
|
value = "********" if value else ""
|
||||||
|
table.add_row(key, str(value))
|
||||||
|
|
||||||
|
console.print(table)
|
||||||
|
|
||||||
|
def save_env_file(env_vars: Dict[str, Any]):
|
||||||
|
with open(".env", "w") as f:
|
||||||
|
for key, value in env_vars.items():
|
||||||
|
f.write(f"{key}={value}\n")
|
||||||
|
console.print(Panel(Text("Environment variables saved to .env file", style="bold green")))
|
||||||
|
|
||||||
|
def get_aws_regions() -> List[str]:
|
||||||
|
try:
|
||||||
|
ec2_client = boto3.client('ec2')
|
||||||
|
regions = [region['RegionName'] for region in ec2_client.describe_regions()['Regions']]
|
||||||
|
return sorted(regions)
|
||||||
|
except Exception as e:
|
||||||
|
console.print(f"[bold red]Error fetching AWS regions: {str(e)}[/bold red]")
|
||||||
|
console.print("[yellow]Falling back to default region list.[/yellow]")
|
||||||
|
return ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-west-1", "eu-central-1", "ap-southeast-1", "ap-southeast-2"]
|
||||||
|
|
||||||
|
emojis = ["🦍", "🗞️", "💵", "🚀", "☕", "🌻", "☀️", "🌴", "🌳", "🌲", "🎋"]
|
||||||
8
todo.md
8
todo.md
@@ -1,11 +1,9 @@
|
|||||||
# TODO Before Launch 🚀
|
# TODO Before Launch 🚀
|
||||||
* Test Creation
|
* Delete KMS KEY test.
|
||||||
|
* Test from 3rd party aws account.
|
||||||
* Add in console setup python script for new project into launch.py
|
* Add in console setup python script for new project into launch.py
|
||||||
* Better Readme.md
|
* Better Readme.md
|
||||||
* Update Lambda Layer Creation Script to be more comprehensive.
|
|
||||||
** Make a layer for every region in existance.
|
|
||||||
** Save this script.
|
|
||||||
** Update the layer references with os.getenv("AWS_REGION")
|
|
||||||
|
|
||||||
# Misc TODO:
|
# Misc TODO:
|
||||||
* More RSS Feed Module - Easy
|
* More RSS Feed Module - Easy
|
||||||
|
|||||||
Reference in New Issue
Block a user