finalized product.

This commit is contained in:
Charles E. Gormley
2024-09-08 20:35:11 -04:00
parent 3cd0bcc412
commit 1da375d25d
7 changed files with 322 additions and 135 deletions

View File

@@ -5,10 +5,6 @@ import boto3
from dotenv import load_dotenv from dotenv import load_dotenv
import logging import logging
from src.infra.lambdas.RSSQueueFiller.deploy_sqs_filler_lambda import deploy_sqs_filler from src.infra.lambdas.RSSQueueFiller.deploy_sqs_filler_lambda import deploy_sqs_filler
from src.infra.deploy_infrastructure import deploy_infrastructure
from src.infra.lambdas.RSSFeedProcessorLambda.deploy_rss_feed_lambda import deploy_lambda
from src.infra.lambdas.lambda_utils.update_lambda_env_vars import update_env_vars
from src.feed_management.upload_rss_feeds import upload_rss_feeds
# Load environment variables # Load environment variables
load_dotenv(override=True) load_dotenv(override=True)
@@ -27,7 +23,10 @@ REGION = os.getenv("AWS_REGION")
current_dir = os.path.dirname(os.path.abspath(__file__)) current_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(current_dir) sys.path.append(current_dir)
from src.infra.deploy_infrastructure import deploy_infrastructure
from src.infra.lambdas.RSSFeedProcessorLambda.deploy_rss_feed_lambda import deploy_lambda
from src.infra.lambdas.lambda_utils.update_lambda_env_vars import update_env_vars
from src.feed_management.upload_rss_feeds import upload_rss_feeds
def main(): def main():
# Deploy infrastructure # Deploy infrastructure

View File

@@ -8,6 +8,10 @@ Parameters:
LambdaKMSKeyArn: LambdaKMSKeyArn:
Type: String Type: String
Description: "ARN of the KMS Key for Lambda environment variable encryption" Description: "ARN of the KMS Key for Lambda environment variable encryption"
Region:
Type: String
Description: "AWS Region for deployment"
Default: "us-east-1"
Resources: Resources:
LambdaExecutionRole: LambdaExecutionRole:
@@ -58,3 +62,8 @@ Outputs:
Value: !Ref LambdaKMSKeyArn Value: !Ref LambdaKMSKeyArn
Export: Export:
Name: !Sub '${AWS::StackName}-LambdaKMSKeyArn' Name: !Sub '${AWS::StackName}-LambdaKMSKeyArn'
Region:
Description: 'AWS Region for deployment'
Value: !Ref Region
Export:
Name: !Sub '${AWS::StackName}-Region'

View File

@@ -9,6 +9,8 @@ from src.utils.retry_logic import retry_with_backoff
import time import time
import sys import sys
from src.infra.deploy_infrastructure import get_or_create_kms_key from src.infra.deploy_infrastructure import get_or_create_kms_key
from dotenv import load_dotenv
load_dotenv()
import logging import logging
logging.basicConfig(level=os.getenv('LOG_LEVEL', 'INFO')) logging.basicConfig(level=os.getenv('LOG_LEVEL', 'INFO'))
@@ -18,6 +20,7 @@ logging.basicConfig(level=os.getenv('LOG_LEVEL', 'INFO'))
LAMBDA_NAME = os.getenv('LAMBDA_FUNCTION_NAME') LAMBDA_NAME = os.getenv('LAMBDA_FUNCTION_NAME')
ACCOUNT_NUM = os.getenv('AWS_ACCOUNT_ID') ACCOUNT_NUM = os.getenv('AWS_ACCOUNT_ID')
REGION = os.getenv("AWS_REGION")
LAMBDA_ROLE_ARN = os.getenv("LAMBDA_ROLE_ARN") LAMBDA_ROLE_ARN = os.getenv("LAMBDA_ROLE_ARN")
LAMBDA_TIMEOUT = int(os.getenv('LAMBDA_TIMEOUT')) LAMBDA_TIMEOUT = int(os.getenv('LAMBDA_TIMEOUT'))
LAMBDA_MEMORY = int(os.getenv('LAMBDA_MEMORY')) LAMBDA_MEMORY = int(os.getenv('LAMBDA_MEMORY'))
@@ -63,18 +66,18 @@ def update_function_configuration(lambda_client, function_name, handler, role, t
'Layers': layers 'Layers': layers
} }
if kms_key_id: if kms_key_id:
config['KMSKeyArn'] = f"arn:aws:kms:{os.environ['AWS_REGION']}:{ACCOUNT_NUM}:key/{kms_key_id}" config['KMSKeyArn'] = f"arn:aws:kms:{REGION}:{ACCOUNT_NUM}:key/{kms_key_id}"
try:
response = lambda_client.update_function_configuration(**config)
print(f"Update request sent successfully for {function_name}.")
try: except ClientError as e:
response = lambda_client.update_function_configuration(**config) if e.response['Error']['Code'] == 'ResourceConflictException':
print(f"Update request sent successfully for {function_name}.") logging.info(f"Function {function_name} is currently being updated. Retrying...")
raise e
except ClientError as e:
if e.response['Error']['Code'] == 'ResourceConflictException':
logging.info(f"Function {function_name} is currently being updated. Retrying...")
raise e
@retry_with_backoff() @retry_with_backoff()
def configure_sqs_trigger(lambda_client, function_name, queue_arn): def configure_sqs_trigger(lambda_client, function_name, queue_arn):
@@ -100,7 +103,7 @@ def configure_sqs_trigger(lambda_client, function_name, queue_arn):
raise e raise e
@retry_with_backoff() @retry_with_backoff()
def create_function(lambda_client, function_name, runtime, role, handler, zip_file, timeout, memory, layers, kms_key_id): def create_function(lambda_client, function_name, runtime, role, handler, zip_file, timeout, memory, layers, kms_key_id, policy):
config = { config = {
'FunctionName': function_name, 'FunctionName': function_name,
'Runtime': runtime, 'Runtime': runtime,
@@ -111,11 +114,19 @@ def create_function(lambda_client, function_name, runtime, role, handler, zip_fi
'MemorySize': memory, 'MemorySize': memory,
'Layers': layers 'Layers': layers
} }
print(policy)
if kms_key_id: if kms_key_id:
config['KMSKeyArn'] = f"arn:aws:kms:{os.environ['AWS_DEFAULT_REGION']}:{ACCOUNT_NUM}:key/{kms_key_id}" config['KMSKeyArn'] = f"arn:aws:kms:{REGION}:{ACCOUNT_NUM}:key/{kms_key_id}"
return lambda_client.create_function(**config) try:
return lambda_client.create_function(**config)
except ClientError as e:
if e.response['Error']['Code'] == 'InvalidParameterValueException':
print(f"Error creating function: {e}")
print("Ensure that the IAM role has the correct trust relationship and permissions.")
print("There might be a delay in role propagation. Please wait a few minutes and try again.")
raise
def get_pillow_layer_arn(): def get_pillow_layer_arn():
url = f"https://api.klayers.cloud/api/v2/p3.11/layers/latest/{os.getenv('AWS_REGION')}/json" url = f"https://api.klayers.cloud/api/v2/p3.11/layers/latest/{os.getenv('AWS_REGION')}/json"
@@ -135,8 +146,32 @@ def get_pillow_layer_arn():
print(f"Error fetching Pillow layer ARN: {e}") print(f"Error fetching Pillow layer ARN: {e}")
return None return None
def get_lambda_policy():
policy = {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": "arn:aws:logs:*:*:*"
},
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject"
],
"Resource": "arn:aws:s3:::your-bucket-name/*"
}
]
}
def deploy_lambda(): def deploy_lambda():
lambda_client = boto3.client('lambda') lambda_client = boto3.client('lambda', region_name=REGION)
print(f"Starting deployment of Lambda function: {LAMBDA_NAME}") print(f"Starting deployment of Lambda function: {LAMBDA_NAME}")
deployment_package = zip_directory('src/infra/lambdas/RSSFeedProcessorLambda/src') deployment_package = zip_directory('src/infra/lambdas/RSSFeedProcessorLambda/src')
@@ -182,7 +217,8 @@ def deploy_lambda():
update_function_code(lambda_client, LAMBDA_NAME, deployment_package) update_function_code(lambda_client, LAMBDA_NAME, deployment_package)
else: else:
print(f"Lambda function '{LAMBDA_NAME}' not found. Creating new function...") print(f"Lambda function '{LAMBDA_NAME}' not found. Creating new function...")
create_function(lambda_client, LAMBDA_NAME, LAMBDA_RUNTIME, LAMBDA_ROLE_ARN, LAMBDA_HANDLER, deployment_package, LAMBDA_TIMEOUT, LAMBDA_MEMORY, layers, kms_key_id) policy = get_lambda_policy()
create_function(lambda_client, LAMBDA_NAME, LAMBDA_RUNTIME, LAMBDA_ROLE_ARN, LAMBDA_HANDLER, deployment_package, LAMBDA_TIMEOUT, LAMBDA_MEMORY, layers, kms_key_id, policy)
# Configure SQS trigger # Configure SQS trigger
queue_arn = os.getenv('SQS_QUEUE_ARN') # Make sure to set this environment variable queue_arn = os.getenv('SQS_QUEUE_ARN') # Make sure to set this environment variable

View File

@@ -1,98 +1,117 @@
#!/bin/bash #!/bin/bash
# TODO: This needs to be completely overhauled
# Update system packages
echo "Updating system packages..."
sudo yum update -y
# Install development tools set -e
echo "Installing development tools..."
sudo yum groupinstall "Development Tools" -y
# Install Python 3.11 ####### Section 1: Checking Python Existence ########
echo "Installing Python 3.11..." echo "Section 1: Checking Python Existence"
sudo amazon-linux-extras enable python3.11
sudo yum install python3.11 -y
# Verify Python 3.11 installation # Ensure python3.12 is installed
if command -v python3.11 &>/dev/null; then if ! command -v python3.12 &> /dev/null; then
echo "Python 3.11 installed successfully:" echo "Python 3.12 is not installed. Please install it before running this script."
python3.11 --version
else
echo "Failed to install Python 3.11. Exiting."
exit 1 exit 1
fi fi
echo "Python 3.12 found. Proceeding..."
# Install pip for Python 3.11 ####### Section 2: Installing Dependencies ########
echo "Installing pip for Python 3.11..." echo "Section 2: Installing Dependencies"
sudo python3.11 -m ensurepip --upgrade
# Verify pip installation # Install dependencies
if command -v pip3.11 &>/dev/null; then python3.12 -m pip install --upgrade Pillow feedfinder2==0.0.4 python-dateutil newspaper3k==0.2.8 feedparser lxml[html5lib] lxml_html_clean lxml[html_clean] -t python/
echo "pip installed successfully:" echo "Dependencies installed successfully."
pip3.11 --version
else
echo "Failed to install pip. Exiting."
exit 1
fi
# Create directory for Lambda layer ####### Section 3: Creating ZIP File ########
echo "Creating directory for Lambda layer..." echo "Section 3: Creating ZIP File"
mkdir -p OpenRSSLambdaLayer/python
cd OpenRSSLambdaLayer
# Install packages
echo "Installing packages..."
pip3.11 install newspaper3k feedparser python-dateutil-t python/
# Create ZIP file # Create ZIP file
echo "Creating ZIP file..."
zip -r OpenRSSLambdaLayer.zip python/ zip -r OpenRSSLambdaLayer.zip python/
echo "ZIP file created."
# Upload to S3 # Check if ZIP file was created and is not empty
echo "Uploading to S3..." if [ ! -s OpenRSSLambdaLayer.zip ]; then
aws s3 cp OpenRSSLambdaLayer.zip s3://rss-feed-processor-layers/OpenRSSLambdaLayer.zip echo "Error: ZIP file is empty or was not created."
exit 1
fi
echo "ZIP file check passed."
# Create Lambda layer ####### Section 4: Getting AWS Regions ########
echo "Creating Lambda layer..." echo "Section 4: Getting AWS Regions"
LAYER_VERSION=$(aws lambda publish-layer-version \
--layer-name OpenRSSLambdaLayer \
--description "Layer with dependencies for RSS processing" \
--license-info "MIT" \
--content S3Bucket=rss-feed-processor-layers,S3Key=OpenRSSLambdaLayer.zip \
--compatible-runtimes python3.11 \
--query 'Version' \
--output text)
# Make layer public # Get list of all AWS regions
echo "Making layer public..." REGIONS=$(aws ec2 describe-regions --query 'Regions[].RegionName' --output text)
aws lambda add-layer-version-permission \ echo "Retrieved AWS regions: $REGIONS"
--layer-name OpenRSSLambdaLayer \
--version-number $LAYER_VERSION \
--statement-id public \
--action lambda:GetLayerVersion \
--principal '*'
# Calculate and print the ARN ####### Section 5: Creating Buckets, Uploading, and Publishing Layer ########
REGION=$(aws configure get region) echo "Section 5: Creating Buckets, Uploading, and Publishing Layer"
ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
ARN="arn:aws:lambda:${REGION}:${ACCOUNT_ID}:layer:OpenRSSLambdaLayer:${LAYER_VERSION}"
echo "Setup complete! OpenRSSLambdaLayer is now available to anyone on the internet." create_bucket_upload_and_publish_layer() {
echo "Layer ARN: $ARN" local region=$1
echo "" local bucket_name="rss-feed-processor-layers-$region"
echo "Copy the ARN below:" local layer_name="ingest-rss-lambda-layer-$region"
echo "$ARN"
# Double-check and verify echo "Processing region: $region"
echo ""
echo "Verification steps:"
echo "1. Verifying S3 upload..."
aws s3 ls s3://rss-feed-processor-layers/OpenRSSLambdaLayer.zip
echo "2. Verifying Lambda layer..." # Create bucket if it doesn't exist
aws lambda get-layer-version --layer-name OpenRSSLambdaLayer --version-number $LAYER_VERSION if ! aws s3api head-bucket --bucket "$bucket_name" --region "$region" 2>/dev/null; then
echo "Creating bucket $bucket_name in $region"
if [ "$region" == "us-east-1" ]; then
aws s3api create-bucket --bucket "$bucket_name" --region "$region"
else
aws s3api create-bucket --bucket "$bucket_name" --region "$region" --create-bucket-configuration LocationConstraint=$region
fi
else
echo "Bucket $bucket_name already exists in $region"
fi
echo "3. Verifying public access..." # Upload ZIP to the region-specific bucket
aws lambda get-layer-version-policy --layer-name OpenRSSLambdaLayer --version-number $LAYER_VERSION echo "Uploading ZIP to $bucket_name"
aws s3 cp OpenRSSLambdaLayer.zip "s3://$bucket_name/" --region "$region"
echo "Script execution completed. Please review the output above for any errors." # Create and publish Lambda layer
echo "Creating Lambda layer in region: $region"
LAYER_VERSION=$(aws lambda publish-layer-version \
--region "$region" \
--layer-name $layer_name \
--description "Layer with dependencies for RSS processing" \
--license-info "MIT" \
--content "S3Bucket=$bucket_name,S3Key=OpenRSSLambdaLayer.zip" \
--compatible-runtimes python3.12 \
--query 'Version' \
--output text
)
if [ -z "$LAYER_VERSION" ]; then
echo "Failed to create Lambda layer in region $region."
return 1
fi
echo "Making layer public in region: $region"
aws lambda add-layer-version-permission \
--region "$region" \
--layer-name $layer_name \
--version-number "$LAYER_VERSION" \
--statement-id public \
--action lambda:GetLayerVersion \
--principal '*'
ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
ARN="arn:aws:lambda:${region}:${ACCOUNT_ID}:layer:$layer_name:${LAYER_VERSION}"
echo "Layer ARN for region $region: $ARN"
echo "$region:$ARN" >> layer_arns.txt
}
# Process all regions
for region in $REGIONS; do
if create_bucket_upload_and_publish_layer "$region"; then
echo "Successfully processed region: $region"
else
echo "Failed to process region: $region. Continuing with next region..."
fi
done
####### Section 6: Completion ########
echo "Section 6: Completion"
echo "Setup complete! OpenRSSLambdaLayer is now available in all processed regions."
echo "Layer ARNs have been saved to layer_arns.txt"
echo "Script execution completed successfully."

View File

@@ -1,45 +1,98 @@
import os import os
from prompt_toolkit import prompt
from prompt_toolkit.completion import WordCompleter
from rich.console import Console from rich.console import Console
from rich.panel import Panel
from rich.text import Text from rich.text import Text
from time import sleep from rich.prompt import Confirm
from utils import animate_text, get_env_value, display_summary, save_env_file, emojis, get_aws_regions
console = Console() console = Console()
def animate_text(text): def check_aws_credentials():
for char in text: return os.environ.get('AWS_ACCESS_KEY_ID') and os.environ.get('AWS_SECRET_ACCESS_KEY')
console.print(char, end='', style="bold green")
sleep(0.05)
print()
def set_env_var(name, value): def check_aws_region():
os.environ[name] = value return os.environ.get('AWS_REGION')
animate_text(f"Environment variable {name} set to {value}")
def list_env_vars():
animate_text("Current environment variables:")
for key, value in os.environ.items():
console.print(f"{key}: {value}")
def main(): def main():
while True: animate_text("Welcome to the Ingest RSS Environment Setup!", emojis)
action = prompt( console.print(Panel(Text("Welcome to the Ingest RSS Environment Setup! 🌴🌻🦍", style="bold yellow")))
"Choose an action (set/list/quit): ",
completer=WordCompleter(['set', 'list', 'quit'])
)
if action == 'set':
name = prompt("Enter variable name: ") console.print(Panel(Text("Let's configure your environment variables", style="bold yellow")))
value = prompt("Enter variable value: ")
set_env_var(name, value) env_vars = {}
elif action == 'list':
list_env_vars() # Determine if we're in advanced mode
elif action == 'quit': advanced_mode = not Confirm.ask("Do you want to use basic mode? \n( We recommend basic for your first time ) ")
animate_text("Goodbye!")
break # AWS Configuration
else:
console.print("Invalid action. Please try again.", style="bold red")
env_vars["AWS_ACCOUNT_ID"] = get_env_value("AWS_ACCOUNT_ID", "Enter AWS Account ID:")
# AWS Credentials
if not check_aws_region():
console.print("AWS region not found in environment variables.")
env_vars["AWS_REGION"] = get_env_value("AWS_REGION", "Enter AWS Region:", options=get_aws_regions())
if not check_aws_credentials():
console.print("AWS credentials not found in environment variables.")
if Confirm.ask("Do you want to set AWS credentials?"):
env_vars["AWS_ACCESS_KEY_ID"] = get_env_value("AWS_ACCESS_KEY_ID", "Enter AWS Access Key ID:")
env_vars["AWS_SECRET_ACCESS_KEY"] = get_env_value("AWS_SECRET_ACCESS_KEY", "Enter AWS Secret Access Key:")
else:
console.print("AWS credentials found in environment variables.")
# Resource Names
env_vars["LAMBDA_FUNCTION_NAME"] = get_env_value("LAMBDA_FUNCTION_NAME", "Enter Lambda Function Name:", options=["RSSFeedProcessor", "CustomRSSProcessor"], advanced=advanced_mode)
env_vars["STACK_BASE"] = env_vars["LAMBDA_FUNCTION_NAME"]
env_vars["LAMBDA_EXECUTION_ROLE_NAME"] = f"rss-feed-processor-role-{env_vars['AWS_REGION']}"
env_vars["LAMBDA_ROLE_ARN"] = f"arn:aws:iam::{env_vars['AWS_ACCOUNT_ID']}:role/{env_vars['LAMBDA_EXECUTION_ROLE_NAME']}"
env_vars["S3_BUCKET_NAME"] = f"open-rss-articles-{env_vars['AWS_REGION']}"
env_vars["DYNAMODB_TABLE_NAME"] = get_env_value("DYNAMODB_TABLE_NAME", "Enter DynamoDB Table Name:", options=["rss-feeds-table", "custom-rss-table"], advanced=advanced_mode)
env_vars["SQS_QUEUE_NAME"] = get_env_value("SQS_QUEUE_NAME", "Enter SQS Queue Name:", options=["rss-feed-queue", "custom-rss-queue"], advanced=advanced_mode)
# Advanced Configuration
env_vars["LAMBDA_LAYER_VERSION"] = get_env_value("LAMBDA_LAYER_VERSION", "Enter Lambda Layer Version:", options=["1", "2", "3"], advanced=advanced_mode)
env_vars["LAMBDA_LAYER_NAME"] = f"ingest-rss-lambda-layer-{env_vars['AWS_REGION']}"
env_vars["LAMBDA_LAYER_ARN"] = f"arn:aws:lambda:{env_vars['AWS_REGION']}:{env_vars['AWS_ACCOUNT_ID']}:layer:{env_vars['LAMBDA_LAYER_NAME']}:{env_vars['LAMBDA_LAYER_VERSION']}"
env_vars["S3_LAYER_BUCKET_NAME"] = f"rss-feed-processor-layers-{env_vars['AWS_REGION']}"
env_vars["S3_LAYER_KEY_NAME"] = get_env_value("S3_LAYER_KEY_NAME", "Enter S3 Layer Key Name:", options=["RSSFeedProcessorDependencies", "CustomDependencies"], advanced=advanced_mode)
env_vars["SQS_QUEUE_URL"] = f"https://sqs.{env_vars['AWS_REGION']}.amazonaws.com/{env_vars['AWS_ACCOUNT_ID']}/{env_vars['SQS_QUEUE_NAME']}"
env_vars["SQS_QUEUE_ARN"] = f"arn:aws:sqs:{env_vars['AWS_REGION']}:{env_vars['AWS_ACCOUNT_ID']}:{env_vars['SQS_QUEUE_NAME']}"
env_vars["DYNAMODB_TABLE_ARN"] = f"arn:aws:dynamodb:{env_vars['AWS_REGION']}:{env_vars['AWS_ACCOUNT_ID']}:table/{env_vars['DYNAMODB_TABLE_NAME']}"
env_vars["PYTHON_VERSION"] = get_env_value("PYTHON_VERSION", "Enter Python Version:", options=["3.8", "3.9", "3.10", "3.11", "3.12"], advanced=advanced_mode)
env_vars["LAMBDA_RUNTIME"] = f"python{env_vars['PYTHON_VERSION']}"
env_vars["LAMBDA_TIMEOUT"] = get_env_value("LAMBDA_TIMEOUT", "Enter Lambda Timeout (in seconds):", options=["60", "120", "300"], advanced=advanced_mode)
env_vars["LAMBDA_MEMORY"] = get_env_value("LAMBDA_MEMORY", "Enter Lambda Memory (in MB):", options=["128", "256", "512", "1024"], advanced=advanced_mode)
env_vars["QUEUE_FILLER_LAMBDA_NAME"] = get_env_value("QUEUE_FILLER_LAMBDA_NAME", "Enter Queue Filler Lambda Name:", options=["RSSQueueFiller", "CustomQueueFiller"], advanced=advanced_mode)
env_vars["QUEUE_FILLER_LAMBDA_S3_KEY"] = get_env_value("QUEUE_FILLER_LAMBDA_S3_KEY", "Enter Queue Filler Lambda S3 Key:", options=["RSSQueueFiller.zip", "CustomQueueFiller.zip"], advanced=advanced_mode)
# Logging Configuration
env_vars["LOG_LEVEL"] = get_env_value("LOG_LEVEL", "Enter Log Level:", options=["DEBUG", "INFO", "WARNING", "ERROR"], advanced=advanced_mode)
# Other Application Settings
env_vars["APP_NAME"] = get_env_value("APP_NAME", "Enter Application Name:", options=["RSS Feed Processor", "Custom RSS Processor"], advanced=advanced_mode)
env_vars["VERSION"] = get_env_value("VERSION", "Enter Version:", options=["1.0.0", "1.1.0", "2.0.0"], advanced=advanced_mode)
env_vars["TEST"] = get_env_value("TEST", "Enter Test Value:", options=["0", "1"], advanced=advanced_mode)
# Storage Strategy
env_vars["STORAGE_STRATEGY"] = get_env_value("STORAGE_STRATEGY", "Choose Storage Strategy:", options=["s3", "pinecone"], advanced=advanced_mode)
# Pinecone Configuration (only if pinecone is selected)
if env_vars["STORAGE_STRATEGY"] == "pinecone":
env_vars["PINECONE_API_KEY"] = get_env_value("PINECONE_API_KEY", "Enter Pinecone API Key:", advanced=advanced_mode)
env_vars["PINECONE_DB_NAME"] = get_env_value("PINECONE_DB_NAME", "Enter Pinecone DB Name:", options=["open-rss-articles", "custom-rss-db"], advanced=advanced_mode)
# Display summary
display_summary(env_vars)
# Save to .env file
save_env_file(env_vars)
animate_text("Environment setup complete! Happy RSS ingesting! 🎉", emojis)
if __name__ == "__main__": if __name__ == "__main__":
main() main()

73
src/launch/utils.py Normal file
View File

@@ -0,0 +1,73 @@
import os
from typing import List, Dict, Any
from rich.console import Console
from rich.prompt import Prompt, Confirm
from rich.panel import Panel
from rich.text import Text
from rich.table import Table
from rich.progress import Progress, SpinnerColumn, TextColumn
import time
import random
import boto3
console = Console()
def animate_text(text: str, emoji_list: List[str], duration: float = 0.05):
with Progress(
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
transient=True,
) as progress:
task = progress.add_task(description="", total=len(text))
for char in text:
time.sleep(duration)
emoji = random.choice(emoji_list)
progress.update(task, advance=1, description=f"{emoji} {text[:progress.tasks[0].completed + 1]}")
console.print()
def create_dropdown(options: List[str], prompt: str) -> str:
table = Table(show_header=False, box=None)
for i, option in enumerate(options, 1):
table.add_row(f"{i}. {option}")
console.print(table)
return Prompt.ask(prompt, choices=[str(i) for i in range(1, len(options) + 1)])
def get_env_value(key: str, prompt: str, options: List[str] = None, advanced: bool = False) -> str:
if advanced and not Confirm.ask("Do you want to configure advanced settings?"):
return os.environ.get(key, "")
if options:
choice = create_dropdown(options, prompt)
return options[int(choice) - 1]
else:
return Prompt.ask(prompt)
def display_summary(env_vars: Dict[str, Any]):
table = Table(title="Environment Variables Summary")
table.add_column("Variable", style="cyan")
table.add_column("Value", style="magenta")
for key, value in env_vars.items():
if key in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"]:
value = "********" if value else ""
table.add_row(key, str(value))
console.print(table)
def save_env_file(env_vars: Dict[str, Any]):
with open(".env", "w") as f:
for key, value in env_vars.items():
f.write(f"{key}={value}\n")
console.print(Panel(Text("Environment variables saved to .env file", style="bold green")))
def get_aws_regions() -> List[str]:
try:
ec2_client = boto3.client('ec2')
regions = [region['RegionName'] for region in ec2_client.describe_regions()['Regions']]
return sorted(regions)
except Exception as e:
console.print(f"[bold red]Error fetching AWS regions: {str(e)}[/bold red]")
console.print("[yellow]Falling back to default region list.[/yellow]")
return ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-west-1", "eu-central-1", "ap-southeast-1", "ap-southeast-2"]
emojis = ["🦍", "🗞️", "💵", "🚀", "", "🌻", "☀️", "🌴", "🌳", "🌲", "🎋"]

View File

@@ -1,11 +1,9 @@
# TODO Before Launch 🚀 # TODO Before Launch 🚀
* Test Creation * Delete KMS KEY test.
* Test from 3rd party aws account.
* Add in console setup python script for new project into launch.py * Add in console setup python script for new project into launch.py
* Better Readme.md * Better Readme.md
* Update Lambda Layer Creation Script to be more comprehensive.
** Make a layer for every region in existance.
** Save this script.
** Update the layer references with os.getenv("AWS_REGION")
# Misc TODO: # Misc TODO:
* More RSS Feed Module - Easy * More RSS Feed Module - Easy