From 38f5b1f4395c8cbf7c1e33d991d48c109eb20ede Mon Sep 17 00:00:00 2001 From: Aljaz Date: Mon, 2 Jun 2025 13:46:01 +0200 Subject: [PATCH] Switch to Redis queue --- launch.py | 4 +- requirements.txt | 3 +- src/feed_management/upload_rss_feeds.py | 53 +++---------- .../deploy_infrastructure.cpython-312.pyc | Bin 7052 -> 0 bytes src/infra/cloudformation/dynamo.yaml | 27 ------- src/infra/cloudformation/lambda_role.yaml | 1 - .../cloudformation/rss_lambda_stack.yaml | 32 +++----- src/infra/cloudformation/sqs.yaml | 36 --------- src/infra/deploy_infrastructure.py | 14 ---- .../deploy_rss_feed_lambda.py | 29 +------- .../RSSFeedProcessorLambda/src/config.py | 14 ++-- .../src/data_storage.py | 29 ++++---- .../src/lambda_function.py | 70 +++++++----------- .../deploy_sqs_filler_lambda.py | 16 +--- .../RSSQueueFiller/lambda/lambda_function.py | 68 +++++++---------- .../lambda_utils/update_lambda_env_vars.py | 13 +--- src/launch/launch_env.py | 7 +- .../create_lambda_layer.cpython-310.pyc | Bin 2201 -> 0 bytes .../create_lambda_layer.cpython-311.pyc | Bin 3829 -> 0 bytes .../create_lambda_layer.cpython-312.pyc | Bin 3418 -> 0 bytes .../create_s3_bucket.cpython-310.pyc | Bin 877 -> 0 bytes .../__pycache__/kms_update.cpython-310.pyc | Bin 1924 -> 0 bytes .../__pycache__/kms_update.cpython-311.pyc | Bin 3121 -> 0 bytes .../__pycache__/kms_update.cpython-312.pyc | Bin 2690 -> 0 bytes .../__pycache__/retry_logic.cpython-310.pyc | Bin 1350 -> 0 bytes .../__pycache__/retry_logic.cpython-311.pyc | Bin 1872 -> 0 bytes .../__pycache__/retry_logic.cpython-312.pyc | Bin 1854 -> 0 bytes .../upload_rss_feeds.cpython-310.pyc | Bin 1829 -> 0 bytes .../upload_rss_feeds.cpython-311.pyc | Bin 3473 -> 0 bytes .../upload_rss_feeds.cpython-312.pyc | Bin 3022 -> 0 bytes src/utils/check_env.py | 7 +- template.env | 10 +-- tree.md | 1 - 33 files changed, 112 insertions(+), 322 deletions(-) delete mode 100644 src/infra/__pycache__/deploy_infrastructure.cpython-312.pyc delete mode 100644 src/infra/cloudformation/dynamo.yaml delete mode 100644 src/infra/cloudformation/sqs.yaml delete mode 100644 src/utils/__pycache__/create_lambda_layer.cpython-310.pyc delete mode 100644 src/utils/__pycache__/create_lambda_layer.cpython-311.pyc delete mode 100644 src/utils/__pycache__/create_lambda_layer.cpython-312.pyc delete mode 100644 src/utils/__pycache__/create_s3_bucket.cpython-310.pyc delete mode 100644 src/utils/__pycache__/kms_update.cpython-310.pyc delete mode 100644 src/utils/__pycache__/kms_update.cpython-311.pyc delete mode 100644 src/utils/__pycache__/kms_update.cpython-312.pyc delete mode 100644 src/utils/__pycache__/retry_logic.cpython-310.pyc delete mode 100644 src/utils/__pycache__/retry_logic.cpython-311.pyc delete mode 100644 src/utils/__pycache__/retry_logic.cpython-312.pyc delete mode 100644 src/utils/__pycache__/upload_rss_feeds.cpython-310.pyc delete mode 100644 src/utils/__pycache__/upload_rss_feeds.cpython-311.pyc delete mode 100644 src/utils/__pycache__/upload_rss_feeds.cpython-312.pyc diff --git a/launch.py b/launch.py index 07a24f3..239d2be 100644 --- a/launch.py +++ b/launch.py @@ -37,7 +37,7 @@ def main(): logging.info("Finished Deploying Lambda") deploy_sqs_filler() - logging.info("Finished Deploying SQS Filler Lambda") + logging.info("Finished Deploying Queue Filler Lambda") # Update Lambda environment variables update_env_vars(os.getenv("LAMBDA_FUNCTION_NAME")) @@ -48,7 +48,7 @@ def main(): if os.path.exists(rss_feeds_file): with open(rss_feeds_file, 'r') as f: rss_feeds = json.load(f) - upload_rss_feeds(rss_feeds, os.getenv('DYNAMODB_TABLE_NAME')) + upload_rss_feeds(rss_feeds, rss_feeds_file) else: print(f"WARNING: {rss_feeds_file} not found. Skipping RSS feed upload.") diff --git a/requirements.txt b/requirements.txt index 4aeda36..7220d27 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ constructs==10.2.69 # Optional, yet necessary for the Pinecone SDK functionality. pinecone openai -tqdm \ No newline at end of file +tqdm +redis diff --git a/src/feed_management/upload_rss_feeds.py b/src/feed_management/upload_rss_feeds.py index 018b767..78fb01d 100644 --- a/src/feed_management/upload_rss_feeds.py +++ b/src/feed_management/upload_rss_feeds.py @@ -1,58 +1,23 @@ import json -import boto3 -from boto3.dynamodb.conditions import Key -from botocore.exceptions import ClientError import logging -# Set up logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -def upload_rss_feeds(rss_feeds, table_name): - dynamodb = boto3.resource('dynamodb') - table = dynamodb.Table(table_name) - - logger.info(f"Uploading RSS feeds to table: {table_name}") +def upload_rss_feeds(rss_feeds, file_path): + """Persist RSS feed definitions to a local JSON file.""" try: - # Get the table's key schema - key_schema = table.key_schema - partition_key = next(key['AttributeName'] for key in key_schema if key['KeyType'] == 'HASH') - except ClientError as e: - logger.error(f"Error getting table schema: {e.response['Error']['Message']}") - return + with open(file_path, "w") as f: + json.dump(rss_feeds, f) + logger.info(f"Saved {len(rss_feeds)} feeds to {file_path}") + except Exception as e: + logger.error(f"Failed to save feeds: {e}") - new_items = 0 - existing_items = 0 - - for feed in rss_feeds: - # Check if the item already exists - try: - response = table.get_item(Key={partition_key: feed['u']}) - except ClientError as e: - logger.error(f"Error checking for existing item: {e.response['Error']['Message']}") - continue - - if 'Item' not in response: - # Item doesn't exist, insert new item - item = {partition_key: feed['u'], 'dt': 0} - feed['dt'] = int(feed['dt']) - item.update() - - try: - table.put_item(Item=item) - new_items += 1 - except ClientError as e: - logger.error(f"Error inserting new item: {e.response['Error']['Message']}") - else: - existing_items += 1 - - logger.info(f"Upload complete. {new_items} new items inserted. {existing_items} items already existed.") if __name__ == "__main__": - table_name = 'rss-feeds-table' - rss_feed_path = 'rss_feeds.json' + rss_feed_path = "rss_feeds.json" with open(rss_feed_path) as f: rss_feeds = json.load(f) logger.info(f"Loaded RSS feeds: {rss_feeds}") - upload_rss_feeds(rss_feeds, table_name) \ No newline at end of file + upload_rss_feeds(rss_feeds, rss_feed_path) diff --git a/src/infra/__pycache__/deploy_infrastructure.cpython-312.pyc b/src/infra/__pycache__/deploy_infrastructure.cpython-312.pyc deleted file mode 100644 index 188e56feacc74c79714101c4a3b6cb0063f71e27..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7052 zcmbtZYit`wexK!X`F=k|y}Y(8*_16hwtP8Tm7P!|<%eupwxq;Xow6(LN~S`Q%I+#M zS?W$r1DqTLwf&)I9g@2f(0))5w{UO)PH8{HNgX&qzbqr=%xrXki>5&D8)Lh@#QxI% zEV(4*D%W1R0%vAt{_}s%&i^-~f3aFk2%bNS{bu6BCWQVCzo?J4(s*bg5W0a_#1bhq zR<4OLLV4GWX_T5ABcax$wA0iWMPM2-rJL4|>6NxNWtcXO84098oblZXTjh6+yo)+GLah{t@X43IAGtJ4S z;gN7;Al?@qj`b(h5MXge_>u5?xC44`AhF^igr+J^z(nf95~|trTCvLIEeKJbcnkfU zoTo&+N}I2`n?NF#qC6~_M?k4zwW^bUfaZ01-IQ6SsMV@_=JgPG_)}*<@To~+-e5#| z4L@Q;HJPj~Pk}c2pOdPd1j=ittTmo$D^K05hESogt0T3VHL6_Zjd|mgvo@l(Hgl=d z!=&h8uGa&!)lPZUnYFr}2Wt($0BghLys3_#UE$}b=Lg*UH7;r^PyJY*H{{8i&UY*E z`A+Lx|4-}O_1a`n8`_TqVopP#3y6TfH`gwc7tkEhhD4XD-A=^2U^P_N2wV1(rJmCT z^yW|J(3W$r&zo6~h@GQ6d9!N4tB8NGj;nV|Z^1_N^z%n}%csivo@>EhZ)3WRZTJp5 z9CdaC6gwI>%_Lp}|FUS_{3CO=u2t@tBfi6P^t`!_Z+6J5E0z$f-4(-S)EJ8|m7$c6Y|_;f59 zKM_8ivq!lUCnnRAv>?JBN9SBL@T7Sz!E*}ZP$&f3oN*YB4&qJBs}ROnJdcYfGSf4- zk<%#@bk0YYM_5{zO(Zx$n3zqavY}iD&1N`(PG>}#yPOn68l)rznn{5iHmgnw$z?tQ zk2k)YeRPHeEo$14SRF0zfI-10N{UHNkWE7j56X%hFQ^L7 zVpequ`{;7d(~8zT7h6i-xjQ28ki+QhHi^EAKK!AwOGSj%&?n4!lxdWPe1>^L^p zTsNNPIF@6T2$aL|m1hNtyf@6s#F z&C52)xo_d*hSs>)xvuqoWwc6;;7ZrZHpzB)wR_F@TnQ1>&f5oz4b8ta{>=E6dC~fN z5_(EHPhB&wJN9D^x~1GwO6q)G@`cwOeI-Ma z(O2}e-1Br5JYB0M$wr)(cd4&R%2&8dDM(N+a4gIVs1s*A<1|4H;!{968ArO)QTKU z50D)cw{QEq?&Vkh_Su3xSoO}WZprt1>Em%gHhykl53D(1US>M|f>xYpBHUK@1Y}2Hs+zWN{ z+Zw>)XiJnzuq|xs72_2XyX|ewBq3I@=T^t4eAUA@>36BJZzfmN#tXCsxtlRC8IM`jXvQ^PGaN3_mhtG4_GF z4*RqZ@))}EcumRrnOS}&BXBwUh0H8H!w3RBJ}W{(%k3FBJxssMWtC(K`##+dxs@K` zGZ>R(c)$_#nM?F^mY&S8vJD^H!jZ_C(ZP}UiKwg{;Ie{DL3x6Ghu{xkQQ8q^Qb<%! zX3&LCY&a)8?1SQOGy{OQZ@@}^$pobpilNiooH{X-GaYzl|ADUk&vfnoVXi5b zX2w$-4PccX6*!(A;`r&LAOPlt2!&Ir%$!U@_;OZ;Pxml$LQj&J?&--xVi1)g(}PQp$LUnPzgCG>8rX8qS%q#3rIedN|UenHV!LS zkVz(;i%Qv&#fVrQpkfKC7gT39<`&@7wq60U$na~Y(TonT@T30P5L z9O7#LK{K#6IJkkq+!S#pXvSyCF45t3lI8bf>0w;p&sV@#A64x5{PW7!Bzw2~g!m-T z9>bIV0UYryAVH_^M)URNCGP#Eb?1?VlSP|<0YKXJg%d^Ba|;7sIo&q`*8@`1o;By* zg_9-ivy|t)yXBsHSHZn&%^h4c0QB=TFZHjx+ZH1wjm{b<`nTWnA1wF}u6D1^eb6iU z53c!-f9fmvhpuTi+`e`AL@)k1lw_{T9q_lki_t9-% zn{|Md94gHosVgFFkKS)+FE)fq6uHe;GNA1{zjhjd%IRvfeNB0NrbSaJggmYH+`9|z z-NlBsqPL^yZZ5X$ycI0CdL`$P4Y&VB&-I?ADX_X^MxNk9r1O}-eg|?jE_D^0yI1@L z=l(^V0-AbOpIzN8*&^$P=zY6)(fZA!GeiX@2EK4KU|1M<^5`5=ne-8-&y6Ppcjd_h? ziRNZg?Yb_Dg3W{cVKAH@g#uCxpU&xp?ogJQPRW|#ZrRi~n|PTMl{*pJaCf|KG%^qy zQLa{CgSS*KAO>I;vQ^)dtR5uzATZ6D*({uPGi9EJsPYo+lPzjX;q8uIfa{tw(Z2Xd zxbIX<5s-7S+ypb561zm49J_!Emoo^j2xV5}@Y!M60>#-`4lo41wXwll{Oo9K^l1jK zV3INeM>(Ix`Amu{Gt!ka;E3!mH)5B$#4OGM=YT}v+juH`x-S}z$Iizhqa$$f9Y1$Q zRnn;{i5V&)WP6zpBsFjgxD-+wAR(l}GET%e1H}Ot2>26#?Y*+CZ+Y_P{}O24fS>R!6nIzf zSaa@PIQf-5aL?XRuy?GCN%oF4`=N#7McA#56kNfT_JZpf*x)?AqsrS^{fzw}8WJ_y_q&gb^lhjd7k8Fs7ZEmFzQ$B|&JKyX4b?Cj26n;TEpISRG zz2$_?W$u4YI4ccLt{u3v)dZ<=@5-gyBFs5_j@anw{jl?c&JRN$grpOkG?`oL`3soT z-cd54!zYPiPptfAHtyZ=U!<98dh4Dm1FaW7n>gkUM8 zgi!h4B&p0~GQndv%WixpmB_$d3U@idDJfbg=XqWEA0<5A0+moIz@ZEwaM*c#KqyBW z=U62oSQWg!<8l5d?t2Xv3f49BPG{KJ6nB(ggg)4Mk?_}0Kwcn-FOlU-Wc(7DzC`BV zqJzIhd;T4@eu2!dTkqTai{n?5SCUJXHCy|Fxo8b6j$XZZ<>FHN%Ac-V4=HOFO^aH~7kVG0^Ndv71Y2rD;^Ppd|i)bvJCy>^% VnEQ+je4!0|u5BujNZY9B``=*f1djj! diff --git a/src/infra/cloudformation/dynamo.yaml b/src/infra/cloudformation/dynamo.yaml deleted file mode 100644 index 864c00d..0000000 --- a/src/infra/cloudformation/dynamo.yaml +++ /dev/null @@ -1,27 +0,0 @@ -AWSTemplateFormatVersion: '2010-09-09' -Description: 'CloudFormation template for RSS Feed Processor DynamoDB Table' - -Parameters: - DynamoDBName: - Type: String - Description: "" - -Resources: - RSSFeedsTable: - Type: AWS::DynamoDB::Table - Properties: - TableName: !Ref 'DynamoDBName' - AttributeDefinitions: - - AttributeName: url - AttributeType: S - KeySchema: - - AttributeName: url - KeyType: HASH - BillingMode: PAY_PER_REQUEST - -Outputs: - TableName: - Description: 'Name of the DynamoDB table for RSS feeds' - Value: !Ref RSSFeedsTable - Export: - Name: !Sub '${AWS::StackName}-RSSFeedsTableName' \ No newline at end of file diff --git a/src/infra/cloudformation/lambda_role.yaml b/src/infra/cloudformation/lambda_role.yaml index bafd81d..58acd32 100644 --- a/src/infra/cloudformation/lambda_role.yaml +++ b/src/infra/cloudformation/lambda_role.yaml @@ -37,7 +37,6 @@ Resources: - Effect: Allow Action: - 'sqs:*' - - 'dynamodb:*' - 's3:*' - 'lambda:*' - 'logs:*' diff --git a/src/infra/cloudformation/rss_lambda_stack.yaml b/src/infra/cloudformation/rss_lambda_stack.yaml index 725d1a0..be04071 100644 --- a/src/infra/cloudformation/rss_lambda_stack.yaml +++ b/src/infra/cloudformation/rss_lambda_stack.yaml @@ -1,22 +1,16 @@ AWSTemplateFormatVersion: '2010-09-09' -Description: SQS Filler Lambda Stack +Description: Redis Queue Filler Lambda Stack Parameters: QueueFillerLambdaName: Type: String Description: Name of the Lambda function - SqsQueueUrl: + RedisUrl: Type: String - Description: URL of the SQS queue - DynamoDbTableName: + Description: URL of the Redis instance + RedisQueueName: Type: String - Description: Name of the DynamoDB table - DynamoDbTableArn: - Type: String - Description: ARN of the DynamoDB table - SqsQueueArn: - Type: String - Description: ARN of the SQS queue + Description: Name of the Redis queue LambdaCodeS3Bucket: Type: String Description: S3 bucket containing the Lambda function code @@ -45,8 +39,8 @@ Resources: Timeout: !Ref LambdaTimeout Environment: Variables: - SQS_QUEUE_URL: !Ref SqsQueueUrl - DYNAMODB_TABLE_NAME: !Ref DynamoDbTableName + REDIS_URL: !Ref RedisUrl + REDIS_QUEUE_NAME: !Ref RedisQueueName Role: !GetAtt SqsFillerFunctionRole.Arn SqsFillerFunctionRole: @@ -70,14 +64,6 @@ Resources: - logs:CreateLogStream - logs:PutLogEvents Resource: arn:aws:logs:*:*:* - - Effect: Allow - Action: - - dynamodb:Scan - Resource: !Ref DynamoDbTableArn - - Effect: Allow - Action: - - sqs:SendMessage - Resource: !Ref SqsQueueArn - Effect: Allow Action: - s3:GetObject @@ -85,8 +71,8 @@ Resources: Outputs: SqsFillerFunctionArn: - Description: ARN of the SQS Filler Lambda Function + Description: ARN of the Queue Filler Lambda Function Value: !GetAtt SqsFillerFunction.Arn SqsFillerFunctionRoleArn: - Description: ARN of the IAM Role for SQS Filler Lambda Function + Description: ARN of the IAM Role for Queue Filler Lambda Function Value: !GetAtt SqsFillerFunctionRole.Arn \ No newline at end of file diff --git a/src/infra/cloudformation/sqs.yaml b/src/infra/cloudformation/sqs.yaml deleted file mode 100644 index 10b8145..0000000 --- a/src/infra/cloudformation/sqs.yaml +++ /dev/null @@ -1,36 +0,0 @@ -AWSTemplateFormatVersion: '2010-09-09' -Description: 'CloudFormation template for RSS Feed Processor SQS Queue' - -Parameters: - SQSQueueName: - Type: String - Description: "" - -Resources: - RSSFeedQueue: - Type: AWS::SQS::Queue - Properties: - QueueName: !Ref SQSQueueName - VisibilityTimeout: 900 # Should be set to the 3rd standard deviation of your lambda runtime distribution. - ReceiveMessageWaitTimeSeconds: 20 - RedrivePolicy: - deadLetterTargetArn: !GetAtt RSSFeedDLQ.Arn - maxReceiveCount: 3 - - RSSFeedDLQ: - Type: AWS::SQS::Queue - Properties: - QueueName: !Sub '${AWS::StackName}-rss-feed-dlq' - -Outputs: - QueueURL: - Description: 'URL of the SQS queue for RSS feeds' - Value: !Ref RSSFeedQueue - Export: - Name: !Sub '${AWS::StackName}-RSSFeedQueueURL' - - DLQueueURL: - Description: 'URL of the Dead Letter Queue for RSS feeds' - Value: !Ref RSSFeedDLQ - Export: - Name: !Sub '${AWS::StackName}-RSSFeedDLQueueURL' \ No newline at end of file diff --git a/src/infra/deploy_infrastructure.py b/src/infra/deploy_infrastructure.py index 25f7575..e843947 100644 --- a/src/infra/deploy_infrastructure.py +++ b/src/infra/deploy_infrastructure.py @@ -143,13 +143,6 @@ def deploy_infrastructure(): key_info = kms_client.describe_key(KeyId=kms_key_id) kms_key_arn = key_info['KeyMetadata']['Arn'] - deploy_cloudformation('dynamo.yaml', 'DynamoDB', - parameters=[ - { - 'ParameterKey': 'DynamoDBName', - 'ParameterValue': os.environ.get('DYNAMODB_TABLE_NAME', 'default-table-name') - } - ]) deploy_cloudformation('s3.yaml', 'S3', @@ -166,13 +159,6 @@ def deploy_infrastructure(): 'ParameterValue': os.getenv('S3_LAMBDA_ZIPPED_BUCKET_NAME') } ]) - deploy_cloudformation('sqs.yaml', 'SQS', - parameters=[ - { - 'ParameterKey': 'SQSQueueName', - 'ParameterValue': os.environ.get('SQS_QUEUE_NAME', 'default-queue-name') - } - ]) deploy_cloudformation('lambda_role.yaml', 'Lambda', force_recreate=True, parameters=[ { diff --git a/src/infra/lambdas/RSSFeedProcessorLambda/deploy_rss_feed_lambda.py b/src/infra/lambdas/RSSFeedProcessorLambda/deploy_rss_feed_lambda.py index cc73d84..5ee5953 100644 --- a/src/infra/lambdas/RSSFeedProcessorLambda/deploy_rss_feed_lambda.py +++ b/src/infra/lambdas/RSSFeedProcessorLambda/deploy_rss_feed_lambda.py @@ -80,26 +80,8 @@ def update_function_configuration(lambda_client, function_name, handler, role, t @retry_with_backoff() def configure_sqs_trigger(lambda_client, function_name, queue_arn): - event_source_mapping = { - 'FunctionName': function_name, - 'EventSourceArn': queue_arn, - 'BatchSize': 1, - 'MaximumBatchingWindowInSeconds': 0, - 'ScalingConfig': { - 'MaximumConcurrency': 50 - } - } - - try: - response = lambda_client.create_event_source_mapping(**event_source_mapping) - print(f"SQS trigger configured successfully for {function_name}") - except ClientError as e: - if e.response['Error']['Code'] == 'ResourceConflictException': - print(f"SQS trigger already exists for {function_name}. Updating configuration...") - # If you want to update existing trigger, you'd need to list existing mappings and update them - # This is left as an exercise as it requires additional error handling and logic - else: - raise e + """Placeholder for backward compatibility. Redis deployment uses no SQS trigger.""" + return @retry_with_backoff() def create_function(lambda_client, function_name, runtime, role, handler, zip_file, timeout, memory, layers, kms_key_id, policy): @@ -219,13 +201,6 @@ def deploy_lambda(): policy = get_lambda_policy() create_function(lambda_client, LAMBDA_NAME, LAMBDA_RUNTIME, LAMBDA_ROLE_ARN, LAMBDA_HANDLER, deployment_package, LAMBDA_TIMEOUT, LAMBDA_MEMORY, layers, kms_key_id, policy) - # Configure SQS trigger - queue_arn = os.getenv('SQS_QUEUE_ARN') # Make sure to set this environment variable - if queue_arn: - configure_sqs_trigger(lambda_client, LAMBDA_NAME, queue_arn) - else: - print("Warning: SQS_QUEUE_ARN not set. Skipping SQS trigger configuration.") - print("Lambda deployment completed successfully!") except Exception as e: diff --git a/src/infra/lambdas/RSSFeedProcessorLambda/src/config.py b/src/infra/lambdas/RSSFeedProcessorLambda/src/config.py index f14ef07..6c36b63 100644 --- a/src/infra/lambdas/RSSFeedProcessorLambda/src/config.py +++ b/src/infra/lambdas/RSSFeedProcessorLambda/src/config.py @@ -1,15 +1,15 @@ import os -# SQS Configuration -SQS_QUEUE_URL = os.environ['SQS_QUEUE_URL'] - +# Redis Configuration +REDIS_URL = os.environ["REDIS_URL"] +REDIS_QUEUE_NAME = os.environ.get("REDIS_QUEUE_NAME", "rss-feed-queue") # Logging Configuration -LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') +LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO") # RSS Feed Processing Configuration -MAX_ARTICLES_PER_FEED = int(os.environ.get('MAX_ARTICLES_PER_FEED', '10')) -FEED_PROCESSING_TIMEOUT = int(os.environ.get('FEED_PROCESSING_TIMEOUT', '90')) +MAX_ARTICLES_PER_FEED = int(os.environ.get("MAX_ARTICLES_PER_FEED", "10")) +FEED_PROCESSING_TIMEOUT = int(os.environ.get("FEED_PROCESSING_TIMEOUT", "90")) # Article Extraction Configuration -ARTICLE_EXTRACTION_TIMEOUT = int(os.environ.get('ARTICLE_EXTRACTION_TIMEOUT', '30')) \ No newline at end of file +ARTICLE_EXTRACTION_TIMEOUT = int(os.environ.get("ARTICLE_EXTRACTION_TIMEOUT", "30")) diff --git a/src/infra/lambdas/RSSFeedProcessorLambda/src/data_storage.py b/src/infra/lambdas/RSSFeedProcessorLambda/src/data_storage.py index 2c4e034..081aee7 100644 --- a/src/infra/lambdas/RSSFeedProcessorLambda/src/data_storage.py +++ b/src/infra/lambdas/RSSFeedProcessorLambda/src/data_storage.py @@ -2,7 +2,6 @@ import boto3 import json import os import logging -from random import randint from datetime import datetime from analytics.embeddings.vector_db import get_index, upsert_vectors, vectorize @@ -10,10 +9,8 @@ from analytics.embeddings.vector_db import get_index, upsert_vectors, vectorize logger = logging.getLogger() s3 = boto3.client('s3') -dynamodb = boto3.resource('dynamodb') -CONTENT_BUCKET = os.getenv("S3_BUCKET_NAME", os.getenv("CONTENT_BUCKET")) -DYNAMODB_TABLE = os.getenv("DYNAMODB_TABLE_NAME") +CONTENT_BUCKET = os.getenv("S3_BUCKET_NAME", os.getenv("CONTENT_BUCKET")) storage_strategy = os.environ.get('STORAGE_STRATEGY') ##### Article Storage ##### @@ -51,8 +48,6 @@ def pinecone_save_article(article:dict): logger.info("Upserting article to Pinecone") upsert_vectors(index, data, namespace) -def dynamodb_save_article(article:dict): - pass def s3_save_article(article:dict): logger.info("Saving article to S3") @@ -94,14 +89,20 @@ def s3_save_article(article:dict): ###### Feed Storage ###### -def update_rss_feed(feed:dict, last_pub_dt:int): +RSS_FEEDS_FILE = os.getenv("RSS_FEEDS_FILE", "rss_feeds.json") + + +def update_rss_feed(feed: dict, last_pub_dt: int): try: - table = dynamodb.Table(DYNAMODB_TABLE) - table.update_item( - Key={'url': feed['u']}, - UpdateExpression='SET dt = :val', - ExpressionAttributeValues={':val': last_pub_dt} - ) - logger.info(f"Updated RSS feed in DynamoDB: {feed['u']} with dt: {feed['dt']}") + if not os.path.exists(RSS_FEEDS_FILE): + return + with open(RSS_FEEDS_FILE, "r") as f: + feeds = json.load(f) + for item in feeds: + if item.get("u") == feed["u"]: + item["dt"] = int(last_pub_dt) + with open(RSS_FEEDS_FILE, "w") as f: + json.dump(feeds, f) + logger.info(f"Updated RSS feed {feed['u']} with dt: {last_pub_dt}") except Exception as e: logger.error(f"Failed to update RSS feed: {str(e)}") \ No newline at end of file diff --git a/src/infra/lambdas/RSSFeedProcessorLambda/src/lambda_function.py b/src/infra/lambdas/RSSFeedProcessorLambda/src/lambda_function.py index ee9c56e..b8e42ff 100644 --- a/src/infra/lambdas/RSSFeedProcessorLambda/src/lambda_function.py +++ b/src/infra/lambdas/RSSFeedProcessorLambda/src/lambda_function.py @@ -1,75 +1,60 @@ import json import time -from feed_processor import process_feed +import os +import redis +from feed_processor import extract_feed from data_storage import save_article, update_rss_feed from utils import setup_logging -from config import SQS_QUEUE_URL -from exceptions import RSSProcessingError, ArticleExtractionError, DataStorageError -from metrics import record_processed_articles, record_processing_time, record_extraction_errors -import boto3 -import os -from feed_processor import extract_feed +from config import REDIS_URL, REDIS_QUEUE_NAME +from exceptions import RSSProcessingError, DataStorageError +from metrics import ( + record_processed_articles, + record_processing_time, + record_extraction_errors, +) -# Set up logging logger = setup_logging() +storage_strategy = os.environ.get("STORAGE_STRATEGY") +redis_client = redis.Redis.from_url(REDIS_URL) -storage_strategy = os.environ.get('STORAGE_STRATEGY') - -# Initialize AWS clients -sqs = boto3.client('sqs') def lambda_handler(event, context): logger.info("Starting RSS feed processing") start_time = time.time() - - try: - # Receive message from SQS - event_source = event["Records"][0]["eventSource"] - if event_source == "aws:sqs": - feed = event["Records"][0]["body"] - logger.info(f"Received message from SQS: {feed}") - feed = json.loads(feed) - - receipt_handle = event["Records"][0]['receiptHandle'] - # Process the feed + try: + feed_data = redis_client.rpop(REDIS_QUEUE_NAME) + if not feed_data: + logger.info("No messages in queue") + return {"statusCode": 200, "body": json.dumps("No feeds to process")} + feed = json.loads(feed_data) + result = extract_feed(feed) logger.info(f"Process Feed Result Dictionary: {result}") - last_pub_dt = result['max_date'] + last_pub_dt = result["max_date"] if result: - # Save articles and update feed - for article in result['articles']: + for article in result["articles"]: try: save_article(article, storage_strategy) except DataStorageError as e: logger.error(f"Failed to save article: {str(e)}") record_extraction_errors(1) - update_rss_feed(result['feed'], last_pub_dt) - - # Delete the message from the queue - logger.info("Deleting sqs queue message") - try: - sqs.delete_message(QueueUrl=SQS_QUEUE_URL, ReceiptHandle=receipt_handle) - except Exception as e: - logger.error(f"Error deleting message from SQS: {str(e)}") - logger.info("We can skip this but delete this block of code if it fails. This means the queue is already deleted when it triggers.") + update_rss_feed(result["feed"], last_pub_dt) logger.info(f"Processed feed: {feed['u']}") - - # Record metrics - record_processed_articles(len(result['articles'])) + record_processed_articles(len(result["articles"])) else: logger.warning(f"Failed to process feed: {feed['u']}") record_extraction_errors(1) except RSSProcessingError as e: logger.error(f"RSS Processing Error: {str(e)}") - return {'statusCode': 500, 'body': json.dumps('RSS processing failed')} + return {"statusCode": 500, "body": json.dumps("RSS processing failed")} except Exception as e: logger.error(f"Unexpected error: {str(e)}") - return {'statusCode': 500, 'body': json.dumps('An unexpected error occurred')} + return {"statusCode": 500, "body": json.dumps("An unexpected error occurred")} finally: end_time = time.time() @@ -77,7 +62,4 @@ def lambda_handler(event, context): record_processing_time(processing_time) logger.info(f"Lambda execution time: {processing_time:.2f} seconds") - return { - 'statusCode': 200, - 'body': json.dumps('RSS feed processed successfully') - } \ No newline at end of file + return {"statusCode": 200, "body": json.dumps("RSS feed processed successfully")} diff --git a/src/infra/lambdas/RSSQueueFiller/deploy_sqs_filler_lambda.py b/src/infra/lambdas/RSSQueueFiller/deploy_sqs_filler_lambda.py index fe8d781..2897f78 100644 --- a/src/infra/lambdas/RSSQueueFiller/deploy_sqs_filler_lambda.py +++ b/src/infra/lambdas/RSSQueueFiller/deploy_sqs_filler_lambda.py @@ -49,20 +49,12 @@ def deploy_sqs_filler(): 'ParameterValue': os.getenv('QUEUE_FILLER_LAMBDA_NAME') }, { - 'ParameterKey': 'SqsQueueUrl', - 'ParameterValue': os.getenv('SQS_QUEUE_URL') + 'ParameterKey': 'RedisUrl', + 'ParameterValue': os.getenv('REDIS_URL') }, { - 'ParameterKey': 'DynamoDbTableName', - 'ParameterValue': os.getenv('DYNAMODB_TABLE_NAME') - }, - { - 'ParameterKey': 'DynamoDbTableArn', - 'ParameterValue': os.getenv('DYNAMODB_TABLE_ARN') - }, - { - 'ParameterKey': 'SqsQueueArn', - 'ParameterValue': os.getenv('SQS_QUEUE_ARN') + 'ParameterKey': 'RedisQueueName', + 'ParameterValue': os.getenv('REDIS_QUEUE_NAME') }, { 'ParameterKey': 'LambdaCodeS3Bucket', diff --git a/src/infra/lambdas/RSSQueueFiller/lambda/lambda_function.py b/src/infra/lambdas/RSSQueueFiller/lambda/lambda_function.py index 6d24bdf..9dc6a18 100644 --- a/src/infra/lambdas/RSSQueueFiller/lambda/lambda_function.py +++ b/src/infra/lambdas/RSSQueueFiller/lambda/lambda_function.py @@ -1,57 +1,39 @@ import json import os -import boto3 -from decimal import Decimal -from datetime import datetime import logging +from datetime import datetime +import redis logger = logging.getLogger() logger.setLevel("INFO") -dynamodb = boto3.resource('dynamodb') -sqs = boto3.client('sqs') +REDIS_URL = os.environ["REDIS_URL"] +REDIS_QUEUE_NAME = os.environ.get("REDIS_QUEUE_NAME", "rss-feed-queue") +RSS_FEEDS_FILE = os.environ.get("RSS_FEEDS_FILE", "rss_feeds.json") -SQS_QUEUE_URL = os.environ['SQS_QUEUE_URL'] -DYNAMODB_TABLE_NAME = os.environ['DYNAMODB_TABLE_NAME'] - -class DecimalEncoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, Decimal): - return int(obj) - return super(DecimalEncoder, self).default(obj) +redis_client = redis.Redis.from_url(REDIS_URL) def handler(event, context): - table = dynamodb.Table(DYNAMODB_TABLE_NAME) messages_sent = 0 + try: + with open(RSS_FEEDS_FILE, "r") as f: + feeds = json.load(f) + except Exception as e: + logger.error(f"Failed to load RSS feed file: {e}") + return {"statusCode": 500, "body": "Failed to load feeds"} - # Scan the DynamoDB table - response = table.scan() - - for item in response['Items']: - rss_url = item.get('url') - rss_dt = item.get('dt') - - logger.debug(f"Processing RSS feed: {rss_url}") - logger.debug(f"Last published date: {rss_dt}") - - if rss_url: - message = { - 'u': rss_url, - 'dt': rss_dt - } - logger.debug("message", message) - try: - sqs.send_message( - QueueUrl=SQS_QUEUE_URL, - MessageBody=json.dumps(message, cls=DecimalEncoder) - ) - messages_sent += 1 - except Exception as e: - logger.error(f"Error sending message to SQS: {str(e)}") - - logger.info(f"Sent {messages_sent} messages to SQS at {datetime.now().isoformat()}") + for feed in feeds: + message = {"u": feed.get("u"), "dt": feed.get("dt")} + try: + redis_client.lpush(REDIS_QUEUE_NAME, json.dumps(message)) + messages_sent += 1 + except Exception as e: + logger.error(f"Error pushing message to Redis: {e}") + logger.info( + f"Sent {messages_sent} messages to Redis at {datetime.now().isoformat()}" + ) return { - 'statusCode': 200, - 'body': json.dumps(f'Sent {messages_sent} RSS URLs to SQS') - } \ No newline at end of file + "statusCode": 200, + "body": json.dumps(f"Sent {messages_sent} RSS URLs to Redis"), + } diff --git a/src/infra/lambdas/lambda_utils/update_lambda_env_vars.py b/src/infra/lambdas/lambda_utils/update_lambda_env_vars.py index 8936636..4aa7b62 100644 --- a/src/infra/lambdas/lambda_utils/update_lambda_env_vars.py +++ b/src/infra/lambdas/lambda_utils/update_lambda_env_vars.py @@ -28,15 +28,10 @@ def update_env_vars(function_name): 'S3_LAMBDA_ZIPPED_BUCKET_NAME': os.environ.get('S3_LAMBDA_ZIPPED_BUCKET_NAME'), 'S3_LAYER_BUCKET_NAME': os.environ.get('S3_LAYER_BUCKET_NAME'), 'S3_LAYER_KEY_NAME': os.environ.get('S3_LAYER_KEY_NAME'), - - # DynamoDB Configuration - 'DYNAMODB_TABLE_NAME': os.environ.get('DYNAMODB_TABLE_NAME'), - 'DYNAMODB_TABLE_ARN': os.environ.get('DYNAMODB_TABLE_ARN'), - - # SQS Configuration - 'SQS_QUEUE_NAME': os.environ.get('SQS_QUEUE_NAME'), - 'SQS_QUEUE_URL': os.environ.get('SQS_QUEUE_URL'), - 'SQS_QUEUE_ARN': os.environ.get('SQS_QUEUE_ARN'), + + # Redis Configuration + 'REDIS_URL': os.environ.get('REDIS_URL'), + 'REDIS_QUEUE_NAME': os.environ.get('REDIS_QUEUE_NAME'), # Queue Filler Lambda Configuration 'QUEUE_FILLER_LAMBDA_NAME': os.environ.get('QUEUE_FILLER_LAMBDA_NAME'), diff --git a/src/launch/launch_env.py b/src/launch/launch_env.py index e222dcf..cdc85cb 100644 --- a/src/launch/launch_env.py +++ b/src/launch/launch_env.py @@ -62,8 +62,8 @@ def main(): env_vars["LAMBDA_EXECUTION_ROLE_NAME"] = f"rss-feed-processor-role-{env_vars['AWS_REGION']}" env_vars["LAMBDA_ROLE_ARN"] = f"arn:aws:iam::{env_vars['AWS_ACCOUNT_ID']}:role/{env_vars['LAMBDA_EXECUTION_ROLE_NAME']}" env_vars["S3_BUCKET_NAME"] = f"open-rss-articles-{env_vars['AWS_REGION']}" - env_vars["DYNAMODB_TABLE_NAME"] = get_env_value("DYNAMODB_TABLE_NAME", "Enter DynamoDB Table Name:", options=["rss-feeds-table", "custom-rss-table"], advanced=advanced_mode) - env_vars["SQS_QUEUE_NAME"] = get_env_value("SQS_QUEUE_NAME", "Enter SQS Queue Name:", options=["rss-feed-queue", "custom-rss-queue"], advanced=advanced_mode) + env_vars["REDIS_URL"] = get_env_value("REDIS_URL", "Enter Redis URL:", options=["redis://localhost:6379"], advanced=advanced_mode) + env_vars["REDIS_QUEUE_NAME"] = get_env_value("REDIS_QUEUE_NAME", "Enter Redis Queue Name:", options=["rss-feed-queue"], advanced=advanced_mode) # Advanced Configuration env_vars["LAMBDA_LAYER_VERSION"] = 3 @@ -72,9 +72,6 @@ def main(): env_vars["S3_LAYER_BUCKET_NAME"] = f"rss-feed-processor-layers-{env_vars['AWS_REGION']}" env_vars["S3_LAMBDA_ZIPPED_BUCKET_NAME"] = f"open-rss-lambda-{env_vars['AWS_REGION']}" env_vars["S3_LAYER_KEY_NAME"] = get_env_value("S3_LAYER_KEY_NAME", "Enter S3 Layer Key Name:", options=["RSSFeedProcessorDependencies", "CustomDependencies"], advanced=advanced_mode) - env_vars["SQS_QUEUE_URL"] = f"https://sqs.{env_vars['AWS_REGION']}.amazonaws.com/{env_vars['AWS_ACCOUNT_ID']}/{env_vars['SQS_QUEUE_NAME']}" - env_vars["SQS_QUEUE_ARN"] = f"arn:aws:sqs:{env_vars['AWS_REGION']}:{env_vars['AWS_ACCOUNT_ID']}:{env_vars['SQS_QUEUE_NAME']}" - env_vars["DYNAMODB_TABLE_ARN"] = f"arn:aws:dynamodb:{env_vars['AWS_REGION']}:{env_vars['AWS_ACCOUNT_ID']}:table/{env_vars['DYNAMODB_TABLE_NAME']}" env_vars["PYTHON_VERSION"] = get_env_value("PYTHON_VERSION", "Enter Python Version:", options=["3.8", "3.9", "3.10", "3.11", "3.12"], advanced=advanced_mode) env_vars["LAMBDA_RUNTIME"] = f"python{env_vars['PYTHON_VERSION']}" diff --git a/src/utils/__pycache__/create_lambda_layer.cpython-310.pyc b/src/utils/__pycache__/create_lambda_layer.cpython-310.pyc deleted file mode 100644 index b3000116fc5f2aafa6d1288d0c98444363423b94..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2201 zcmZuy&2QX96rUM?tk?UICT&B5L@`tlEu~q?76~dqwA-x|HC@_lOBG)%$1~YD*|nGP zY}#0x3ejHk7eJ)+*gJm%7p|P<%83I9j#O&PoADBDC2Y-`55IYD-kbOPIjz-P1kdk( z{kCJJ$i^A zqDHMN!T58?A!mI^K#+t(1}U|#kE2~Ca;NCM<$QkG3kRHqbl+o-;tcX4EuC>`wo}S{ z|ml)uylJAxIa;qzA9A9JmnyUSjr zKhl*(Nm`q!NL$j5^!>`pHnl3A?aTAib#OI3LcA3usR)Yf)xb^mA&5FAq)HZX9bN~YB6VUK1e=7s zwqGsSrcrtR9zX;XRO;ki}x(j9iNLT?`KeCke z1ntzp@|@D5r+y}MzK`j`p?N^S%jpTBXO;lOM`U`6IXHnj4>~>tR@KO)XVo{I^u=yo z)z#1(?ThU+xd*Uv^yLTXZvN@bIEk}uM!nVIPU58R-3g=Ju+K7Yi>E_xHj|2kqyZe{ zTiRxR?qt6ok47__rCn4aFGDK1sf5aKUcNsXq#*@|`J5+Guh;U5x&&;K<(2%x4F;u6 zSrWzS04evN%)OfoHbC*K`barch0|+Y*8udbUh5Xy&)p)r*2?9}4bv~LDFW>RiO9+g zmPI@sDGu;U0CNFs>Myj@;Rvw4IbiGKM8rdu<>%W4v!N89lputD+jP(Kb1zd*#RN+|fr%FOS>uZ(p*H87;(uinv?{j;O-<^V1> z&_M*#Ab@#>f-|NVXqfJ?h9PTH!YFu%FDNf_N+<%F0oLjN;g=SuWFLd56W4HX-GpX1 zhHKREB3ASPF98yHWt#Rs3$AOpxJvj%*dU!C7y=guK?8Rib47&blt)jA3rgj*V%;d^ zY=u3Fn7+TP;8`2M&yqdQ@p3Gx6xiG+Q`d4pTZFu&*)f d%FcsukOTDR;3`(9HL;6b;=JROor^Gg{V(a;ROSEx diff --git a/src/utils/__pycache__/create_lambda_layer.cpython-311.pyc b/src/utils/__pycache__/create_lambda_layer.cpython-311.pyc deleted file mode 100644 index 546b0ef85e8235b6bb6bdba8c36f245d2825999d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3829 zcma)8O>7fM7Or+z+wI@j!A{~pz%#%E6No1m7)VwFArp2ZLY7HjBs|i#9CsyY;=ir# z05QqRT5;eoE6st?M4FA1%di>2N^@Chud~;^w4)vH&pUcGwX`-|W2K~RkU{&V3|FGBxjKaIlI8!!J3jjxf6WG;(R_Loa>*1sd= zu)chXhp!`xb3#hskOL9NEoVcJ%s=7~dJL~!Qm#GYc+a>f3n@2oeS>7@rX%H%y|U{* zm-5QueU$RaZlHb=kUh%|CXf0+n*R~B9^2x9a-i#+^~QbwVZAeP&h$=X6_VE`DJ@Vl zG?$d7$8S!~N>?UtN;BhEC(UrP>(Y&hE0fo)-qMgtGl^_EH!r89g_V3pQwn+bUnNvc zQ1bf~g_0adRfn~^S_u!Alw!tGtF8+(0yE^x`>_45k=8TZUdL-#7VAczUgKo9-BulIT)7u?Z60`YIyBy;Yd-q(+_GnlQ@AJIPHVh#f4;Z( zmv^l3f8=h0ldkiB0G9+Ouv&MxJLoQd6W!tB{u$GuCcguJpkxsaQ_81v1l+@UX(h8v zw329lV}qs8U@V=bBrUJT$X!K+Gd2a=d+0N^5 zrFly;Qz%jpNtrvDA|+}OmLaC!o?T~(3#Qj*xAtW^?M<3)H7V6)a>Voyb_t{m>_TPd z9%JRrcw(uLBZ&-IJbx}Rr^w_QE!-j*O-=l^Nb<9Da|w1ARx~B6ChCjUFNSp=hKsA_ z!FoTeR@XvVkn#mhvJFTFU@H%>6GTIGB9{M(}Z{OB~XDh+8M(}KTs)l=n@G~*= zR284EBGpLG!_=0` zh#cGQF(My6@Kn8#jmVDopzb~RtShpyf2Zq+-gRW#xgB^Gj6Mwx?gR(7@9Dw8N^r~w zj_FpdIZ>$J9;@#E;91{^?S&feNc7YM6gu_-38C&+jMNx;?L+=Zwuv!&;df*Vy2e)c2yx;i(VZrr^kyv2FdfUb(AXPWu{i@Xro8w8%Z_euTmL@Y-eF(P904 zYYsWw0>5w2rem)j9cSFtkOQX*v8|Quz^OKY9?mV;;A_|Etr^YJ1wV=G9ks4&&rHc@ zh22DPRa+|LuLFdMR+NXvgF0& zA$KtaOJ=FBIX|Toi*1ulrycSsvo&S0hqBvNa^EOs3uze^EOo`SLToM>vjG5b)G;_* zI{XO%43SAbqc8;uGzNGQyG&pLpbU#F1q$&*oJ(G+gWX*63Rx|A?A;}YM@Hhq@**#In6#-Z0vt)K9uPHgAmJUwX@EV)8vbdPF!`N*| z;9%-#%yfNbH%o^)7%U_Mj#vgnW{F1Li-$}{K{dr(dYQ23W(6-ZG&}j}jj7qmtCKU= z=cLP1(~~p|7BQXb5`#+9$JnJb%`7RmAv)SgCrdafyUw(m5eAEg3?gQ2E`R9FycOwHefe(3Vh!GJD#5X>!b)jJL z1rmeKSB#W@^E!+KkM6m+eeU}yV+h1wgj&Jl1|HY(IMdjB|LT{672I#&ejWGkg4w42 zv{b>pU<4iaK4<0`s^Aj_KA|@%{Q&H2ijtIrWKojh+)TX7c8m95fU;Ad3>eLQEG^6w zC>bVqGsJRw)lLMRfVCulo1SC5=NUP}$QUEctt=yHPHs39J-vrI9mpl1yTI+%sdq z#%V*f(q^Sf3sNOgEl5$d4_+k?`>+p{`qcJuUmS2mI;0X(dFfjUsZ`OI{%6LH4Q16m z(wuYV{I_%d^Z(y}z7GVv2-^7a_frKQLVu?}^~TjEFTaDyeWW6lNu#7QGD*h%XOpZw za!C$GHqB>*q`)8+VRqJESJW6)ZiR9FlI|bm;dgjMRY-cU`zxfn9QmjHF-e~) zeuk2M)dPJ12UYLf5rl{c`T{F8Qw%CF1t#YI2mK$7F_y1Ct>LUWK!{GP@OVNV?wc4G zlZOT-T4ifun?nP1FK9fK0K@aS)Nod9K9G+E-p*bw<_I2@};#`+2^^tD^1H zOC?72(3!!On1x-iqrmHO^zZc@WurgDrFu(@#;p44XMD@<{mr}2vL)`<%mf77Joihk zP#bSDH_5M4eMq*R*VlvWk0l85~#e^trR~aZSZn zi9U-J(}@2xhqGhj<8d0=*+_47T+dBo{8ZnV)vTZz5HoK?5>I@H^snRacTEbpmOL; zCHVfjc=q3$zP1;rRfudv+wLdtC70c6(Ie%ywdjdO?}jhB6n*5|`>Z9pwEI!Z!Lq9y zd=~0_66#qC^^|W{LOtuD3tKJ}?sAqkb{~G$@osr)i(}($TLKCn*+xRRb=!{ut#?0O z`gCce+*N5kz3zVxgmy(1KiYPqz?uJS?E~|#j5avO@Ah9nfAm~F%>9Y)>$tq1`}2MQ zeqKvznof37pNGl+PimD_xe|ipWTstpTmy7MF=5o=a$3%EYY<_nbh^kMC-6a>SzaSQ znIpP_4a4#(Ggz6E72p6$xEXciDJ=~FV5Z14Hf2>K1_cjEB70!4A~n5ovj{l^q@(Z| zbON`=2cPhrYkcR17+5T9i2h1&AM}f}TW)^0>tBc$+%JIcRIS=4IOR@EhW_HyV8rmCcZ#{{>K?k@GH00nf z_)w@@WB`_DX^y_pntavLkZtHxLl4+#q=tXbthTQ3j_^F&FO=U$~W8@q=4wltJl#IU74^l4#sxw6O{~)HYi5+kh3%70ekudkw%}wDC#S=ZdGU zXj#pe!KyUqq?neSmae9hxzsc^q$#3jq~_e9Mcm?`hFgNIv)HjCpVV@>CSl9vfF+@E zM<^f=niECO^;}v{sbH|!BAL20o{${qgbKv=^cD|%hyi_7oK-Zs0i8&I#L^WE0)Wpn z-E{&luSXquml@s?+pK2k=Z99q5)v7rzx zN2{6ThJt0ni^Xc~*E2b&T9au!md~161{=k_{Z8?chV2)RWP3rn0goLJJug}APn>CS z-)jtZod6C=0hMWN;{CC(#p;G7W>RyQ!jg?TR(NdSXV(YE21W)(uZ_!B28Rbo1iWUs zj2T*pEI(zJQ$(53ZUC$~4>bBXBQ#+N%~iYTSiYtwn&r#eK6kKCOb%YPHCxeKeliW0 zFWZNZYlnaunlTB+w3cS{8#u;Ufm+>jDk`CYO=!v+6h)wvI$*l%Ek%-i02~R*%0NUf zpy#~n^PxLKcZ21~I^VnPLPBS?^Jsar%3s*zJr(c8Dt~EPM1t=L-@eATKkzOOl;5xN z37dGZ${zxeXxIJWd&A2e-`=i9`WHs1k^^5~sPb``h1;A0`#AK@!r%rUaDe;4k*|lo z>!|YQLCxaeZ)ff%mwvT+yPWyMDaRsIx={C%mQ!mjr#4&nY(&mI?>Mj-?f$30g}pCC zv}cUjYC)mIOC*L|3mi# z06WDQLx*mQV;JTs@;^o4r^xdZ1^$MPfnS2{i(_9-d@)gJKl(UuY{9z$3zJ`JUucz% z-p9fCf^Q?zS3%*8V8`Ommm(G4-Uaa`-^oN?oEI4Pi`E!(eCrbi@&3i)Bd+tWeET}z Mxy2#A%ihp`0b|NB<^TWy diff --git a/src/utils/__pycache__/create_s3_bucket.cpython-310.pyc b/src/utils/__pycache__/create_s3_bucket.cpython-310.pyc deleted file mode 100644 index ae2fec85150de897cf8e5cd15f12ede88a733134..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 877 zcmZ8fOKTKC5bo}IWcHDjY(xoy?8z{P-ATwI3KGOc@E}oKBsiC0Gu=sMvpcsfh$n!DG9>N-n)??mtu3QkffG#Y7tr=UhDODZ zlEQk2_t7h0T3Bx^!&iNs;#1@|_cW0^WPlm#(6x_>ViCX_uFe9U4cgJ(0Az8DSz zDdHeg$wUTAMu4oSdb7E{{r_Q<97ZWuQKc9Bp-A2V)53%_$qn4XE#hI1w6INx*}vUi zZP&J1KfSl)ySNUi`OP`cL^n&5*BR?pR9*8#c`cJ&Hj=8l?c+V)6*b7#>qfem3)W+2 uF`Jo(kfm59m}o(cxMqx*_lj9J#omJ{r`d$3 diff --git a/src/utils/__pycache__/kms_update.cpython-310.pyc b/src/utils/__pycache__/kms_update.cpython-310.pyc deleted file mode 100644 index 4101d85ead8dde551873920a1c59fa56f2e5f898..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1924 zcmZuxOK;mo5auq$hp311aGJG?JQ~M0aU^cXLfdWzn%GZ;zq+#;5Ys8 zx6xOi{Vt8Se-$u3hrzc&5QU&namp`pP+?;S!xx9@xZ+fhLR37>z8W~R_dGc%qkFR%m*)?8+uSebfg94X`!tgB`Q%gAYRhMXny>k)!U9d9HH7^ZC2|ja#dI_xAn4%Dwf?fjj7LtU0H)hr`>Hk}Vedl=GND zjjBW8`TGNJOy8*xhRSF#hO$BnAOa;*zO2qr1{7&2AW=*C34e`;N*hS9nTSau?$D`I zE7b>Hn9yIrm*b_YxgHmOX23>;_~&}}Q9PzypYDEky*msD1x$O?7reVUp%LKha^`mv z5rn)8w{nvS@dRC(Ompi734GzlC~OXVc<`fZiKP#h5Y8y zo2XnbuFRQaq+^tLxifdy8}AVB&c+a~(Pxpuz+;2U?r#rzc6z$>+R|N4 zIY!_2<0KNPb?xf%wa)U@&hn>e1Hg0$9eGJ8I?L(BwaD9ni1q%4eVMl51G;nWt7s14ZKT3VBvu2P>(CnB9I4eqmGhr)GB z?^;xP=cV@x8c_z-SoMSlD;CqHoZY6!ix%5zZakn&-pRr1q@b9fW9S6Aah!>^>Sz$U zP8@5iD3l~%u9te+k*Nl3c3DKbEG@Bl;!EC+k}#awKzq3cN^#|_%GgmXz?DSA4kDXN zZrdgK_)>bg=tSjqL9}bXdrZ%6H*#$!7V#&!?iZaa$1%?h8hsNmNT|9?MLtJpnH#s* zoafpEGB2F7;B~YvfBguHe^Xl5SKAjY$f;Y?B_1r49 zbR(IIsy}i|5RL`ueXYamKGEmHg0b!>iHeGsfhMDT0&JqWaeQ${7Px=`oO0%lqZkm^ zg^AqqO4@+baFe_6Qlh#qC`bMw$HYti2AiGyR}(cHEl_){38pdouWchhPquTlt>*E{j&3H@C{!W7nz+ z-mWa4bFaENg!zxrsz5~S-g2?o+9+{tdg^x2fz;p*zp;SD$AErf&tRHWuL}`@cpq~h zg~w@NolDwJ%=2$1T8?qle=VCKd1FD-R846@LFt8uhv(HG(0yba(rq?*%(XAE!@ULP zI)Zb(1;-!3@kS$v;L0ZR$8HCQqzxHD*o}quKTANV_Ym)E@$=|0V4gH_J7QUUi&y?5t9Z&H4zw)oNJg+}2H{}Y^EvwBNW3`Ih6$CW zH(|>BK{Hia7SRz;amS&xy=9N~Gu(;8xC=*cH|{BWKSwzF1)t$?FMbF2JrZy~)0CwQ zw}J4$A2>(4;lhKD_#I!__2#53d+;$M((+={E)zB```Y92)$zx0u1}h0dt|RC-W(5% zlI<~(+pR*f1L8C?qAeeWkT$&IZ;RwfqZJHPK5|lTit(?&?vDSL{`=g0^b!A4bf1fz zTBac-s}}LPs^!u~M$J=xE2mRlH+0&4GjZ$cd?K}YZ~5BYrQ6G?<;1N8+Ic6LTqFdq zXsSYVUDcoxy-6djY%QmAq^%BF?y@+(ngFtoGN_E+5ASeg_-CTr5dOVZP#p5L*uBoa~5+ES)G0Kg_1j<8nskpGd4$-CF6)lrjGKF-O ziV4NeAL$O!)uN`5(tvGOu6t~#a#1I;@gb4Hx0q1!Go$lF(Y6alslyrQik4X;uw6&H zkax7*O#35}CmI~ueA-9@VyqNm(mSLM8jJDN^B&RIo~UaH+evAV9NZApQ;!5d1egj+ z-->;-EM=+a@TEx>YVb{Vj$L+=0n2((VVS;O%x1S`C=`Z)qj>kw$;f%t097%M0Fp z8;3(HK;#V?0c8r3GfT7-WPRSq7~3=ig>xu>se4lgI|44|3Oe;Je58?)9U?fn$&2zn}ZvoH=&Iyn44XaL*dJXS(01x#L3UWn}Pq zWa3$5;%Ui@OjIIQt;p3qscM({KYnc9d*8bGb2IgUbu(?QZCN+7Q0tYhoYj@v^Sq26 z+u!do)=5W;vv6=@3NB)D{!O(vhsWI}u z019@QUAMm;2LinkSg`^tW}|-_2%B9$s01!qflFqifA_7wyM|l>SZEMCP#6BnKTZ5u z{EKLYE0N?y2kzy*xAVzV1<=RY2!@BZbI!mBOylE?PzECYw1!>#ljb0OwyB{Xmj{YCv<~J<> diff --git a/src/utils/__pycache__/kms_update.cpython-312.pyc b/src/utils/__pycache__/kms_update.cpython-312.pyc deleted file mode 100644 index 8f2e76cbf386ebdaab69e9f7a0eac83a74151ddf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2690 zcmai0O>7&-6`tjC`A_PHB5nOyE9)nwEJ~4;wl-s{rYyOTEz5!=r>FpTv)UOcOD%Va z-DPDi$ifJE(4jyg^dJRE5Thp-&Ot?c=+OI72_2Nmi&G(88J zp6}444>W1LkwBF59rf$-Is2?{4D7{BBMB965S6K1_ze{h%jiqrffuICV1GNT;($Yg zHl0Er`2>%M1kX*_U-6ZFIApKG9Jc3Qd43u7qKY37|F@naa$rAD_T#9%rxMhoFgXLq z%7IVO=LcD#&=IVu9DE%s)48U`&cpd$oyES+=yh4eih#a>O*?cwS`bWQoAdWBN;lW+aWune30qabM_%z zvr6RWkqv4C-J*YjHmFqJoD~x!rHEG*HP7p!B3tx&UbART*Q~bL?6t|MEO+_F+@+vP;(_S&@7ys)@KQN-}q?E>Fl`l$qdd%pYmiZOfF_XQ<-q483&J#5+Y8r&ASxi=v zlnsl{{&>-d@~WKSH?)k%=QA0D$*78=8w_M5Gh|UiDaN7~MQr(|S67LkTfuo%lm)TC zOO`JyIN1v<5KSqn0x>!ryKMc$y22JU!s;Io7S5JpjIeEViU?|{pc_rDpb4tDLLgih zot9m+%|?GjWTNsqnc{UGG=_|ry}Cf^N<%5y^1nw^J0{CBgPoG-Bo8l|<*#P~R2bL_ zU}#>EM4{vs%;k-=ooYB6TW~P8Sis;8vszKG>%3Z&q!J5lV(UOD-a1klR#tR?iZW(J znbquGU=_FWBSw$&qPpfpd5wMDO@>WHEoMd0m2t}_I4`i}Q8dd>>`V28oQd;uD9`j$WnZli0u z3eH20Nlvz#Hz=%f^+SXpy$3wph6djzIUUQb7G?kxLf%}((Ce%i+ z7l;gZa#T}=(V{L&+NkZ(E#F9?WF=gdYn;RNqMipA_9sO9Equ^!qbJRY?Z{Td>`w1B zkKCSl(vsZn+UnXNdoBI9ue>33c7KuiB=g0@Cllt-MRRg#uk(iKJzn!>nAp>J*Q5Aw zH9ov+nDODg_~d3_e|E{dakD!6Q(yE5udVs}gCBo?=X5nbc-MEgQayE{ z8oyw+<~RL7o+i7u*SFTqljD2IcQ(WO9ldu(s~u-Ho1O%sA4hlAcW#=o(+>i}Aezm9 znZ0Ms*!Y9MJ2f9lvX2tu)x`K-;(NPdHF0q>^p9{7*1JyaCkJ3=FYKR8J?-kfb8|Pd zJNyS?_D$7j&tL~YeM9^GZ#`=YB$BnZL%kn5@N=K9ucH=21E-#K#y}y_exmi6zdIS* zyj+_=(PNJy1J%gDezJGJv;WDl&TV7MxHGtSELHQP=-^A_jmBO^P_)%-%iceIzuRoR z{xCBCuSkp8`tHNX`Tss&qflI?{_`?Np-AF6@-Q(cw$b0yyV>9Q?%pzEXCDO4?FXX& zt1$q+(#D{4f9&mu(tyD?cib_UE<0fgU z{fToL%64GiB#u-2?X>}$i~#u!dI1!v?*e%hIG zoJw&5(o6dVO2C1>;{^5GP7*vMJ%Y(6no%JDI3H= zphOf=fYovb&UT{Ry*ZlWIf*I9oILkuG^db}9!4&FwFoFKa1%st13(CxlR1sqRT$1) zp%4bn>s0~Gb?ajg{W}21GqNC%gDmJNQsLBJ;2cSm<6rU5_z^mWEC!u_kI{@_G<{%j z&YnR(a`uF9vV%AUVP{Bv?HIUs3!M^m=K|%FS8}q6w$RKo^pf%Fn-RYt?<0?S=Oe z*o=2s$qOxqet&^xB&Q2>7tLsjFVG|O9sHvi`+=oixlix}l@)8>RC#?tfvfR?KEoHd z-8@vTQ`Rg03ER!MXp`Q~kjiR-)M1tuBt$@be*0y zyJ?h-QzMk%t?63OIub^m%XH9^X=~bOX%S^9*Fg|WxA(?r;Z; zq|iazSB}(rBPCN~eI@iLOSQ1oBpVEX)H0dInPo-}h4pkI#K_jJ%W79F3)Mi|%IT!o zt5?LfdJ;ybSsYs|dbLR?P2VxJnccW@^oQfb$PrAevMh!=?M<0VBg3S>3T+I-r~NC! zSLpAc^EexdP9z3*zwGR%1EI~)@o`71s53S)(H(d7ybp!TC>WjF>O_U3kx(`GKz$p4 zVTOG|@H$?@KHk7hOh7)xb+GD~kmf)BkK`}^_4>`}=Ox!fn#Zehf_q68g^9i&aOkiK zO{V@kx=N)!fHZBQc3@K<0lcE>meb#J@IF8rm(CACx3y~~A585Y(FSJkOh0?8>}4`m z(3S~`R4rgBT*|#HRV3Q@KcO3_BI7z{@2XG0YE$LjWHD;W6$Hbv4x8OP$xH^HECR8b zQ(cUWT61oaY1=pnwT#?2%Ym&7gc0dkL0VSg+EQvoIsUz2#>a`c?;eM40{Ey2Kf%Xa Hc#ZrE&p1@I diff --git a/src/utils/__pycache__/retry_logic.cpython-311.pyc b/src/utils/__pycache__/retry_logic.cpython-311.pyc deleted file mode 100644 index fbe939cd491684ddd0de4f06ad8638886d5255d1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1872 zcma)6T}&fY6u$FQrlqh-3+=G{4S$mEh8B$wiAdZf2xbu#VzP@5(4Dzu2c|Ri&S0S# z3~op$4;oFDP@@|$Ym|*X@aUuQ)t6z@hh`FzO?=|52{9r2a2RPeEK?5<*)))HbL`>p-@}!-|+yc9YJ* zZUz?V2t*zQ5Q{bi8>z@eEw5JZqu0+Uk>(Q(TgC=ePnY=mCfcrZO(_#s_)`lif|e3Z;|4+eSm?7zJF`8 zgfls4fhN*3`-O2f3%Y7ye%9tTt8yXmI33j8f&E|QKjU@(Nth6eO^*^DHoJXEE~FF#^do^ z%~CUPZHUXKK$iQG zs;ico)MhJz!E|zY_LR<~w^s6O4eZ%q;+~Pl2^r51zLA(#75o7)7O-rYiT5*DpPHUd zm_$x2S*m6xD8>NAqEXkJnHs63vg5BHj71r=iBettX)#C)87BYwKD|O>KqOcj}#{zTF4_E%klJ z2hol)3f8rr!$Eyxq5bRe2jgFjm!jf>+)ht1dTn>07=6tdNIOwlN=+R*i+eYIFrGB^ zJ5Bwi=Jrz8_1#bQ#va@IT|boEQI`f-Q?0x(9NU?VuI%v6_Jat<2ebFe2 zh9epWV)J*^JA?aTZ&B=Z#NLwFw)5&YH3b12r?Y=w>@Nrf;i=eC_Mt|J`t)}URt|%7 z+6pLqwX)j3TT!^#X}-1hF~CXL4^0$96Hd*9*#NovLTEr+)i+<^zQ6I-&35jOb|3hU zQOrl0;47gI6rs-{X{OKuVA8`qC&50GR`A@w*=FRrFSTbh-h30_81|^lh6C?0Wpru0 zhF*^5uN*-*A`(E1_f(ze^OUwjuSwsMxw$#%L}TbhVWbmA&{1X|0OXBoiTI*r^uG{C zH4P45PdXqd#t};3-F89`9%!k9#tq*!ECWs`j^oNxVR|>$o79u-5qRRB`G?YmvZUc* k(hb4%;Wmc>${fQmB@|rqekF9>dA^i+k!dKS{{WBwAM%svBLDyZ diff --git a/src/utils/__pycache__/retry_logic.cpython-312.pyc b/src/utils/__pycache__/retry_logic.cpython-312.pyc deleted file mode 100644 index cf610f9b08295b6239799b1863e80a71e2a18f49..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1854 zcma)6O>7fK6rTOD*Bd(l;^3boB!(oVhT_ojBakQ(ppXI~L7-M5RdI}WY&Kc%y0e=& zS#7EaA;pnUrAkFOst~T)0|*Y`)E=uIs!F}A`BB}4N=QBB)Tl%)O)q`3wo^(gwIl7D zx9`pS-uGta&3DbsO$f%7(~mOW@Cf}xe;UJESDsCSave!XViYt=XI5cy>?q6BY}_d4 z`uI_vMVK3R5gQ=28SMOc;O{p2Cvg&ghppM~dhRea4R8g?dxaSlW|+8dnHq=VjN?D1 z$XGQ72+@cm4vh>ZhX&pm7;^X%!^cO!?@{m+R?=Jz#FbR1-?MfQ^Jt!#XRo6oTV#qT z!%55)-z;Zw=TMv-PSVD6@2CV*Yz zVau3Vc7f(#7sLrDlZYZ>(Sl$h8ClHd%!dGCY-m78O$*{DNb=v{8XIw$=|M)rjd~H8 zpXH79`YgV&jzJbbPD#HsBsK~biE|}Cg~)r9iR3Zih$dbIKFnogagx_!P2g^N5SsNE zLIY%+*7MSkg#XS2HTWzKNb$xCM{4*RQ2%xjtQS^VEHG>aF092Zc+FPOI2D+lhwC=W zf55B0+FW1?Xqr2Z3e5WcGFR_iy&Vq>6MCl{PwlGk$21AYStoiH>)Iqq<71kdQRK8S zFrCJEL)KI$)Nq|p-`4TEeVJa0Wr}462Gq1RsT!DIDQ31LVrQ{IW@I%Rlhv5%kLftA zsgj;ZB+St9Ni|JjW0{n!!17cE!eXWvBM>8FJrVa2dWw!GPh{0_d<5%xP1Ug@C|WiP zpbjsq8O`AhIfosOu3(&Z#C2NnCWlXvtnPRx3hpe_fgRyuDm|fPG7f_sv5=CDq>Ip6 zHz#wGiXrErjflf%fTgoRR%Js@Dal5lKbM+L*5P#8$uW1#!B~5(|B{x&{b`&%c%XkI zkJYp1&h_gg-9Kr_ir!D7W|B~WoKECt9HBr``8+1({+gp889d= z@3^(U5`M$ppR0s()sAiDiKQc-YWF+#RXevk*%OP;`+{-egEqKXt(PdLHs_4eq)2; zX=Y^y+88djJOdjdJs{$|o2r)7P2USvoxVI{V`H&e;?{p=Bd@>;`mVV6YjfyVHslGf zBv4?p&9{+0@Wi?0Hke2Yh-C^0Jwgpm@Sp#RgWV+>|b97v80|<~x+rq)7!ICc7Y(Hn+Y9#0tkS%wy#L5$$}8S{@^@ih^^( h_b6CF!BV`m?Ph#2UOu%HdeDC05jwcaN0`kn=%4Vv-R1xQ diff --git a/src/utils/__pycache__/upload_rss_feeds.cpython-310.pyc b/src/utils/__pycache__/upload_rss_feeds.cpython-310.pyc deleted file mode 100644 index c2b1e08c905a4ec17e70896d1f2fdec97bab9ec7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1829 zcma)6PmkO*6t^8uCX@NI)7@>$RttoJ5Oe4ZhyzentCkj8M7u?7VIfi#Wn<4Y+e|WR zXS$szgNn8%q>2wvRhk}q1o18S2EKCYSLlKA>~yyUB!r{q*w23N=l6Th&f0AkL8EW| zJ{;B&`a@4%Z4OM%gUH{YV~F7d`PCo$80I7)W6QU&)>#Q1*Zi6ppZE_%d%F5I6(gGSX{4g_EPrCsvh7=EWrD zY5tiInSg06;dgiv5xwh0Yr($>auG!Sgb~Wo0yF#=&q+xZq|_rTw-%Ja_z=42l-6!- zPIH@CjNV5jy@U?%0iM^kQR-2Y*JtN3n%fu^t6=M}T50b#n7xKreGMIY*@mzf?E1lG z2m&ypeY}sDGp~a;D}MVzo{M;An)A(Y%r~E)9#N$KzOrufnb5bzbk8;hx3V!GM11tg$Ahhy@vlU- z%cEQlu1|OhhaE@}4W@aV$idVE5eO**6G4A6Q;l;Yp7OgBv9YSpOQ9q8PNU-Oe{DXO zWKo#NjsB5Ev2JSH8}U5X)EG8A8QtRJ@T{k5W>wX%aVf(Q7n&r6r$Q4{oUTk^Z**JR z4m08LyRkHm;+&7c0sR90im<$gMHh^oY7fP!$}&zR7iM?L_r1!ZI9BD!i?Z<~;W_Vn z#c}U&U3!%bXP`c5Or#ekf`@EYVZ%zVt!g`2o_(ks!DTiT5%+61beff&WFrVe(KsDu z%7r`xRji5vgSpV$sT$YU-55_YIHKwf|yMz#%oWLD?3dFfb{-l&*2ehvlUCZbk+9fU~q=j3ceOK$AyVg1pr!|5T1miGH zgP>S~0MD3eJ7dBrI!6)rcV(6q%U9uIarW|Fo%MRFit0770>1n;laBu3UF=6$$_x`y z=xe4{^m>#D-sg8CKG7V>Di?K~Feon7+zDkIUCh#9Jc6p){QSCVKw7R=`Gmu2(t|)8 z2Y)62@w!HCA09QBg!wH+Gq_O-4;9sKfmqj@Pw4LGHdZ6Uv2zX{j%mU-bj?alih~^B a3fRLP%LP82_b8^ocMGpS?~)Ft&;9~S$mAvf diff --git a/src/utils/__pycache__/upload_rss_feeds.cpython-311.pyc b/src/utils/__pycache__/upload_rss_feeds.cpython-311.pyc deleted file mode 100644 index 0583ed2f084657f5b2afc7b83db7a8213ba2d61e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3473 zcmbsrOKcn0@h!Paa>*q{NtE@mq^x4aD&vH*U^S3chk#lHdLO-VOgT_bjGY5qXS+=sq&iehZ zOJF`jVGcV5+meMyNjC+1*ey819yr;AJ@xWGM`17U(PDE|qqkln+#*oqxewYN85)Ts zg;K6lXb8#lisiO-7OhYenguR}u&dZw(RFkdSlsd@`r1;j-SCIQE!f?tU!Z0niZjUX z{DG(mJ*0H&6khdHy6a*hj#J8nB#9C%nFRh6Ct_98K$wKm3crH_;4P$7!-vpfl?IpU zLc3J&VnUN9p^5KvereWBDqE@?h1_{F+M|CdK}#c z)}t!B!Ad4|=@e2t)os*|r1NXGI*Y35Q&c_tg*zuqERSS^)HDmlVt@*?M-p&G27sqGd1ywZzKPc zZ_c0U+eYLS&sj>f+lqmgyIw92vZV_mKtpZ(sf{YxY?XB~iBzGfPIB$?!I4e1VH zH5E;Y!d#~JdO9ISg?MU#zdAF+&tojeydv^SbS{BM`B=4VA&jK7&%y5?2xtm=&KAm| zn;mOZ7CLP$!34_efPeA;-%{iM5^X`Z+UOLlU<9q(p{rb5O=F#&9GjWc-D8R(#pjk4 z9Ev7!=ppi3bW7l@Ze7BwGHDMy{~|beOHAUy7+yF(GB^_#@JEukh+~R8_+c8S!2Us5 ziVZF+@q|3MZ1@$CWI19u6-ckXU@t7-6uzC7f|*mV>|aQTv1mdL2C70nc&q#lpnt=Y zLu$7G@Fcc-Zg=(XLG@B{Z6)VgXU(pI-rrYS3`~p@K5@aK;yd1lQ zlhILLXN+Cleg(^NbOB4mQJq?rNT@Tfn@2%5wnWa(ixQ7-$7MrNT)|0DKwpMgw+M=# zl1{*(AIODey3CXDl#C@qI)ztw^F*fEym3AzCesP5-~gX#<170zZ=S&du-lCzG9OJy zI4Z1~?}i<}TW97(MLefFBrJ=|QVi?NHIgv8Eg>#|8#*0N&5ODd?k-~bs?!j7MIs$o zXE6DM7>Cm$n5a7-IuT-?Zd*1acQkq}l3>!H{|h?M_F* zStTx}BJf_F!a56X8G2>iQwiM5HBFL2>Lcf9@@ARz3I52CykyVYm?lyH*rfl+!*EwQ zwAbX@ST8J=n*3UmKQ~$Ce0M{4LWR*1cUt34=O+G1JAXg*+bPxi?vpM+03|x8(Lt3C z?$b7=>3Lh1*7io;x!2Kc=GS%@=2Ewy0qzjA4k7Q(yAStVC-#wrX{w<;hvzPLhf~|G zJsJD+2iq&!@cX;0HhdKr042wD&2e3ITrWF3d0XWXff?!+e~}7mO_W>>H~Q0 z>njdFu>Rrv1L4V-+WJm@yv%v;PTiR*&?T-%<9bxC2ds2|LY2L3Tb_HK;`pYg|(fWX^y~bV8#OY6av;$etYs zKjud}e&>5{S$EC_-W#xfJHP<_?MUy%cFT9|&hbv>yS~7SS<`8EV%VoEUTNtVdhMsFqiF;T(+d^?8Iq|#)wqV1&Yp+)Jgxu_hE zT@+LE@dfBjp-Ug?EEM;&*%iQ8Qo50d)D8NKQj$&+&F+e+k#tnKrPCs`bDf&kY4UB7 z-XOv~1d%^&QVT((I$t<0z&C6;frHWoSdz{szYiELiK5ELl{20);&R4QMt-#(%IJ(* z4`t-Z84uhq<;&SM)LIz$>Ma!wl+b{N27oBDUHNv6?JA6v*nW-e&)N3r(^L`?>5v5dD1mi|ITI z#xs4!mVy35gnlHb#ygH8DS3?4^Gf?UTbxzBX8}s|kVX%w^iY}hskCo%w&*O;XEgeZ fT7i8xqPxIsnFM;Ly?Jcdx-(3Vjo5d7K>_|Rv3+7> diff --git a/src/utils/__pycache__/upload_rss_feeds.cpython-312.pyc b/src/utils/__pycache__/upload_rss_feeds.cpython-312.pyc deleted file mode 100644 index 041f12d8bc34fe88265129f269286b9fcd0b020d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3022 zcmbUjZ%iB4{XP56_St817#o8F;RvL`w1H$~Q(y^Fipd}lw+-o5oTirXJ%}CKXWbno zwhZ#7N&^WA(4^t1sz$%1N+z*SBcIx|$;8wz=Rg$qLZwLUhqj-rBvL!_W$&FakaSbE zK6LN>-tYbX|1+Q4?N)#n)%^MRxEX-2$fdGOJHQh^1Yj8mK!9-&ET1q4@!b$NB#c2L zBsgQ7O3*=Chhf|lG#LOg#T*r3dnd+nd$$U7le&SmkOhIx=`7#&@5oxA3EBkuJrHD( zRWMy`13>B*%t5?PQehGpc#(>zN{c_4M;wn@@rvS;{4MX__0%VIF`7wvU zAm{tw14D&Rx0FRrWZ&aV!!&FG$AJu|3=oWBZy=a&yJj2(Q_yGqmIw`e1U2IjO8cN@ z>x)NGQt6i@QNmX`j^0J_$c`Dh0XQkZh`%T8cwYv}PLKeI?WfX70ZvfGv}r@u5Tkd{ z`;P1$g;A*_7{r);Bo$ePEPUQs24syfyW16j%sZtNg-?V$zpv>AVv zEbxQNzXf2L(dD@-oO@@M&N70f9wVagCbKKSD%i5tccIj^_b#x4Ick`;WomnjwSXtXmdqo2TS+n5CU|*)q^B1y~9oRDe76pDHR$SKnJ`k$27Q8z@p!X!E zK?_iJo1hg)znC`dlU4nKIru72SySx46u&?d^92*xpU=xEZog^Mj48cyPQfWqF1Y_& zxAo`hp8u|F!~fLX^^<;A{aoFPaNhT#c3(|Evk2*AI3WsSnS*bp;$m2cCMWm{BP0Ac zLW0aIBCmwU;;4s@>^2pSEortc{5?5|w;|lU%ODFbtW|Et+c)5c9JtpYtVY(m*3WzJ|) zMv_jQL{og3BU4>YFCP&TsW?(l2cL2Cm3Nsh^B@7EJvxibhvO0o3+b|M{Nl4~^q8oK zU7AHgvN$P4kVao5)uEZ<;ska>qoT=iQM2N7h04Ba6ppnbks+(mh)i&uIVEBfH4BbT zh?u9DCR0LK!I;!!8RKI$W;bn`RS#bX7b=oSlr>sMNR0%xr^47-B`PLESg!_=#$dN} zxw7V{1a9X}kvfog!bcI!GK)ce-;&7BuExDs8PZXVw&73y5Jx%(wyInUGjlWf*hZBv zH&A3<3xT;nzGs7dDcAosW&PvO?}t?9u~qLXt8%9|sooN0qN^Ucy${@n3+}_S)~&h* zTr+FI%?4qkPMEcqtf1PN8!WnObAhk8`dO+(QH;CjX;_|Fnz?>@t#wVl&6eeuM4YZ)S6$-`im^LFf=!mr#9I}EXDpdEOPE;$C6{k zzv$THd_}H4|HhJIi78ot{XjlkV4LOwr6Zu)`?MXPSrG zuHL8OKRoRQtmiQ>&{bO&#{xUYs-CXZGygdE_j7Agg%iJ7X9_1Ss9htf`{JhMO-%Bv zL}S`#Pip~p0JAafE!Md(JU6T!IK3ABICd+x?kV)XpF7 zdlR&W_qjBRyh(MWvg-8?>ZnZJj)<5P^x~BzioDm5_lPK=4sc4Fx~>=&7a(0T63ny)Jre{ORqyfv9p None: "LAMBDA_EXECUTION_ROLE_NAME", "LAMBDA_ROLE_ARN", "S3_BUCKET_NAME", - "DYNAMODB_TABLE_NAME", - "SQS_QUEUE_NAME", + "REDIS_URL", + "REDIS_QUEUE_NAME", "LAMBDA_LAYER_VERSION", "LAMBDA_LAYER_NAME", "LAMBDA_LAYER_ARN", "S3_LAYER_BUCKET_NAME", "S3_LAYER_KEY_NAME", - "SQS_QUEUE_URL", - "SQS_QUEUE_ARN", - "DYNAMODB_TABLE_ARN", "PYTHON_VERSION", "LAMBDA_RUNTIME", "LAMBDA_TIMEOUT", diff --git a/template.env b/template.env index 4142e73..dc169e5 100644 --- a/template.env +++ b/template.env @@ -13,8 +13,8 @@ STACK_BASE=${LAMBDA_FUNCTION_NAME} LAMBDA_EXECUTION_ROLE_NAME=rss-feed-processor-role-${AWS_REGION} LAMBDA_ROLE_ARN=arn:aws:iam::${AWS_ACCOUNT_ID}:role/${LAMBDA_EXECUTION_ROLE_NAME} S3_BUCKET_NAME=open-rss-articles-${AWS_REGION} -DYNAMODB_TABLE_NAME=rss-feeds-table -SQS_QUEUE_NAME=rss-feed-queue +REDIS_URL=redis://localhost:6379 +REDIS_QUEUE_NAME=rss-feed-queue LAMBDA_LAYER_VERSION=6 # This is fixed. @@ -25,10 +25,6 @@ S3_LAMBDA_ZIPPED_BUCKET_NAME=open-rss-lambda-${AWS_REGION} S3_LAYER_BUCKET_NAME=rss-feed-processor-layers-${AWS_REGION} S3_LAYER_KEY_NAME= RSSFeedProcessorDependencies - -SQS_QUEUE_URL=https://sqs.${AWS_REGION}.amazonaws.com/${AWS_ACCOUNT_ID}/${SQS_QUEUE_NAME} -SQS_QUEUE_ARN=arn:aws:sqs:${AWS_REGION}:${AWS_ACCOUNT_ID}:${SQS_QUEUE_NAME} -DYNAMODB_TABLE_ARN=arn:aws:dynamodb:${AWS_REGION}:${AWS_ACCOUNT_ID}:table/${DYNAMODB_TABLE_NAME} PYTHON_VERSION=3.12 LAMBDA_RUNTIME=python${PYTHON_VERSION} LAMBDA_TIMEOUT=300 @@ -58,4 +54,4 @@ VECTOR_EMBEDDING_DIM=*** VECTOR_SEARCH_METRIC=*** OPENAI_API_KEY=sk** -OPENAI_EMBEDDING_MODEL=text-embedding-3-large \ No newline at end of file +OPENAI_EMBEDDING_MODEL=text-embedding-3-large diff --git a/tree.md b/tree.md index ff89dad..cca3797 100644 --- a/tree.md +++ b/tree.md @@ -17,7 +17,6 @@ │   │   ├── __pycache__ │   │   │   └── deploy_infrastructure.cpython-312.pyc │   │   ├── cloudformation -│   │   │   ├── dynamo.yaml │   │   │   ├── lambda_role.yaml │   │   │   ├── rss_lambda_stack.yaml │   │   │   ├── s3.yaml