aljaz/IngestRSS
1
0
Fork 0
mirror of https://github.com/aljazceru/IngestRSS.git synced 2025-12-18 22:44:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

fixing issue with embedding model initialization.

Browse Source
This commit is contained in:
Charles E. Gormley
2024-10-12 13:43:29 -04:00
parent 891740723c
commit 65fb2b1837
5 changed files with 5 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/infra/cloudformation/eventbridge.yaml
View File

@@ -28,7 +28,7 @@ Resources:
Properties: Properties:
Name: rss-feed-processor-schedule Name: rss-feed-processor-schedule
Description: Runs the RSS Feed Processor Lambda function every hour Description: Runs the RSS Feed Processor Lambda function every hour
ScheduleExpression: rate(5 minutes) ScheduleExpression: rate(30 minutes)
FlexibleTimeWindow: FlexibleTimeWindow:
Mode: FLEXIBLE Mode: FLEXIBLE
MaximumWindowInMinutes: 1 MaximumWindowInMinutes: 1

3
src/infra/lambdas/RSSFeedProcessorLambda/src/analytics/embeddings/vector_db.py
View File

@@ -11,8 +11,7 @@ embedding_dim = os.getenv("VECTOR_EMBEDDING_DIM")
vector_search_metric = os.getenv("VECTOR_SEARCH_METRIC") vector_search_metric = os.getenv("VECTOR_SEARCH_METRIC")
index_name = os.getenv("PINECONE_DB_NAME") index_name = os.getenv("PINECONE_DB_NAME")
client = OpenAI() client = OpenAI() # For Embedding Models, Not LLMs
pc = Pinecone(api_key=api_key) pc = Pinecone(api_key=api_key)
def get_index(): def get_index():

1
src/infra/lambdas/RSSFeedProcessorLambda/src/lambda_function.py
View File

@@ -36,7 +36,6 @@ def lambda_handler(event, context):
# Process the feed # Process the feed
result = extract_feed(feed) result = extract_feed(feed)
print(type(result))
logger.info("Process Feed Result Dictionary: ", result) logger.info("Process Feed Result Dictionary: ", result)
last_pub_dt = result['max_date'] last_pub_dt = result['max_date']

3
src/infra/lambdas/lambda_utils/update_lambda_env_vars.py
View File

@@ -20,7 +20,8 @@ def update_env_vars(function_name):
'VECTOR_EMBEDDING_MODEL': os.environ.get('VECTOR_EMBEDDING_MODEL'), 'VECTOR_EMBEDDING_MODEL': os.environ.get('VECTOR_EMBEDDING_MODEL'),
'VECTOR_EMBEDDING_DIM': os.environ.get('VECTOR_EMBEDDING_DIM'), 'VECTOR_EMBEDDING_DIM': os.environ.get('VECTOR_EMBEDDING_DIM'),
'VECTOR_SEARCH_METRIC': os.environ.get('VECTOR_SEARCH_METRIC'), 'VECTOR_SEARCH_METRIC': os.environ.get('VECTOR_SEARCH_METRIC'),
'PINECONE_DB_NAME': os.environ.get('PINECONE_DB_NAME') 'PINECONE_DB_NAME': os.environ.get('PINECONE_DB_NAME'),
'OPENAI_API_KEY': os.environ.get('OPENAI_API_KEY')
} }
return lambda_client.update_function_configuration( return lambda_client.update_function_configuration(

2
todo.md
View File

@@ -1,5 +1,5 @@
# Testing🧪 # Testing🧪
* Testing from 3rd party aws account. [ Today ] * Testing from 3rd party aws account.
* Test Large Amounts of Feeds ( Decrease the cadence of ingesting. ) [ Today ] * Test Large Amounts of Feeds ( Decrease the cadence of ingesting. ) [ Today ]
* Test out Vector Databases at Small Scale * Test out Vector Databases at Small Scale
* Test out Vector Databases at Scale. * Test out Vector Databases at Scale.