IngestRSS/monthly_ingestion.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RSS Article Batch Download Examples\n",
    "\n",
    "This notebook demonstrates how to batch download RSS articles from S3."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from time import time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Root imports\n",
    "\n",
    "from src.search.batch import S3BatchDownloader\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv(override=True)\n",
    "\n",
    "downloader = S3BatchDownloader()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize the Downloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "List permission: Allowed\n"
     ]
    }
   ],
   "source": [
    "import boto3\n",
    "from botocore.exceptions import ClientError\n",
    "\n",
    "s3 = boto3.client('s3')\n",
    "\n",
    "# Test ListObjects\n",
    "try:\n",
    "    response = s3.list_objects_v2(Bucket=os.getenv(\"S3_BUCKET_NAME\"))\n",
    "    print(\"List permission: Allowed\")\n",
    "except ClientError as e:\n",
    "    print(\"List permission: Denied\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 59592 objects to process\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 31707/59592 [04:33<6:08:33,  1.26object/s]IOStream.flush timed out\n",
      "100%|██████████| 59592/59592 [08:38<00:00, 114.83object/s]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloaded data to November-1.csv in 605.81 seconds\n"
     ]
    }
   ],
   "source": [
    "start = time()\n",
    "output_path = \"November-1.csv\"  # or \"consolidated_data.json\"\n",
    "\n",
    "# Define date range\n",
    "start_date = \"2024-11-17\" # FIXME: Fix the error where data can't be collected before the date you started collecting.\n",
    "end_date = \"2024-11-22\"\n",
    "\n",
    "# Start downloading\n",
    "downloader.download_to_file(\n",
    "    output_path=output_path,\n",
    "    file_format=\"csv\",  # or \"json\"\n",
    "    start_date=start_date,\n",
    "    end_date=end_date\n",
    ")\n",
    "\n",
    "print(f\"Downloaded data to {output_path} in {time() - start:.2f} seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 50037 objects to process\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████▌   | 33067/50037 [04:07<11:49, 23.92object/s]  IOStream.flush timed out\n",
      "100%|██████████| 50037/50037 [07:11<00:00, 116.06object/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloaded data to November-2.csv in 539.74 seconds\n"
     ]
    }
   ],
   "source": [
    "start = time()\n",
    "output_path = \"November-2.csv\"  # or \"consolidated_data.json\"\n",
    "\n",
    "# Define date range\n",
    "start_date = \"2024-11-23\" # FIXME: Fix the error where data can't be collected before the date you started collecting.\n",
    "end_date = \"2024-11-27\"\n",
    "\n",
    "# Start downloading\n",
    "downloader.download_to_file(\n",
    "    output_path=output_path,\n",
    "    file_format=\"csv\",  # or \"json\"\n",
    "    start_date=start_date,\n",
    "    end_date=end_date\n",
    ")\n",
    "\n",
    "print(f\"Downloaded data to {output_path} in {time() - start:.2f} seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = time()\n",
    "output_path = \"November-3.csv\"  # or \"consolidated_data.json\"\n",
    "\n",
    "# Define date range\n",
    "start_date = \"2024-11-8\" # FIXME: Fix the error where data can't be collected before the date you started collecting.\n",
    "end_date = \"2024-11-30\"\n",
    "\n",
    "# Start downloading\n",
    "downloader.download_to_file(\n",
    "    output_path=output_path,\n",
    "    file_format=\"csv\",  # or \"json\"\n",
    "    start_date=start_date,\n",
    "    end_date=end_date\n",
    ")\n",
    "\n",
    "print(f\"Downloaded data to {output_path} in {time() - start:.2f} seconds\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Aggregating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}