diff --git a/AUTHORS.rst b/AUTHORS.rst index 59662ec..1a481a0 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -11,3 +11,4 @@ Contributors ------------ * Dmitry Budaev +* Bharat Kalluri diff --git a/HISTORY.rst b/HISTORY.rst index 2b617f5..47eb4de 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,12 @@ History ======= +0.4.0 (2017-10-22) +------------------ + +* Make tests run in 1/10th the time with ``vcr.py`` (@condemil #32) +* Add support for returning python objects instead of dictionaries (@BharatKalluri #33) + 0.3.0 (2017-09-17) ------------------ diff --git a/README.rst b/README.rst index 3d84db2..6fb886e 100644 --- a/README.rst +++ b/README.rst @@ -97,11 +97,17 @@ Features * Easy programmatic access to Tiingo API * Reuse requests session across API calls for better performance -* Coming soon: - * Client-side validation of tickers - * Data validation of returned responses - * Case insensitivity for ticker names +* On most methods, pass in `fmt="object"` as a keyword to have your responses come back as `NamedTuples`, which should have a lower memory impact than regular Python dictionaries. +Roadmap: +-------- + +* Client-side validation of tickers +* Data validation of returned responses +* Case insensitivity for ticker names +* More documentation / code examples + +Feel free to file a PR that implements any of the above items. Credits --------- diff --git a/requirements_dev.txt b/requirements_dev.txt index 002d144..80e7a1e 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -6,7 +6,7 @@ flake8==3.4.1 tox==2.9.1 coverage==4.4.1 Sphinx==1.6.4 -cryptography==2.0.3 +cryptography==2.1.1 PyYAML==3.12 pytest==3.2.3 pytest-runner==2.12.1 diff --git a/tests/test_tiingo.py b/tests/test_tiingo.py index 4a35af2..26977e5 100644 --- a/tests/test_tiingo.py +++ b/tests/test_tiingo.py @@ -1,11 +1,8 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- """Tests for `tiingo` package.""" import csv - from unittest import TestCase - import vcr from tiingo import TiingoClient @@ -14,13 +11,11 @@ from tiingo.restclient import RestClientError # TODO # Add tests for -# Invalid API key -# Invalid ticker -# Use unittest asserts rather than regular asserts +# - Invalid API key +# - Invalid ticker +# Use unittest asserts rather than regular asserts if applicable # Wrap server errors with client side descriptive errors # Coerce startDate/endDate to string if they are passed in as datetime -# Use VCR.py to enable offline testing -# Expand test coverage def test_client_repr(): @@ -35,47 +30,57 @@ class TestTickerPrices(TestCase): def setUp(self): self._client = TiingoClient() - # Stub all endpoints that get reused - with vcr.use_cassette('tests/fixtures/ticker_price.yaml'): - self._ticker_price_response = \ - self._client.get_ticker_price("GOOGL") - with vcr.use_cassette('tests/fixtures/ticker_metadata.yaml'): - self._ticker_metadata_response = \ - self._client.get_ticker_metadata("GOOGL") + @vcr.use_cassette('tests/fixtures/ticker_metadata.yaml') + def test_ticker_metadata(self): + """Refactor this with python data schemavalidation""" + metadata = self._client.get_ticker_metadata("GOOGL") + + assert metadata.get('ticker') == "GOOGL" + assert metadata.get("name") + + @vcr.use_cassette('tests/fixtures/ticker_metadata.yaml') + def test_ticker_metadata_as_object(self): + metadata = self._client.get_ticker_metadata("GOOGL", fmt="object") + assert metadata.ticker == "GOOGL" # Access property via ATTRIBUTE + assert metadata.name # (contrast with key access above + + @vcr.use_cassette('tests/fixtures/ticker_price.yaml') def test_ticker_price(self): - """Test the EOD Prices Endpoint""" - assert len(self._ticker_price_response) == 1 - assert self._ticker_price_response[0].get('adjClose') + """Test that EOD Prices Endpoint works""" + prices = self._client.get_ticker_price("GOOGL") + assert len(prices) == 1 + assert prices[0].get('adjClose') + @vcr.use_cassette('tests/fixtures/ticker_price.yaml') + def test_ticker_price_as_object(self): + """Test that EOD Prices Endpoint works""" + prices = self._client.get_ticker_price("GOOGL", fmt="object") + assert len(prices) == 1 + assert hasattr(prices[0], 'adjClose') + + @vcr.use_cassette('tests/fixtures/ticker_price_with_date.yaml') def test_ticker_price_with_date(self): """Test the EOD Prices Endpoint with data param""" - with vcr.use_cassette('tests/fixtures/ticker_price_with_date.yaml'): - prices = self._client.get_ticker_price("GOOGL", - startDate="2015-01-01", - endDate="2015-01-05") + prices = self._client.get_ticker_price("GOOGL", + startDate="2015-01-01", + endDate="2015-01-05") self.assertGreater(len(prices), 1) + @vcr.use_cassette('tests/fixtures/ticker_price_with_date_csv.yaml') def test_ticker_price_with_csv(self): """Confirm that CSV endpoint works""" - with vcr.use_cassette('tests/fixtures/ticker_price_with_date_csv.yaml'): - prices_csv = self._client.get_ticker_price("GOOGL", - startDate="2015-01-01", - endDate="2015-01-05", - fmt='csv') + prices_csv = self._client.get_ticker_price("GOOGL", + startDate="2015-01-01", + endDate="2015-01-05", + fmt='csv') reader = csv.reader(prices_csv.splitlines(), delimiter=",") rows = list(reader) assert len(rows) > 2 # more than 1 day of data - def test_ticker_metadata(self): - """Refactor this with python data schemavalidation""" - assert self._ticker_metadata_response.get('ticker') == "GOOGL" - assert self._ticker_metadata_response.get("name") - + @vcr.use_cassette('tests/fixtures/list_stock_tickers.yaml') def test_list_stock_tickers(self): - """Update this test when the method is added.""" - with vcr.use_cassette('tests/fixtures/list_stock_tickers.yaml'): - tickers = self._client.list_stock_tickers() + tickers = self._client.list_stock_tickers() assert len(tickers) > 1 assert all(ticker['assetType'] == 'Stock' for ticker in tickers) @@ -96,38 +101,55 @@ class TestNews(TestCase): 'crawlDate', 'id' ] - - def test_get_news_articles(self): - """Confirm that news article work""" - NUM_ARTICLES = 1 - - search_params = { + # Search for articles about a topic + self.num_articles = 1 + self.search_params = { "tickers": ["aapl", "googl"], "tags": ["Technology", "Bitcoin"], "startDate": "2016-01-01", "endDate": "2017-08-31", "sources": ['washingtonpost.com', 'altcointoday.com'], - "limit": NUM_ARTICLES + "limit": self.num_articles } - with vcr.use_cassette('tests/fixtures/news.yaml'): - articles = self._client.get_news(**search_params) - assert len(articles) == NUM_ARTICLES + @vcr.use_cassette('tests/fixtures/news.yaml') + def test_get_news_articles(self): + articles = self._client.get_news(**self.search_params) + assert len(articles) == self.num_articles for article in articles: assert all(key in article for key in self.article_keys) + @vcr.use_cassette('tests/fixtures/news_bulk.yaml') def test_get_news_bulk(self): """Fails because this API key lacks institutional license""" - - with self.assertRaises(RestClientError),\ - vcr.use_cassette('tests/fixtures/news_bulk.yaml'): + with self.assertRaises(RestClientError): value = self._client.get_bulk_news(file_id="1") assert value + @vcr.use_cassette('tests/fixtures/news_bulk_file_ids.yaml') def test_get_news_bulk_ids(self): """Fails because this API key lacks institutional license""" - - with self.assertRaises(RestClientError),\ - vcr.use_cassette('tests/fixtures/news_bulk_file_ids.yaml'): + with self.assertRaises(RestClientError): value = self._client.get_bulk_news() assert value + + # Tests "object" formatting option + @vcr.use_cassette('tests/fixtures/news.yaml') + def test_get_news_as_objects(self): + articles = self._client.get_news(fmt="object", **self.search_params) + assert len(articles) == self.num_articles + for article in articles: # check if attribute access works + assert all(hasattr(article, key) for key in self.article_keys) + + @vcr.use_cassette('tests/fixtures/news_bulk_file_ids.yaml') + def test_get_news_bulk_ids_as_objects(self): + """Fails because this API key lacks institutional license""" + with self.assertRaises(RestClientError): + value = self._client.get_bulk_news(fmt="object") + assert value + + @vcr.use_cassette('tests/fixtures/news_bulk.yaml') + def test_news_bulk_as_objects(self): + """Fails because this API key lacks institutional license""" + with self.assertRaises(RestClientError): + assert self._client.get_bulk_news(file_id="1", fmt="object") diff --git a/tiingo/__version__.py b/tiingo/__version__.py index 32aa532..0301595 100644 --- a/tiingo/__version__.py +++ b/tiingo/__version__.py @@ -1,2 +1,2 @@ # -*- coding: utf-8 -*- -__version__ = '0.3.2' +__version__ = '0.4.0' diff --git a/tiingo/api.py b/tiingo/api.py index d404bd5..2a63e0c 100644 --- a/tiingo/api.py +++ b/tiingo/api.py @@ -4,6 +4,8 @@ import os import sys import pkg_resources import csv +import json +from collections import namedtuple from zipfile import ZipFile @@ -34,6 +36,15 @@ def get_buffer_from_zipfile(zipfile, filename): return TextIOWrapper(BytesIO(zipfile.read(filename))) +def dict_to_object(item, object_name): + """Converts a python dict to a namedtuple, saving memory.""" + fields = item.keys() + values = item.values() + return json.loads(json.dumps(item), + object_hook=lambda d: + namedtuple(object_name, fields)(*values)) + + class TiingoClient(RestClient): """Class for managing interactions with the Tiingo REST API @@ -79,7 +90,7 @@ class TiingoClient(RestClient): return [row for row in reader if row.get('assetType') == 'Stock'] - def get_ticker_metadata(self, ticker): + def get_ticker_metadata(self, ticker, fmt='json'): """Return metadata for 1 ticker Use TiingoClient.list_tickers() to get available options @@ -88,7 +99,11 @@ class TiingoClient(RestClient): """ url = "tiingo/daily/{}".format(ticker) response = self._request('GET', url) - return response.json() + data = response.json() + if fmt == 'json': + return data + elif fmt == 'object': + return dict_to_object(data, "Ticker") def get_ticker_price(self, ticker, startDate=None, endDate=None, @@ -108,7 +123,7 @@ class TiingoClient(RestClient): """ url = "tiingo/daily/{}/prices".format(ticker) params = { - 'format': fmt, + 'format': fmt if fmt != "object" else 'json', # conversion local 'frequency': frequency } @@ -122,13 +137,17 @@ class TiingoClient(RestClient): response = self._request('GET', url, params=params) if fmt == "json": return response.json() + elif fmt == "object": + data = response.json() + return [dict_to_object(item, "TickerPrice") for item in data] else: return response.content.decode("utf-8") # NEWS FEEDS # tiingo/news def get_news(self, tickers=[], tags=[], sources=[], startDate=None, - endDate=None, limit=100, offset=0, sortBy="publishedDate"): + endDate=None, limit=100, offset=0, sortBy="publishedDate", + fmt='json'): """Return list of news articles matching given search terms https://api.tiingo.com/docs/tiingo/news @@ -155,9 +174,13 @@ class TiingoClient(RestClient): 'endDate': endDate } response = self._request('GET', url, params=params) - return response.json() + data = response.json() + if fmt == 'json': + return data + elif fmt == 'object': + return [dict_to_object(item, "NewsArticle") for item in data] - def get_bulk_news(self, file_id=None): + def get_bulk_news(self, file_id=None, fmt='json'): """Only available to institutional clients. If ID is NOT provided, return array of available file_ids. If ID is provided, provides URL which you can use to download your @@ -169,4 +192,8 @@ class TiingoClient(RestClient): url = "tiingo/news/bulk_download" response = self._request('GET', url) - return response.json() + data = response.json() + if fmt == 'json': + return data + elif fmt == 'object': + return dict_to_object(data, "BulkNews")