Merge pull request #112 from dcwtx/issue#85

Issue#85: return pandas formatted data
This commit is contained in:
Cameron Yick
2018-06-14 01:07:48 -04:00
committed by GitHub
11 changed files with 339 additions and 12 deletions

View File

@@ -12,6 +12,7 @@ deploy:
install:
- python setup.py develop
- pip install -U pytest-cov codecov vcrpy
- tools/install_pandas.sh
# - pip install -U tox-travis pytest
language: python
@@ -22,6 +23,10 @@ python:
- 2.7
# - 2.6
env:
- WITH_PANDAS=false
- WITH_PANDAS=true
script:
- export TIINGO_API_KEY=0000000000000000000000000000000000000000
- py.test --cov=./tiingo

View File

@@ -25,7 +25,7 @@ Tiingo Python
Tiingo is a financial data platform that makes high quality financial tools available to all. Tiingo has a REST and Real-Time Data API, which this library helps you to access. Presently, the API includes support for the following endpoints:
* Stock Market Ticker Closing Prices + Metadata. Data includes full distribution details and is validated using a proprietary EOD Price Engine.
* Curated news from top financial news sources + blogs. Stories are tagged with topic tags and relevant stock tickers by Tiingo's algorithms.
* Curated news from top financial news sources + blogs. Stories are tagged with topic tags and relevant stock tickers by Tiingo's algorithms.
Usage
@@ -37,6 +37,12 @@ First, install the library from PyPi:
pip install tiingo
If you prefer to receive your results in ``pandas DataFrame`` or ``Series`` format, and you do not already have pandas installed, install it as an optional dependency:
.. code-block:: shell
pip install tiingo[pandas]
Next, initialize your client. It is recommended to use an environment
variable to initialize your client for convenience.
@@ -67,7 +73,7 @@ Alternately, you may use a dictionary to customize/authorize your client.
Now you can use ``TiingoClient`` to make your API calls. (Other parameters are available for each endpoint beyond what is used in the below examples, inspect the docstring for each function for details.).
.. code-block:: python
# Get Ticker
ticker_metadata = client.get_ticker_metadata("GOOGL")
@@ -86,13 +92,39 @@ Now you can use ``TiingoClient`` to make your API calls. (Other parameters are a
tickers = client.list_stock_tickers()
# Get news articles about given tickers or search terms from given domains
articles = client.get_news(tickers=['GOOGL', 'APPL'],
tags=['Laptops'],
articles = client.get_news(tickers=['GOOGL', 'AAPL'],
tags=['Laptops'],
sources=['washingtonpost.com'],
startDate='2017-01-01',
endDate='2017-08-31')
To receive results in ``pandas`` format, use the ``get_dataframe()`` method:
.. code-block:: python
#Get a pd.DataFrame of the price history of a single symbol (default is daily):
ticker_history = client.get_dataframe("GOOGL")
#The method returns all of the available information on a symbol, such as open, high, low, close,
#adjusted close, etc. This page in the tiingo api documentation lists the available information on each
#symbol: https://api.tiingo.com/docs/tiingo/daily#priceData.
#Frequencies and start and end dates can be specified similarly to the json method above.
#Get a pd.Series of only one column of the available response data by specifying one of the valid the
#'metric_name' parameters:
ticker_history = client.get_dataframe("GOOGL", metric_name='adjClose')
#Get a pd.DataFrame for a list of symbols for a specified metric_name (default is adjClose if no
#metric_name is specified):
ticker_history = client.get_dataframe(['GOOGL', 'AAPL'],
frequency='weekly',
metric_name='volume',
startDate='2017-01-01',
endDate='2018-05-31')
Further Docs
--------
@@ -110,7 +142,7 @@ Roadmap:
--------
* Client-side validation of tickers
* Data validation of returned responses
* Data validation of returned responses
* Case insensitivity for ticker names
* More documentation / code examples

View File

@@ -57,6 +57,7 @@ setup(
packages=find_packages(include=[NAME]),
include_package_data=True,
install_requires=requirements,
extras_require={'pandas': ['pandas>=0.18']},
license="MIT license",
zip_safe=False,
keywords=['tiingo', 'finance', 'stocks'],

View File

@@ -0,0 +1,24 @@
interactions:
- request:
body: null
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Authorization: [Token 0000000000000000000000000000000000000000]
Connection: [keep-alive]
Content-Type: [application/json]
User-Agent: [tiingo-python-client 0.6.0]
method: GET
uri: https://api.tiingo.com/tiingo/daily/GOOGL/prices?format=json&resampleFreq=daily&startDate=2018-01-05&endDate=2018-01-19
response:
body: {string: '[{"date":"2018-01-05T00:00:00.000Z","close":1110.29,"high":1113.58,"low":1101.8,"open":1103.45,"volume":1493389,"adjClose":1110.29,"adjHigh":1113.58,"adjLow":1101.8,"adjOpen":1103.45,"adjVolume":1493389,"divCash":0.0,"splitFactor":1.0},{"date":"2018-01-08T00:00:00.000Z","close":1114.21,"high":1119.16,"low":1110.0,"open":1111.0,"volume":1148958,"adjClose":1114.21,"adjHigh":1119.16,"adjLow":1110.0,"adjOpen":1111.0,"adjVolume":1148958,"divCash":0.0,"splitFactor":1.0},{"date":"2018-01-09T00:00:00.000Z","close":1112.79,"high":1118.44,"low":1108.2,"open":1118.44,"volume":1335995,"adjClose":1112.79,"adjHigh":1118.44,"adjLow":1108.2,"adjOpen":1118.44,"adjVolume":1335995,"divCash":0.0,"splitFactor":1.0},{"date":"2018-01-10T00:00:00.000Z","close":1110.14,"high":1112.78,"low":1103.98,"open":1107.0,"volume":1027781,"adjClose":1110.14,"adjHigh":1112.78,"adjLow":1103.98,"adjOpen":1107.0,"adjVolume":1027781,"divCash":0.0,"splitFactor":1.0},{"date":"2018-01-11T00:00:00.000Z","close":1112.05,"high":1114.85,"low":1106.48,"open":1112.31,"volume":1102461,"adjClose":1112.05,"adjHigh":1114.85,"adjLow":1106.48,"adjOpen":1112.31,"adjVolume":1102461,"divCash":0.0,"splitFactor":1.0},{"date":"2018-01-12T00:00:00.000Z","close":1130.65,"high":1131.3,"low":1108.01,"open":1110.1,"volume":1914460,"adjClose":1130.65,"adjHigh":1131.3,"adjLow":1108.01,"adjOpen":1110.1,"adjVolume":1914460,"divCash":0.0,"splitFactor":1.0},{"date":"2018-01-16T00:00:00.000Z","close":1130.7,"high":1148.88,"low":1126.66,"open":1140.31,"volume":1783881,"adjClose":1130.7,"adjHigh":1148.88,"adjLow":1126.66,"adjOpen":1140.31,"adjVolume":1783881,"divCash":0.0,"splitFactor":1.0},{"date":"2018-01-17T00:00:00.000Z","close":1139.1,"high":1139.32,"low":1123.49,"open":1136.36,"volume":1353097,"adjClose":1139.1,"adjHigh":1139.32,"adjLow":1123.49,"adjOpen":1136.36,"adjVolume":1353097,"divCash":0.0,"splitFactor":1.0},{"date":"2018-01-18T00:00:00.000Z","close":1135.97,"high":1140.59,"low":1124.46,"open":1139.35,"volume":1333633,"adjClose":1135.97,"adjHigh":1140.59,"adjLow":1124.46,"adjOpen":1139.35,"adjVolume":1333633,"divCash":0.0,"splitFactor":1.0},{"date":"2018-01-19T00:00:00.000Z","close":1143.5,"high":1143.78,"low":1132.5,"open":1138.03,"volume":1418376,"adjClose":1143.5,"adjHigh":1143.78,"adjLow":1132.5,"adjOpen":1138.03,"adjVolume":1418376,"divCash":0.0,"splitFactor":1.0}]'}
headers:
Allow: ['GET, HEAD, OPTIONS']
Content-Length: ['2349']
Content-Type: [application/json]
Date: ['Sun, 10 Jun 2018 18:24:41 GMT']
Server: [nginx/1.10.1]
Vary: ['Accept, Cookie']
X-Frame-Options: [SAMEORIGIN]
status: {code: 200, message: OK}
version: 1

View File

@@ -0,0 +1,24 @@
interactions:
- request:
body: null
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Authorization: [Token 0000000000000000000000000000000000000000]
Connection: [keep-alive]
Content-Type: [application/json]
User-Agent: [tiingo-python-client 0.6.0]
method: GET
uri: https://api.tiingo.com/tiingo/daily/GOOGL/prices?format=json&resampleFreq=daily
response:
body: {string: '[{"adjClose":1132.71,"adjHigh":1138.78,"adjLow":1123.23,"adjOpen":1131.21,"adjVolume":1364226,"close":1132.71,"date":"2018-06-08T00:00:00+00:00","divCash":0.0,"high":1138.78,"low":1123.23,"open":1131.21,"splitFactor":1.0,"volume":1364226}]'}
headers:
Allow: ['GET, HEAD, OPTIONS']
Content-Length: ['239']
Content-Type: [application/json]
Date: ['Sun, 10 Jun 2018 18:21:17 GMT']
Server: [nginx/1.10.1]
Vary: ['Accept, Cookie']
X-Frame-Options: [SAMEORIGIN]
status: {code: 200, message: OK}
version: 1

View File

@@ -0,0 +1,24 @@
interactions:
- request:
body: null
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Authorization: [Token 0000000000000000000000000000000000000000]
Connection: [keep-alive]
Content-Type: [application/json]
User-Agent: [tiingo-python-client 0.6.0]
method: GET
uri: https://api.tiingo.com/tiingo/daily/GOOGL/prices?format=json&resampleFreq=weekly&startDate=2018-01-05&endDate=2018-01-19
response:
body: {string: '[{"date":"2018-01-05T00:00:00.000Z","close":1110.29,"high":1113.58,"low":1053.02,"open":1053.02,"volume":5889084,"adjClose":1110.29,"adjHigh":1113.58,"adjLow":1053.02,"adjOpen":1053.02,"adjVolume":5889084,"divCash":0.0,"splitFactor":1.0},{"date":"2018-01-12T00:00:00.000Z","close":1130.65,"high":1131.3,"low":1103.98,"open":1111.0,"volume":6529655,"adjClose":1130.65,"adjHigh":1131.3,"adjLow":1103.98,"adjOpen":1111.0,"adjVolume":6529655,"divCash":0.0,"splitFactor":1.0},{"date":"2018-01-19T00:00:00.000Z","close":1135.97,"high":1148.88,"low":1123.49,"open":1140.31,"volume":4470611,"adjClose":1135.97,"adjHigh":1148.88,"adjLow":1123.49,"adjOpen":1140.31,"adjVolume":4470611,"divCash":0.0,"splitFactor":1.0}]'}
headers:
Allow: ['GET, HEAD, OPTIONS']
Content-Length: ['708']
Content-Type: [application/json]
Date: ['Sun, 10 Jun 2018 18:25:46 GMT']
Server: [nginx/1.10.1]
Vary: ['Accept, Cookie']
X-Frame-Options: [SAMEORIGIN]
status: {code: 200, message: OK}
version: 1

View File

@@ -0,0 +1,46 @@
interactions:
- request:
body: null
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Authorization: [Token 0000000000000000000000000000000000000000]
Connection: [keep-alive]
Content-Type: [application/json]
User-Agent: [tiingo-python-client 0.6.0]
method: GET
uri: https://api.tiingo.com/tiingo/daily/GOOGL/prices?format=json&resampleFreq=weekly&startDate=2018-01-05&endDate=2018-01-19
response:
body: {string: '[{"date":"2018-01-05T00:00:00.000Z","close":1110.29,"high":1113.58,"low":1053.02,"open":1053.02,"volume":5889084,"adjClose":1110.29,"adjHigh":1113.58,"adjLow":1053.02,"adjOpen":1053.02,"adjVolume":5889084,"divCash":0.0,"splitFactor":1.0},{"date":"2018-01-12T00:00:00.000Z","close":1130.65,"high":1131.3,"low":1103.98,"open":1111.0,"volume":6529655,"adjClose":1130.65,"adjHigh":1131.3,"adjLow":1103.98,"adjOpen":1111.0,"adjVolume":6529655,"divCash":0.0,"splitFactor":1.0},{"date":"2018-01-19T00:00:00.000Z","close":1135.97,"high":1148.88,"low":1123.49,"open":1140.31,"volume":4470611,"adjClose":1135.97,"adjHigh":1148.88,"adjLow":1123.49,"adjOpen":1140.31,"adjVolume":4470611,"divCash":0.0,"splitFactor":1.0}]'}
headers:
Allow: ['GET, HEAD, OPTIONS']
Content-Length: ['708']
Content-Type: [application/json]
Date: ['Sun, 10 Jun 2018 18:33:00 GMT']
Server: [nginx/1.10.1]
Vary: ['Accept, Cookie']
X-Frame-Options: [SAMEORIGIN]
status: {code: 200, message: OK}
- request:
body: null
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Authorization: [Token 0000000000000000000000000000000000000000]
Connection: [keep-alive]
Content-Type: [application/json]
User-Agent: [tiingo-python-client 0.6.0]
method: GET
uri: https://api.tiingo.com/tiingo/daily/AAPL/prices?format=json&resampleFreq=weekly&startDate=2018-01-05&endDate=2018-01-19
response:
body: {string: '[{"date":"2018-01-05T00:00:00.000Z","close":175.0,"high":175.37,"low":169.26,"open":170.16,"volume":99095223,"adjClose":173.6258731716,"adjHigh":173.9929678748,"adjLow":167.9309445315,"adjOpen":168.8238775936,"adjVolume":99095223,"divCash":0.0,"splitFactor":1.0},{"date":"2018-01-12T00:00:00.000Z","close":177.09,"high":177.36,"low":173.0,"open":174.35,"volume":107548622,"adjClose":175.6994621711,"adjHigh":175.9673420898,"adjLow":171.6415774782,"adjOpen":172.9809770712,"adjVolume":107548622,"divCash":0.0,"splitFactor":1.0},{"date":"2018-01-19T00:00:00.000Z","close":179.26,"high":180.1,"low":175.07,"open":177.9,"volume":92146251,"adjClose":177.8524229985,"adjHigh":178.6858271897,"adjLow":173.6953235208,"adjOpen":176.503101927,"adjVolume":92146251,"divCash":0.0,"splitFactor":1.0}]'}
headers:
Allow: ['GET, HEAD, OPTIONS']
Content-Length: ['787']
Content-Type: [application/json]
Date: ['Sun, 10 Jun 2018 18:33:01 GMT']
Server: [nginx/1.10.1]
Vary: ['Accept, Cookie']
X-Frame-Options: [SAMEORIGIN]
status: {code: 200, message: OK}
version: 1

View File

@@ -0,0 +1,82 @@
#!/usr/bin/env python
"""Unit tests for pandas functionality in tiingo"""
import vcr
from unittest import TestCase
from tiingo import TiingoClient
from tiingo.api import APIColumnNameError, InstallPandasException
try:
import pandas as pd
pandas_is_installed = True
except ImportError:
pandas_is_installed = False
class TestTiingoWithPython(TestCase):
def setUp(self):
if pandas_is_installed:
self._client = TiingoClient()
else:
self.skipTest("test_tiingo_pandas: Pandas not installed.")
@vcr.use_cassette('tests/fixtures/ticker_price_pandas_weekly.yaml')
def test_return_pandas_format(self):
"""Test that valid pandas format is returned when specified"""
prices = self._client.get_dataframe("GOOGL", startDate='2018-01-05',
endDate='2018-01-19', frequency='weekly')
self.assertTrue(isinstance(prices, pd.DataFrame))
assert len(prices.index) == 3
@vcr.use_cassette('tests/fixtures/ticker_price_pandas_weekly_multiple_tickers.yaml')
def test_return_pandas_format_multiple(self):
"""Test that valid pandas format is returned when specified"""
tickers = ["GOOGL", "AAPL"]
prices = self._client.get_dataframe(tickers, startDate='2018-01-05',
endDate='2018-01-19', metric_name='adjClose', frequency='weekly')
self.assertTrue(isinstance(prices, pd.DataFrame))
assert len(prices.columns) == 2
assert len(prices.index) == 3
@vcr.use_cassette('tests/fixtures/ticker_price_pandas_daily.yaml')
def test_return_pandas_daily(self):
"""Test that valid pandas format is returned when specified"""
prices = self._client.get_dataframe("GOOGL", startDate='2018-01-05',
endDate='2018-01-19', frequency='daily')
self.assertTrue(isinstance(prices, pd.DataFrame))
assert len(prices.columns) == 12
@vcr.use_cassette('tests/fixtures/ticker_price_pandas_daily_metric_name.yaml')
def test_return_pandas_daily(self):
"""Test that one column is returned when a metric name is specified"""
prices = self._client.get_dataframe("GOOGL", startDate='2018-01-05', metric_name='adjClose',
endDate='2018-01-19', frequency='daily')
self.assertTrue(isinstance(prices, pd.Series))
assert len(prices.index) == 10
def test_metric_name_column_error(self):
with self.assertRaises(APIColumnNameError):
self._client.get_dataframe(['GOOGL', 'AAPL'], startDate='2018-01-05',
endDate='2018-01-19', metric_name='xopen', frequency='weekly')
@vcr.use_cassette('tests/fixtures/ticker_price_pandas_single.yaml')
def test_pandas_edge_case(self):
"""Test single price/date being returned as a frame"""
prices = self._client.get_dataframe("GOOGL")
assert len(prices) == 1
assert len(prices.index) == 1
class TestTiingoWithoutPython(TestCase):
def setUp(self):
if pandas_is_installed:
self.skipTest("test_tiingo_without_pandas: Pandas is installed.")
else:
self._client = TiingoClient()
@vcr.use_cassette('tests/fixtures/ticker_price_pandas_single.yaml')
def test_get_dataframe_without_pandas(self):
with self.assertRaises(InstallPandasException):
self._client.get_dataframe("GOOGL")

View File

@@ -7,10 +7,13 @@ import csv
import json
from collections import namedtuple
from zipfile import ZipFile
from tiingo.restclient import RestClient
import requests
try:
import pandas as pd
pandas_is_installed = True
except ImportError:
pandas_is_installed = False
VERSION = pkg_resources.get_distribution("tiingo").version
@@ -44,6 +47,13 @@ def dict_to_object(item, object_name):
object_hook=lambda d:
namedtuple(object_name, fields)(*values))
class InstallPandasException(Exception):
pass
class APIColumnNameError(Exception):
pass
class TiingoClient(RestClient):
"""Class for managing interactions with the Tiingo REST API
@@ -60,6 +70,7 @@ class TiingoClient(RestClient):
api_key = self._config['api_key']
except KeyError:
api_key = os.environ.get('TIINGO_API_KEY')
self._api_key = api_key
if not(api_key):
raise RuntimeError("Tiingo API Key not provided. Please provide"
@@ -155,6 +166,76 @@ class TiingoClient(RestClient):
else:
return response.content.decode("utf-8")
def get_dataframe(self, tickers,
startDate=None, endDate=None, metric_name=None, frequency='daily'):
""" Return a pandas.DataFrame of historical prices for one or more ticker symbols.
By default, return latest EOD Composite Price for a list of stock tickers.
On average, each feed contains 3 data sources.
Supported tickers + Available Day Ranges are here:
https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip
or from the TiingoClient.list_tickers() method.
Args:
tickers (string/list): One or more unique identifiers for a stock ticker.
startDate (string): Start of ticker range in YYYY-MM-DD format.
endDate (string): End of ticker range in YYYY-MM-DD format.
metric_name (string): Optional parameter specifying metric to be returned for each
ticker. In the event of a single ticker, this is optional and if not specified
all of the available data will be returned. In the event of a list of tickers,
this parameter is required.
frequency (string): Resample frequency (defaults to daily).
"""
valid_columns = ['open', 'high', 'low', 'close', 'volume', 'adjOpen', 'adjHigh', 'adjLow',
'adjClose', 'adjVolume', 'divCash', 'splitFactor']
if metric_name is not None and metric_name not in valid_columns:
raise APIColumnNameError('Valid data items are: ' + str(valid_columns))
params = {
'format': 'json',
'resampleFreq': frequency
}
if startDate:
params['startDate'] = startDate
if endDate:
params['endDate'] = endDate
if pandas_is_installed:
if type(tickers) is str:
stock = tickers
url = "tiingo/daily/{}/prices".format(stock)
response = self._request('GET', url, params=params)
df = pd.DataFrame(response.json())
if metric_name is not None:
prices = df[metric_name]
prices.index = df['date']
else:
prices = df
prices.index = df['date']
del (prices['date'])
else:
prices = pd.DataFrame()
for stock in tickers:
url = "tiingo/daily/{}/prices".format(stock)
response = self._request('GET', url, params=params)
df = pd.DataFrame(response.json())
df.index = df['date']
df.rename(index=str, columns={metric_name: stock}, inplace=True)
prices = pd.concat([prices, df[stock]], axis=1)
prices.index = pd.to_datetime(prices.index)
return prices
else:
error_message = ("Pandas is not installed, but .get_ticker_price() was "
"called with fmt=pandas. In order to install tiingo with "
"pandas, reinstall with pandas as an optional dependency. \n"
"Install tiingo with pandas dependency: \'pip install tiingo[pandas]\'\n"
"Alternatively, just install pandas: pip install pandas.")
raise InstallPandasException(error_message)
# NEWS FEEDS
# tiingo/news
def get_news(self, tickers=[], tags=[], sources=[], startDate=None,

View File

@@ -6,19 +6,21 @@ import re
import argparse
fixtures_directory = 'tests/fixtures/'
# restclient api header configuration
zero_api_regex = r'(\[Token )0{40}(\])'
real_api_regex = r'(\[Token ).{40}(\])'
zero_token_string = '[Token ' + 40 * '0' + ']'
def has_api_key(file):
def has_api_key(file_name):
"""
Detect whether the file contains an api key in the Token object that is not 40*'0'.
See issue #86.
:param file: path-to-file to check
:return: boolean
"""
f = open(file, 'r')
f = open(file_name, 'r')
text = f.read()
if re.search(real_api_regex, text) is not None and \
re.search(zero_api_regex, text) is None:
@@ -26,15 +28,15 @@ def has_api_key(file):
return False
def remove_api_key(file):
def remove_api_key(file_name):
"""
Change the api key in the Token object to 40*'0'. See issue #86.
:param file: path-to-file to change
"""
with open(file, 'r') as fp:
with open(file_name, 'r') as fp:
text = fp.read()
text = re.sub(real_api_regex, zero_token_string, text)
with open(file, 'w') as fp:
with open(file_name, 'w') as fp:
fp.write(text)
return

6
tools/install_pandas.sh Executable file
View File

@@ -0,0 +1,6 @@
#!/bin/bash
if $WITH_PANDAS
then
pip install pandas
echo "pandas installed"
fi