fixed yahoo->google finance

Author: mdipierro

docs/book_numerical.pdf

@@ -1767,8 +1767,8 @@ \section*{Acknowledgments}
     but Yahoo changed API and blocked them, moving to Google finance.
     """
     URL_CURRENT = 'http://finance.yahoo.com/d/quotes.csv?s=%(symbol)s&f=%(columns)s'
-    URL_HISTORICAL = 'https://www.google.com/finance/historical?output=csv&q=%s'
-'
+    URL_HISTORICAL = 'https://www.google.com/finance/historical?output=csv&q=%(symbol)s'
+
     def __init__(self,symbol):
         self.symbol = symbol.upper()
 
@@ -1805,14 +1805,9 @@ \section*{Acknowledgments}
                 current[row[0]] = raw_data[i]
         return current
 
-    def historical(self,start=None, stop=None):
+    def historical(self, start=None, stop=None):
         import datetime, time, urllib, math
-        start =  start or datetime.date(1900,1,1)
-        stop = stop or datetime.date.today()
-        url = self.URL_HISTORICAL % dict(
-            s=self.symbol,
-            a=start.month-1,b=start.day,c=start.year,
-            d=stop.month-1,e=stop.day,f=stop.year)
+        url = self.URL_HISTORICAL % dict(symbol=self.symbol)
         # Date,Open,High,Low,Close,Volume,Adj Close
         lines = urllib.urlopen(url).readlines()
         if any('CAPTCHA' in line for line in lines):
@@ -1824,6 +1819,8 @@ \section*{Acknowledgments}
         raw_data.reverse()
         for row in raw_data:
             if row[1] == '-': continue
+            date = datetime.datetime.strptime(row[0],'%d-%b-%y')
+            if (start and date<start) or (stop and date>stop): continue            
             open, high, low = float(row[1]), float(row[2]), float(row[3])
             close, vol = float(row[4]), float(row[5])
             adjusted_close = float(row[5]) if len(row)>5 else close
@@ -1836,7 +1833,7 @@ \section*{Acknowledgments}
                 arithmetic_return = log_return = None
             previous_adjusted_close = adjusted_close
             series.append(dict(
-               date = datetime.datetime.strptime(row[0],'%Y-%m-%d'),
+               date = date,
                open = open,
                high = high,
                low = low,
@@ -1853,7 +1850,7 @@ \section*{Acknowledgments}
 
     @staticmethod
     def download(symbol='goog',what='adjusted_close',start=None,stop=None):
-        return [d[what] for d in YStock(symbol).historical(start,stop)]
+        return [d[what] for d in YStock(symbol).historical(start, stop)]
 \end{lstlisting}
 
 Many web services return data in JSON format. JSON is slowly replacing XML as a favorite protocol for data transfer on the web. It is lighter, simpler to use, and more human readable. JSON can be thought of as serialized JavaScript. the JSON data can be converted to a Python object using a library called {\ft json}:

docs/book_numerical.tex

@@ -13,9 +13,12 @@ class YStock:
     >>> h = google.historical()
     >>> last_adjusted_close = h[-1]['adjusted_close']
     >>> last_log_return = h[-1]['log_return']
+    previous version of this code user Yahoo for historical data
+    but Yahoo changed API and blocked them, moving to Google finance.
     """
     URL_CURRENT = 'http://finance.yahoo.com/d/quotes.csv?s=%(symbol)s&f=%(columns)s'
-    URL_HISTORICAL = 'http://ichart.yahoo.com/table.csv?s=%(s)s&a=%(a)s&b=%(b)s&c=%(c)s&d=%(d)s&e=%(e)s&f=%(f)s'
+    URL_HISTORICAL = 'https://www.google.com/finance/historical?output=csv&q=%(symbol)s'
+
     def __init__(self,symbol):
         self.symbol = symbol.upper()
 
@@ -52,24 +55,25 @@ def current(self):
                 current[row[0]] = raw_data[i]
         return current
 
-    def historical(self,start=None, stop=None):
+    def historical(self, start=None, stop=None):
         import datetime, time, urllib, math
-        start =  start or datetime.date(1900,1,1)
-        stop = stop or datetime.date.today()
-        url = self.URL_HISTORICAL % dict(
-            s=self.symbol,
-            a=start.month-1,b=start.day,c=start.year,
-            d=stop.month-1,e=stop.day,f=stop.year)
+        url = self.URL_HISTORICAL % dict(symbol=self.symbol)
         # Date,Open,High,Low,Close,Volume,Adj Close
         lines = urllib.urlopen(url).readlines()
-        raw_data = [row.split(',') for row in lines[1:] if row.count(',')==6]
+        if any('CAPTCHA' in line for line in lines):
+            print url
+            raise
+        raw_data = [row.split(',') for row in lines[1:] if 5 <= row.count(',') <= 6]
         previous_adjusted_close = 0
         series = []
         raw_data.reverse()
         for row in raw_data:
+            if row[1] == '-': continue
+            date = datetime.datetime.strptime(row[0],'%d-%b-%y')
+            if (start and date<start) or (stop and date>stop): continue
             open, high, low = float(row[1]), float(row[2]), float(row[3])
             close, vol = float(row[4]), float(row[5])
-            adjusted_close = float(row[6])
+            adjusted_close = float(row[5]) if len(row)>5 else close
             adjustment = adjusted_close/close
             if previous_adjusted_close:
                 arithmetic_return = adjusted_close/previous_adjusted_close-1.0
@@ -79,7 +83,7 @@ def historical(self,start=None, stop=None):
                 arithmetic_return = log_return = None
             previous_adjusted_close = adjusted_close
             series.append(dict(
-               date = datetime.datetime.strptime(row[0],'%Y-%m-%d'),
+               date = date,
                open = open,
                high = high,
                low = low,
@@ -96,7 +100,7 @@ def historical(self,start=None, stop=None):
 
     @staticmethod
     def download(symbol='goog',what='adjusted_close',start=None,stop=None):
-        return [d[what] for d in YStock(symbol).historical(start,stop)]
+        return [d[what] for d in YStock(symbol).historical(start, stop)]
 
 import os
 import uuid
@@ -1093,7 +1097,7 @@ def eval_fitting_function(f,c,x):
         weight = 1.0/points[i][2] if len(points[i])>2 else 1.0
         b[i,0] = weight*float(points[i][1])
         for j in xrange(A.ncols):
-            A[i,j] = weight*f[j](points[i][0])
+            A[i,j] = weight*f[j](float(points[i][0]))
     c = (1.0/(A.T*A))*(A.T*b)
     chi = A*c-b
     chi2 = norm(chi,2)**2
@@ -1447,7 +1451,7 @@ def optimize_newton_multi(f, x, ap=1e-6, rp=1e-4, ns=20):
         if k>2 and norm(x-x_old)<max(ap,norm(x)*rp): return x.flatten()
     raise ArithmeticError('no convergence')
 
-def optimize_newton_multi_imporved(f, x, ap=1e-6, rp=1e-4, ns=20, h=10.0):
+def optimize_newton_multi_improved(f, x, ap=1e-6, rp=1e-4, ns=20, h=10.0):
     """
     Finds the extreme of multidimensional function f near point x.
 
@@ -1482,7 +1486,7 @@ def g(b, data=data, f=fs, constraint=constraint):
             if constraint: chi2+=constraint(b)
             return chi2
         if isinstance(b,(list,tuple)):
-            b = optimize_newton_multi_imporved(g,b,ap,rp,ns)
+            b = optimize_newton_multi_improved(g,b,ap,rp,ns)
         else:
             b = optimize_newton(g,b,ap,rp,ns)
         return b, g(b,data,constraint=None)
@@ -1503,7 +1507,7 @@ def g(b,data=data,fs=fs,constraint=constraint):
             if constraint:
                 chi += constraint(b)
             return chi2
-        b = optimize_newton_multi_imporved(g,b,ap,rp,ns)
+        b = optimize_newton_multi_improved(g,b,ap,rp,ns)
         a, chi2 = core(b,data,fs)
         return a+b,chi2
 
@@ -1572,7 +1576,7 @@ def next(self):
     def random(self):
         return float(self.next())/self.m
 
-class MarsenneTwister(object):
+class MersenneTwister(object):
     """
     based on:
     Knuth 1981, The Art of Computer Programming
@@ -2590,4 +2594,4 @@ def test132():
 if __name__=='__main__':
     import os,doctest
     if not os.path.exists('images'): os.mkdir('images')
-    doctest.testmod(optionflags=doctest.ELLIPSIS)
+    doctest.testmod(optionflags=doctest.ELLIPSIS)