modified: drawFunc.py

	modified:   func.py
	modified:   main.py
	modified:   optMethods.py

drawFunc.py

@@ -16,5 +16,6 @@ def draw(f_x):
 
 # take this pencil
     levels = [0.0, 1, 2, 5, 20, 30, 50, 80, 100, 200, 400]
-    cs = plt.contour(X, Y, Z, levels, colors='k')
-    plt.clabel(cs, fmt='%.1f', inline=1)
+    #cs = plt.contour(X, Y, Z, levels, colors='k')
+    cs = plt.contourf(X, Y, Z, 20, cmap=plt.cm.YlGnBu)
+    #plt.clabel(cs, fmt='%.1f', inline=1)

func.py

@@ -31,7 +31,7 @@ def __init__(self):
         pass
 
     def f_x(self, x):
-        f = 10 * (x[1] - (x[0])**2)**2 + (1-x[0])**2
+        f = 10.0 * (x[1] - (x[0])**2)**2 + (1-x[0])**2
         return f[0, 0]
 
     def g_x(self, x):
@@ -40,7 +40,7 @@ def g_x(self, x):
         return np.matrix([[g0[0,0]], [g1[0,0]]])
 
     def G_x(self, x):
-        G00 = 120.0 * x[0]**2 - 40.0*x[1] + 2
+        G00 = 120.0 * x[0]**2 - 40.0 * x[1] + 2.0
         G01 = -40.0*x[0]
         G10 = -40.0*x[0]
         G11 = 20.0

main.py

@@ -34,17 +34,17 @@ def rosen():
     plt.figure()
 
     drawFunc.draw(obj.f_x)
-    start_point = np.matrix([[-1.5], [0.0]])
-    plt.annotate('Start', xy=(-1.5, 0.0), xytext=(-1.8, 0.5),
+    start_point = np.matrix([[-1.5], [-4.0]])
+    plt.annotate('Start', xy=(-1.5, -4.0), xytext=(-1.8, -3.5),
             arrowprops=dict(facecolor='black', shrink=0.02, frac=0.5))
     plt.annotate('Optimal', xy=(1, 1), xytext=(1.1, 1.4),
             arrowprops=dict(facecolor='black', shrink=0.02, frac=0.5))
 
-    v = 0.0
-    str1 = "Newton"
-    track = opt.newton(start_point, obj, v)
-    p1, = plt.plot(track[0,:].tolist()[0], track[1,:].tolist()[0],
-            'o-', linewidth=2.0)
+    #v = 0.0
+    #str1 = "Newton"
+    #track = opt.newton(start_point, obj, v)
+    #p1, = plt.plot(track[0,:].tolist()[0], track[1,:].tolist()[0],
+    #        'o-', linewidth=2.0)
 
     v = 0.1
     str2 = "Modified_Newton %.2f"%v
@@ -68,18 +68,15 @@ def rosen():
     p5, = plt.plot(track[0,:].tolist()[0], track[1,:].tolist()[0],
             'o-', linewidth=2.0)
 
-    str6 = "BFGS_trivial"
-    track = opt.quasi_newton(start_point, obj, 150, 1)
+    str6 = "BFGS_Armijo"
+    track = opt.BFGS(start_point, obj, 350, 0)
     p6, = plt.plot(track[0,:].tolist()[0], track[1,:].tolist()[0],
             'o-', linewidth=2.0)
 
-    str7 = "BFGS_trivial"
-    track = opt.quasi_newton(start_point, obj, 150, 0)
-    p7, = plt.plot(track[0,:].tolist()[0], track[1,:].tolist()[0],
-            'o-', linewidth=2.0)
-
-    plt.legend([p1, p2, p3, p4, p5, p6, p7],\
-            [str1, str2, str3, str4, str5, str6, str7])
+    #plt.legend([p1, p2, p3, p4, p5, p6, p7],\
+    #        [str1, str2, str3, str4, str5, str6, str7])
+    plt.legend([p2, p3, p4, p5, p6],\
+            [str2, str3, str4, str5, str6])
     plt.show()
 
 #quadratic()

optMethods.py

@@ -1,4 +1,5 @@
-import numpy as np
+from numpy import *
+import pdb
 
 def newton(start_point, obj_fun, modified=0, iterations=5):
 # with second order information
@@ -9,15 +10,15 @@ def newton(start_point, obj_fun, modified=0, iterations=5):
     while k < iterations:
 
         if modified > 0:
-            vI = modified * np.matrix([[1, 0], [0, 1]])
-            G_ = np.linalg.inv(obj_fun.G_x(x) + vI) # inverse?
+            vI = modified * matrix([[1, 0], [0, 1]])
+            G_ = linalg.inv(obj_fun.G_x(x) + vI) # inverse?
         else:
-            G_ = np.linalg.inv(obj_fun.G_x(x)) # inverse?
+            G_ = linalg.inv(obj_fun.G_x(x)) # inverse?
         g_ = obj_fun.g_x(x)
-        delta = -1.0 * np.dot(G_, g_)
+        delta = -1.0 * dot(G_, g_)
         x = x + delta
 
-        track = np.concatenate((track, x), axis=1)
+        track = concatenate((track, x), axis=1)
         k += 1
 
     return track
@@ -28,48 +29,47 @@ def BFGS(start_point, obj_fun, iteration=10, interpolation=0):
     track = x
 
     k = 0
-    H = np.matrix([[.1, 0], [0, .1]])
+    #H = dot(obj_fun.g_x(x), obj_fun.g_x(x).T)
+    H = matrix([[.10, .0], [.0, .10]])
+    gamma = 1.0
 
-    while k < iteration:
+    while k < iteration and linalg.norm(gamma) > 1e-10:
 
-        s = -1.0 * np.dot(H, obj_fun.g_x(x))
+        p = -dot(H, obj_fun.g_x(x))
 
         if interpolation > 0:
-            alpha_k = _backtracking_line_search(obj_fun, x, s)
+            alpha_k = _backtracking_line_search(obj_fun, x, p)
         else:
-            alpha_k = _armijo_line_search(obj_fun, x, s)
+            alpha_k = _armijo_line_search(obj_fun, x, p)
 
-        delta = alpha_k * s
-        x_k_1 = x + delta
+        s = alpha_k * p
+        g_k = obj_fun.g_x(x)
+        x = x + alpha_k * p
+        g_k_1 = obj_fun.g_x(x)
 
-        gamma = obj_fun.g_x(x_k_1) - obj_fun.g_x(x)
-        delta_T_gamma = np.dot(delta.T, gamma)
-        gamma_T_H_gamma = np.dot(np.dot(gamma.T, H), gamma)
-        delta_delta_T = np.dot(delta, delta.T)
-
-        H = H + (1 + gamma_T_H_gamma/delta_T_gamma) *\
-                (delta_delta_T / delta_T_gamma)
+        y = g_k_1 - g_k
 
-        delta_gamma_H = np.dot(np.dot(delta,delta.T), H)
-        H_gamma_delta_T = np.dot(np.dot(H, gamma), delta.T)
-        H = H - (delta_gamma_H + H_gamma_delta_T) / delta_T_gamma
+        z = dot(H, y)
+        sTy = dot(s.T, y)
+        H += outer(s, s) * (sTy + dot(y.T, z))[0,0]/(sTy**2) \
+                - (outer(z, s) + outer(s, z))/sTy
 
-        track = np.concatenate((track, x), axis=1)
-        x = x_k_1
+        track = concatenate((track, x), axis=1)
         k += 1
 
     return track
+
 def quasi_newton(start_point, obj_fun, iteration=10, interpolation=0):
 # with first order information
     x = start_point
     track = x
 
     k = 0
-    H = np.matrix([[.1, 0], [0, .1]])
+    H = matrix([[.01, 0], [0, .01]])
 
     while k < iteration:
 
-        s = -1.0 * np.dot(H, obj_fun.g_x(x))
+        s = -1.0 * dot(H, obj_fun.g_x(x))
 
         if interpolation > 0:
             alpha_k = _backtracking_line_search(obj_fun, x, s)
@@ -80,14 +80,14 @@ def quasi_newton(start_point, obj_fun, iteration=10, interpolation=0):
         x_k_1 = x + delta
 
         gamma = obj_fun.g_x(x_k_1) - obj_fun.g_x(x)
-        u = delta - np.dot(H, gamma)
+        u = delta - dot(H, gamma)
 
-        scale_a = np.dot(u.T, gamma)
+        scale_a = dot(u.T, gamma)
         if scale_a == 0: # :(
             scale_a = 0.000001
-        H = H + np.dot(u, u.T) / scale_a
+        H = H + dot(u, u.T) / scale_a
 
-        track = np.concatenate((track, x), axis=1)
+        track = concatenate((track, x), axis=1)
         x = x_k_1
         k += 1
 
@@ -111,9 +111,8 @@ def _backtracking_line_search(obj_fun, x, s, c=1e-4):
     return alpha
 
 
-def _armijo_line_search(obj_fun, x, s, c=1e-4):
+def _armijo_line_search(obj_fun, x, s, c=1e-4, alpha0=1.0):
 # adapted from scipy.optimisation
-    alpha0 = 1.0
     amin = 0.0
     f_alpha = obj_fun.about_alpha(x, s)
     g_alpha = obj_fun.about_alpha_prime(x, s)
@@ -137,7 +136,7 @@ def _armijo_line_search(obj_fun, x, s, c=1e-4):
             alpha1**3 * (f_alpha(alpha0) - f_alpha(0) - g_alpha(0) * alpha0)
         b = b / factor
 
-        alpha2 = (-b + np.sqrt(abs(b**2 - 3 * a * g_alpha(0)))) / (3.0 * a)
+        alpha2 = (-b + sqrt(abs(b**2 - 3 * a * g_alpha(0)))) / (3.0 * a)
         if(f_alpha(alpha2) <= f_alpha(0) + c * alpha2 * g_alpha(0)):
             return alpha2