```python
import numpy as np
# import sympy as sp
import matplotlib.pyplot as plt
# sp.init_printing()
```

# Homework 5
## Linear Regression Error

### Question 1


```python
N = sp.symbols('N')
s, d = 0.1, 8
```


```python
sp.solve(s**2*(1 - (d+1)/N) - 0.008, N)
```

## Gradient Descent
### Question 4


```python
u, v = sp.symbols('u v')
E = (u*sp.exp(v)-2*v*sp.exp(-u))**2
E.diff(u)
```


```python
E.diff(v)
```

### Question 5


```python
def E(x):
    u = x[0]
    v = x[1]
    return (u*np.exp(v) - 2*v*np.exp(-u))**2
```


```python
def grad_E(x):
    grad = np.zeros(2)
    u = x[0]
    v = x[1]
    grad[0] = 2*(u*np.exp(v) - 2*v*np.exp(-u))*(np.exp(v) + 2*v*np.exp(-u))
    grad[1] = 2*(u*np.exp(v) - 2*v*np.exp(-u))*(u*np.exp(v) - 2*np.exp(-u))
    return grad
```


```python
def grad_descent(E, grad_E, init=np.ones(2), eta=0.1, tol=1e-6, max_iter=10**4):
    iters=0
    u = init
    error = E(u)
    while error > tol and iters < max_iter:
        u = u - eta*grad_E(u)
        error = E(u)
        iters = iters + 1
        print(u)
    return u, iters, error
```


```python
def coord_descent(E, grad_E, init=np.ones(2), eta=0.1, tol=1e-6, max_iter=10**5):
    iters=0
    u = init
    error = E(u)
    while error > tol and iters < max_iter:
        u[0] = u[0] - eta*grad_E(u)[0]
        u[1] = u[1] - eta*grad_E(u)[1]
        error = E(u)
        iters = iters+1
        print(u)
    return u, iters, error
```


```python
u_g, iters_g, error_g = grad_descent(E, grad_E, tol=1e-14)
print(' ')
u_c, iters_c, error_c = coord_descent(E, grad_E, init=np.ones(2), tol=1e-30, max_iter=15)
print(u_g, u_c)
print(iters_g, iters_c)
print(error_g, error_c)
```


```python
X, Y = np.meshgrid(np.linspace(0,2), np.linspace(0,2))
def error(x, y):
    return (x*np.exp(y) - 2*y*np.exp(-x))**2

Z = error(X, Y)

```

## Logistic Regression


```python
def generate_target(low=-1, high=1):
    # Generate the target function f
    div_points = np.random.uniform(low=low, high=high, size=(2, 2))
    w1 = (div_points[1, 1] - div_points[0, 1])/(div_points[1, 0] - div_points[0, 0])
    w0 = div_points[0, 1] - div_points[0, 0]*w1
    w = np.array([w0, w1, -1])
    return w
```


```python
def generate_sample(num_points, w, low=-1, high=1):
    # Generate the test points and classify
    test_points = np.ones([num_points, 4])
    test_points[:, 1:3] = np.random.uniform(low=low, high=high, size=(num_points, 2))
    test_points[:, 3] = np.sign(np.dot(test_points[:, 0:3], w))
    return test_points
```

### Question 8


```python
def log_regression(training, g_init=np.zeros(3), tol=1e-6, max_iter=10**4, eta=0.1):
    g = g_init
    iters = 0
    n_points, n_cols = training.shape
    y = training[:, -1]
    X = training[:, 0:n_cols-1]
    error = tol+1

    while error > tol and iters < max_iter:
        old = g
        for i in np.random.permutation(n_points):
            xn = X[i,:]
            yn = y[i]
            g = g + eta*(yn*xn)/(1 + np.exp(yn*np.dot(xn, g)))
        iters += 1
        error = np.linalg.norm(g - old)

    return g, iters
```


```python
NTRIALS=10
NPOINTS=100
MCPOINTS = 10**6
mcdata = np.ones([MCPOINTS, 3])
mcdata[:, 1:3] = np.random.random([MCPOINTS, 2])
e_out = np.zeros(NTRIALS)
g = np.zeros([NTRIALS, 3])
iters = np.zeros(NTRIALS)
for i in range(0, NTRIALS):
    w = generate_target()
    training = generate_sample(NPOINTS, w)
    g, iters[i] = log_regression(training, tol=0.01)
    g = -g/g[2]
    e_out[i] = np.mean(np.log(1+np.exp(-np.sign(np.dot(mcdata,w))*np.dot(mcdata,g))))
#     print(w, g, iters)

print(np.mean(e_out))
print(np.mean(iters))
```

    0.337346593227
    1149.4



```python
w0, w1 = w[0], w[1]
g0, g1 = g[0], g[1]

# Split the test points into the two classification groups
positives = training[training[:,-1]>0,:]
negatives = training[training[:,-1]<0,:]

# Plot f, g, and \mathbf{x}
x = np.linspace(0,1)
xpos = positives[:,1]
ypos = positives[:,2]
xneg = negatives[:,1]
yneg = negatives[:,2]

plt.plot(xneg,yneg,'.g')
plt.plot(xpos,ypos,'.r')
plt.plot(x,w1*x+w0)
plt.plot(x,g1*x+g0,'--k')
plt.axis([0,1,0,1])
#plt.legend(['Negative','Positive'])
plt.show()
```


```python
def perceptron(training, g_init=np.zeros(3), max_iter=10**5, pocket=True):
    # Implement PLA, count the iterations to convergence, and the error
    g = g_init
    iters = 0
    n_points, n_cols = training.shape
    
    test_points = np.zeros([n_points, n_cols+2])
    test_points[:, 0:n_cols] = training
    test_points[:, -2] = np.sign(np.dot(test_points[:, 0:3], g))
    test_points[:, -1] = np.log(1+np.exp(-test_points[:, ncols-1]*))
    wrong = test_points[test_points[:, -1] < 1, :]
    nrows = wrong.shape[0]
    
    if pocket:
        minrows = nrows
        best = g
    
    while nrows > 0 and iters < max_iter:
        if nrows == 1:
            g = g + wrong[0, 0:3]*wrong[0, 3]
        else:
            rind = np.random.randint(0, nrows)
            g = g + wrong[rind, 0:3]*wrong[rind, 3]
        
        iters += 1
        test_points[:, -2] = np.sign(np.dot(test_points[:, 0:3], g))
        test_points[:, -1] = test_points[:, -3] == test_points[:, -2]
        wrong = test_points[test_points[:, -1] < 1, :]
        nrows = wrong.shape[0]
        if pocket and nrows < minrows:
            minrows = nrows
            best = g
    
    if pocket:
        g = best
        
    return g, iters
```


```python
npoints = 10**2
w = generate_target()
training = generate_sample(npoints, w)
inds = np.random.choice(npoints, npoints//10)
training[inds, -1] = -training[inds, -1]
g, iters = perceptron(training)
g = -g/g[2]
print(g)
print(w)
print(iters)
```


```python
w0, w1 = w[0], w[1]
g0, g1 = g[0], g[1]

# Split the test points into the two classification groups
positives = training[training[:,-1]>0,:]
negatives = training[training[:,-1]<0,:]

# Plot f, g, and \mathbf{x}
x = np.linspace(0,1)
xpos = positives[:,1]
ypos = positives[:,2]
xneg = negatives[:,1]
yneg = negatives[:,2]

plt.plot(xneg,yneg,'.g')
plt.plot(xpos,ypos,'.r')
plt.plot(x,w1*x+w0)
plt.plot(x,g1*x+g0,'--k')
plt.axis([0,1,0,1])
#plt.legend(['Negative','Positive'])
plt.show()
```


```python

```

9ad04f10b595156db2e061cdb239c200e01d58ec
ipynb
Jupyter Notebook
codes/HW5.ipynb
wandrewjam/machine-learning
530c559abd631d509eb847880927cdcb92ab8243
MIT
codes/HW5.ipynb
wandrewjam/machine-learning
530c559abd631d509eb847880927cdcb92ab8243
MIT
codes/HW5.ipynb
wandrewjam/machine-learning
530c559abd631d509eb847880927cdcb92ab8243
MIT
Qwen/Qwen-72B
1. YES
2. YES