Numbers(pr6)

import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt
from sklearn.metrics import log_loss, accuracy_score, classification_report, mean_squared_error
from imblearn.over_sampling import SMOTE
from mpl_toolkits.mplot3d import Axes3D
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
import plotly.express as px

class MultiNeuralNetwork:
    def __init__(self, type='reg', learning_rate=0.01, epochs=100000, hidden_layers=[10], C=1, early_stopping_rounds=np.inf):
        self.type = type # 'reg', 'binary', 'multi'
        self.learning_rate = learning_rate
        self.epochs = epochs
        self.hidden_layers = hidden_layers
        self.C = C
        self.early_stopping_rounds = early_stopping_rounds
        self.weights = []
        self.biases = []
        self.loss_ = []
        self.epochs_ = []

    def sigmoid(self, z):
        return 1 / (1 + np.exp(-z))

    def sigmoid_derivative(self, z):
        s = self.sigmoid(z)
        return s * (1 - s)

    def linear(self, z, C=1):
        return z * C

    def softmax(self, z):
        exp_values = np.exp(z - np.max(z))
        return exp_values / np.sum(exp_values, axis=0)

    def fit(self, x, y):
        def initialize_parameters(input_size, output_size):
            layers = [input_size] + self.hidden_layers + [output_size]
            self.weights = [np.random.randn(layers[i], layers[i + 1]) * np.sqrt(1 / layers[i]) for i in range(len(layers) - 1)]
            self.biases = [np.random.randn(layers[i + 1]) * 0.01 for i in range(len(layers) - 1)]

        best_err, count = np.inf, 0

        try:
            input_size = x.shape[1]
        except:
            x = x.reshape(-1, 1)
            input_size = x.shape[1]

        output_size = len(np.unique(y)) if self.type == 'multi' else 1
        initialize_parameters(input_size=input_size, output_size=output_size)
        for epoch in range(self.epochs):
            y_pred = []
            for i, x_sample in enumerate(x):
                inputs, activations, outputs = [x_sample], [], []
                input = x_sample

                len_ = len(self.weights)
                d_weights, d_biases = [None] * len_ , [None] * len_

                for idx, _ in enumerate(zip(self.weights, self.biases)):
                    w, b = _[0], _[1]

                    activation = np.dot(input, w) + b
                    activations.append(activation)

                    if w.shape[1] == 1 and self.type == 'reg':
                        output = self.linear(activation, C=self.C)
                        outputs.append(output)
                    elif w.shape[1] == 1 and self.type == 'binary':
                        output = self.sigmoid(activation)
                        outputs.append(output)
                    elif idx == (len_-1) and self.type == 'multi':
                        output = self.softmax(activation)
                        outputs.append(output)
                    else:
                        input = self.sigmoid(activation)
                        outputs.append(input)
                        inputs.append(input)

                y_pred.append(output)

                if self.type == 'multi':
                    delta = output - np.eye(output_size)[y[i]]
                else:
                    delta = output - y[i]

                d_biases[len_-1] = delta * self.C
                d_weights[len_-1] = np.dot(inputs[len_-1].reshape(-1, 1), d_biases[len_-1].reshape(-1, 1).T)

                for idx in reversed(range(len_ - 1)):
                    d_biases[idx] = np.dot(d_biases[idx+1], self.weights[idx+1].T * self.sigmoid_derivative(activations[idx]))
                    d_weights[idx] = np.dot(inputs[idx].reshape(-1, 1), d_biases[idx].reshape(-1, 1).T)

                for idx in (range(len_)):
                    self.weights[idx] -= self.learning_rate * d_weights[idx]
                    self.biases[idx] -= self.learning_rate * d_biases[idx]

            if self.type == 'reg':
                err = mean_squared_error(y_true=y, y_pred=y_pred)
                print(f'Epoche={epoch}, MSE={err}')
            elif self.type == 'binary' or self.type == 'multi':
                err = log_loss(y, y_pred)
                print(f'Epoche={epoch}, logloss={err}')
            if err < best_err:
                count = 0
                best_err = err
            else:
                count += 1
            self.epochs_.append(epoch+1)
            self.loss_.append(err)

            if count >= self.early_stopping_rounds:
                print(f"Навчання завершено на епохі {epoch} з помилкою {err}")
                break

    def predict(self, x):

        try:
            _ = x.shape[1]
        except:
            x = x.reshape(-1, 1)

        y_pred = []
        len_ = len(self.weights)
        for x_sample in x:
            input = x_sample
            for idx, _ in enumerate(zip(self.weights, self.biases)):
                w, b = _[0], _[1]
                activation = np.dot(input, w) + b

                if w.shape[1] == 1 and self.type == 'reg':
                    output = self.linear(activation, C=self.C)
                elif w.shape[1] == 1 and self.type == 'binary':
                    output = (self.sigmoid(activation) > 0.5).astype(int)
                elif idx == (len_-1) and self.type == 'multi':
                    output = np.argmax(self.softmax(activation))
                else:
                    input = self.sigmoid(activation)
            y_pred.append(output)
        return np.array(y_pred)



mnist = tf.keras.datasets.mnist
(X_train, y_train), (X_test, y_test) = mnist.load_data()

plt.figure(figsize=(10,5))
for i in range(5):
    plt.subplot(1, 5, i + 1)
    plt.imshow(X_train[i], cmap='gray_r')
    plt.title(f"Label: {y_train[i]}")
    plt.axis("off")
plt.show()

x_train = X_train.reshape(X_train.shape[0], -1) / 255.0
x_test = X_test.reshape(X_test.shape[0], -1) / 255.0

df = pd.DataFrame(x_train)
df['target'] = y_train


n = 3
x_pca = PCA(n_components=n).fit_transform(StandardScaler().fit_transform(x_train))

df_pca = pd.DataFrame(x_pca, columns=[f'PCA{i+1}' for i in range(n)], index=df.index)
df_pca['target'] = y_train

fig = px.scatter_3d(
    df_pca,
    x='PCA1',
    y='PCA2',
    z='PCA3',
    color='target',
    labels={'target': 'Клас'},
    title="3D візуалізація PCA з класами"
)

num = 10
fig.update_layout(
    scene=dict(
        xaxis=dict(range=[df_pca['PCA1'].min() - num, df_pca['PCA1'].max() + num], title='PCA1'),
        yaxis=dict(range=[df_pca['PCA2'].min() - num, df_pca['PCA2'].max() + num], title='PCA2'),
        zaxis=dict(range=[df_pca['PCA3'].min() - num, df_pca['PCA3'].max() + num], title='PCA3')
    )
)

fig.update_traces(marker=dict(size=5))
fig.show()



def plot_values_counts(df):
    plt.figure(figsize=(15, 4))
    df['target'].value_counts().plot(kind='barh', color='skyblue')
    for index, value in enumerate(df['target'].value_counts()):
        plt.text(value, index, f'{value}', va='center')
    plt.grid()
    plt.title('Кількість кожного з класів в навчальній вибірці')
    plt.ylabel('Клас')
    plt.xlabel('Кількість')
    plt.show()

plot_values_counts(df)


sampler = SMOTE(random_state=42)
x_train_resampled, y_train_resampled = sampler.fit_resample(x_train, y_train)
df_sampled = pd.DataFrame(x_train_resampled)
df_sampled['target'] = y_train_resampled

plot_values_counts(df_sampled)


model = MultiNeuralNetwork(type='multi', learning_rate=0.01, epochs=1000, hidden_layers=[20], early_stopping_rounds=10)
model.fit(x_train_resampled, y_train_resampled)

y_pred_train = model.predict(x_train)
y_pred = model.predict(x_test)

plt.figure(figsize=(10,5))
for i in range(5):
    plt.subplot(1, 5, i + 1)
    plt.imshow(X_test[i], cmap='gray_r')
    plt.title(f"Real: {y_test[i]} | Pred: {y_pred[i]}")
    plt.axis("off")
plt.show()

print(f"Точність на тренувальних даних: {accuracy_score(y_train, y_pred_train) * 100:.2f}%")
print(f"Точність на тестових даних: {accuracy_score(y_test, y_pred) * 100:.2f}%")

print(classification_report(y_test, y_pred))


model_wthout_res = MultiNeuralNetwork(type='multi', learning_rate=0.01, epochs=1000, hidden_layers=[20], early_stopping_rounds=10)
model_wthout_res.fit(x_train, y_train)

y_pred_train = model_wthout_res.predict(x_train)
y_pred = model_wthout_res.predict(x_test)

plt.figure(figsize=(10,5))
for i in range(5):
    plt.subplot(1, 5, i + 1)
    plt.imshow(X_test[i+5], cmap='gray_r')
    plt.title(f"Real: {y_test[i+5]} | Pred: {y_pred[i+5]}")
    plt.axis("off")
plt.show()

print(f"Точність на тренувальних даних: {accuracy_score(y_train, y_pred_train) * 100:.2f}%")
print(f"Точність на тестових даних: {accuracy_score(y_test, y_pred) * 100:.2f}%")

print(classification_report(y_test, y_pred))


plt.figure(figsize=(15, 5))
plt.title('Залежність помилки від епохи навчання')
plt.plot(model.epochs_, model.loss_, c='blue', label=f'З оверсемплінгом (lr={model.learning_rate})')
plt.plot(model_wthout_res.epochs_, model_wthout_res.loss_, c='red', label=f'Без оверсемплінгу (lr={model_wthout_res.learning_rate})')
plt.ylabel('logloss')
plt.xlabel('Epoche')
plt.grid()
plt.legend()
plt.show()