optuna relu tanh

import numpy as np
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, Input, Activation
from tensorflow.keras.layers import LeakyReLU
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import confusion_matrix
from sklearn.utils.class_weight import compute_class_weight
import matplotlib.pyplot as plt
import seaborn as sns
import optuna
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression

# Trénovací data (2D)
# Trénovací data (2D)
train_data = np.array([
    [1141, 1050, 1], [1499, 1050, 0], [1451, 1077, 1], [1519, 1077, 1], [1191, 1093, 0],
    [1590, 1093, 1], [1777, 1093, 0], [1141, 1124, 1], [1499, 1124, 0], [1601, 1124, 0],
    [1606, 1141, 0], [1608, 1141, 0], [1870, 1141, 0], [1794, 1191, 0], [1329, 1211, 1],
    [1687, 1211, 0], [1918, 1211, 0], [1608, 1212, 0], [1050, 1228, 0], [2107, 1228, 0],
    [2202, 1266, 0], [1551, 1327, 0], [1608, 1327, 1], [1660, 1327, 0], [1329, 1332, 1],
    [1093, 1346, 0], [1546, 1419, 1], [1327, 1435, 0], [1774, 1435, 0], [1794, 1451, 0],
    [1077, 1458, 1], [1093, 1458, 1], [1731, 1458, 0], [1777, 1491, 0], [1212, 1499, 1],
    [1211, 1519, 1], [1608, 1519, 1], [1918, 1519, 0], [1458, 1538, 1], [1918, 1538, 0],
    [1794, 1545, 0], [1903, 1545, 0], [1435, 1546, 1], [1758, 1546, 0], [2076, 1546, 0],
    [1077, 1551, 1], [1690, 1551, 0], [1050, 1590, 1], [1093, 1601, 1], [1327, 1601, 0],
    [1050, 1606, 1], [1491, 1606, 1], [1519, 1608, 0], [1266, 1660, 1], [1839, 1660, 0],
    [1332, 1687, 0], [1519, 1687, 0], [1538, 1690, 1], [1870, 1690, 0], [1903, 1731, 1],
    [1918, 1731, 0], [1419, 1758, 0], [1839, 1758, 0], [1077, 1774, 1], [1519, 1774, 1],
    [2202, 1774, 0], [1538, 1777, 0], [1903, 1777, 1], [2107, 1777, 0], [1660, 1794, 0],
    [2107, 1794, 0], [1124, 1839, 1], [1519, 1839, 1], [1546, 1839, 1], [1870, 1839, 1],
    [2202, 1839, 0], [1419, 1870, 1], [2107, 1870, 0], [2202, 1870, 0], [1191, 1903, 1],
    [1601, 1903, 1], [1606, 1903, 1], [1660, 1903, 1], [1491, 1918, 1], [1212, 2076, 1],
    [1690, 2076, 1], [1546, 2107, 1], [1903, 2107, 1], [2183, 2107, 0], [1229, 2183, 1],
    [1731, 2183, 1], [1758, 2183, 0], [1918, 2183, 1], [2076, 2183, 0], [1538, 2202, 1],
    [1601, 2202, 1], [2076, 2202, 0], [1660, 2258, 1], [1777, 2258, 0], [2202, 2258, 0],
    [1477, 1141, 0], [1519, 1141, 0], [1519, 1211, 1], [1549, 1211, 0], [1731, 1211, 0],
    [1848, 1211, 0], [1458, 1229, 1], [1528, 1229, 1], [1354, 1327, 0], [1448, 1327, 0],
    [1458, 1327, 1], [1499, 1327, 0], [1758, 1327, 0], [1405, 1332, 0], [1519, 1332, 0],
    [1549, 1332, 0], [1605, 1332, 0], [1606, 1332, 0], [1606, 1332, 1], [1794, 1332, 1],
    [1376, 1346, 0], [1608, 1346, 0], [1551, 1354, 1], [1731, 1354, 0], [1849, 1354, 0],
    [1918, 1354, 0], [1332, 1376, 0], [1690, 1376, 0], [1606, 1405, 1], [1608, 1405, 0],
    [1687, 1405, 0], [1774, 1448, 0], [1870, 1448, 0], [1477, 1450, 0], [1551, 1450, 0],
    [1731, 1450, 1], [2076, 1450, 0], [1687, 1458, 0], [1448, 1477, 0], [1794, 1477, 0],
    [2183, 1477, 0], [1211, 1499, 1], [1774, 1499, 0], [1782, 1499, 0], [1848, 1499, 0],
    [1211, 1519, 0], [1229, 1519, 1], [1332, 1519, 0], [2258, 1519, 0], [1332, 1528, 0],
    [1327, 1546, 0], [1332, 1546, 1], [1606, 1546, 0], [1918, 1546, 1], [1499, 1549, 0],
    [2258, 1549, 0], [1327, 1551, 0], [1870, 1551, 0], [1211, 1590, 1], [1229, 1590, 1],
    [1601, 1590, 1], [1211, 1601, 0], [1332, 1601, 1], [1354, 1601, 1], [1405, 1601, 0],
    [1229, 1605, 0], [1332, 1605, 1], [1448, 1605, 0], [1354, 1606, 0], [1927, 1606, 0],
    [1450, 1608, 0], [1519, 1608, 1], [1849, 1608, 0], [1477, 1687, 1], [1848, 1687, 0],
    [1141, 1690, 1], [1327, 1690, 1], [1549, 1731, 1], [1590, 1731, 0], [2250, 1731, 0],
    [1332, 1747, 1], [1927, 1747, 0], [1450, 1758, 1], [2076, 1758, 0], [1747, 1774, 0],
    [1794, 1774, 0], [1332, 1782, 1], [1687, 1782, 0], [1747, 1782, 0], [1758, 1782, 1],
    [2076, 1782, 1], [1458, 1794, 1], [1590, 1794, 1], [2153, 1794, 0], [1747, 1848, 0],
    [1758, 1848, 0], [1870, 1848, 0], [2183, 1848, 0], [1332, 1849, 1], [1731, 1849, 1],
    [1870, 1849, 1], [2183, 1849, 0], [2250, 1849, 0], [1758, 1870, 0], [1918, 1870, 1],
    [1549, 1918, 0], [1448, 1927, 1], [1758, 1927, 1], [2183, 1927, 1], [2250, 1927, 0],
    [2258, 1927, 0], [1601, 2076, 1], [1605, 2076, 1], [1849, 2076, 1], [2258, 2076, 0],
    [1458, 2153, 1], [1927, 2153, 0], [2076, 2153, 0], [1690, 2183, 1], [2076, 2183, 1],
    [2153, 2183, 0], [1499, 2250, 1], [1690, 2250, 1], [1747, 2250, 0], [1918, 2250, 1],
    [2183, 2250, 0], [1747, 2258, 1]
])

# Testovací data (2D)
test_data = np.array([
    [1451, 1050, 0], [1758, 1050, 0], [1346, 1211, 1], [1546, 1332, 1], [1608, 1451, 1],
    [1839, 1458, 0], [1435, 1538, 1], [1077, 1546, 1], [2183, 1551, 0], [1458, 1590, 0],
    [1538, 1606, 0], [1077, 1608, 1], [2258, 1608, 0], [1419, 1690, 1], [1545, 1731, 1],
    [1774, 1758, 0], [1545, 1774, 1], [2183, 1777, 0], [1228, 1794, 1], [1774, 1794, 0],
    [2258, 1870, 1], [1546, 1903, 1], [1774, 1918, 0], [2076, 1918, 0], [1758, 2076, 1],
    [1839, 2076, 0], [2107, 2076, 1], [2258, 2107, 1], [1731, 2202, 1], [1327, 2258, 1],
    [1354, 1229, 0], [1774, 1229, 0], [1546, 1376, 0], [1918, 1405, 0], [1605, 1450, 0],
    [1605, 1477, 0], [1448, 1519, 0], [1450, 1519, 1], [1141, 1528, 0], [1346, 1551, 1],
    [1608, 1606, 0], [1376, 1608, 1], [2153, 1687, 0], [1458, 1690, 0], [1590, 1690, 1],
    [1774, 1731, 0], [1229, 1747, 1], [2250, 1758, 0], [1346, 1848, 1], [1376, 1870, 1],
    [2258, 1870, 0], [1590, 1918, 1], [1849, 2153, 0], [1782, 2183, 1], [2153, 2258, 0]
])

# Rozdělení na vstupy (X) a výstupy (y)
X_train = train_data[:, 0:2]
y_train = train_data[:, 2]
X_test = test_data[:, 0:2]
y_test = test_data[:, 2]

# Normalizace dat pomocí Z-score
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

# Výpočet váhy tříd
class_weights = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)
class_weight_dict = dict(enumerate(class_weights))

# Funkce pro vytvoření modelu s různými aktivačními funkcemi
def create_model(activation_fns, model_type='dnn'):
    if model_type == 'dnn':
        model = Sequential([
            Input(shape=(2,)),  # Vstupní vrstva pro 2D data
            Dense(128),  # První skrytá vrstva
            Activation(activation_fns[0]),  # Aktivace první vrstvy
            Dense(64),
            Activation(activation_fns[1]),  # Aktivace druhé vrstvy
            Dense(64),
            Activation(activation_fns[2]),  # Aktivace třetí vrstvy
            Dense(64),
            Activation(activation_fns[3]),  # Aktivace čtvrté vrstvy
            Dense(64),
            Activation(activation_fns[4]),  # Aktivace páté vrstvy
            Dense(32),
            Activation(activation_fns[5]),  # Aktivace šesté vrstvy
            Dense(32),
            Activation(activation_fns[6]),  # Aktivace sedmé vrstvy
            Dense(32),
            Activation(activation_fns[7]),  # Aktivace osmé vrstvy
            Dense(32),
            Activation(activation_fns[8]),  # Aktivace deváté vrstvy
            Dense(16),
            Activation(activation_fns[9]),  # Aktivace desáté vrstvy
            Dense(16),
            Activation(activation_fns[10]),  # Aktivace jedenácté vrstvy
            Dense(16),
            Activation(activation_fns[11]),  # Aktivace dvanácté vrstvy
            Dense(8),
            Activation(activation_fns[12]),  # Aktivace třinácté vrstvy
            Dense(1, activation='sigmoid')  # Výstupní vrstva
        ])
    elif model_type == 'cnn':
        model = Sequential([
            Input(shape=(2, 1)),  # Vstupní vrstva pro 2D data
            Flatten(),  # Plochý vstup
            Dense(128),
            Activation(activation_fns[0]),  # Aktivace první vrstvy
            Dense(64),
            Activation(activation_fns[1]),  # Aktivace druhé vrstvy
            Dense(64),
            Activation(activation_fns[2]),  # Aktivace třetí vrstvy
            Dense(64),
            Activation(activation_fns[3]),  # Aktivace čtvrté vrstvy
            Dense(64),
            Activation(activation_fns[4]),  # Aktivace páté vrstvy
            Dense(32),
            Activation(activation_fns[5]),  # Aktivace šesté vrstvy
            Dense(32),
            Activation(activation_fns[6]),  # Aktivace sedmé vrstvy
            Dense(32),
            Activation(activation_fns[7]),  # Aktivace osmé vrstvy
            Dense(32),
            Activation(activation_fns[8]),  # Aktivace deváté vrstvy
            Dense(16),
            Activation(activation_fns[9]),  # Aktivace desáté vrstvy
            Dense(16),
            Activation(activation_fns[10]),  # Aktivace jedenácté vrstvy
            Dense(16),
            Activation(activation_fns[11]),  # Aktivace dvanácté vrstvy
            Dense(8),
            Activation(activation_fns[12]),  # Aktivace třinácté vrstvy
            Dense(1, activation='sigmoid')  # Výstupní vrstva
        ])
    return model

# Funkce pro optimalizaci pomocí Optuna
def objective(trial, model_type='dnn'):
    # Náhodný výběr aktivačních funkcí pro každou vrstvu
    activation_fns = []
    for i in range(13):
        activation_fn = trial.suggest_categorical(f'activation_fn_{i}', ['sigmoid', 'tanh', 'relu', 'leaky_relu'])
        activation_fns.append(activation_fn)

    # Vytvoření modelu s náhodně vybranými aktivačními funkcemi
    model = create_model(activation_fns, model_type)
    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
    model.fit(X_train, y_train, epochs=200, batch_size=10, verbose=0, class_weight=class_weight_dict)

    # Predikce pravděpodobností pro testovací sadu
    if model_type == 'cnn':
        X_test_input = X_test.reshape(-1, 2, 1)
    else:
        X_test_input = X_test

    y_probs = model.predict(X_test_input)

    # Optimalizace threshold pro minimalizaci FP + FN
    threshold = trial.suggest_float('threshold', 0.0, 1.0)
    y_pred_optimal = (y_probs > threshold).astype("int32")

    # Výpočet confusion matrix
    conf_matrix = confusion_matrix(y_test, y_pred_optimal)
    FP = conf_matrix[0, 1]
    FN = conf_matrix[1, 0]

    # Cílem je minimalizovat FP + FN
    return FP + FN

# Optimalizace pro DNN model
study_dnn = optuna.create_study(direction='minimize')
study_dnn.optimize(lambda trial: objective(trial, model_type='dnn'), n_trials=50)

# Optimalizace pro CNN model
study_cnn = optuna.create_study(direction='minimize')
study_cnn.optimize(lambda trial: objective(trial, model_type='cnn'), n_trials=50)

# Výsledky optimalizace pro DNN model
optimal_threshold_dnn = study_dnn.best_params["threshold"]
best_activation_fns_dnn = [study_dnn.best_params[f'activation_fn_{i}'] for i in range(13)]
print(f'DNN - Optimální threshold: {optimal_threshold_dnn}')
print(f'DNN - Nejlepší kombinace aktivačních funkcí: {best_activation_fns_dnn}')

# Výsledky optimalizace pro CNN model
optimal_threshold_cnn = study_cnn.best_params["threshold"]
best_activation_fns_cnn = [study_cnn.best_params[f'activation_fn_{i}'] for i in range(13)]
print(f'CNN - Optimální threshold: {optimal_threshold_cnn}')
print(f'CNN - Nejlepší kombinace aktivačních funkcí: {best_activation_fns_cnn}')

# Vytvoření nejlepšího modelu s optimalizovanými aktivačními funkcemi pro DNN
best_dnn_model = create_model(best_activation_fns_dnn, model_type='dnn')
best_dnn_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
best_dnn_model.fit(X_train, y_train, epochs=200, batch_size=10, verbose=0, class_weight=class_weight_dict)

# Predikce s nejlepšími aktivačními funkcemi a optimálním threshold pro DNN
y_probs_dnn_best = best_dnn_model.predict(X_test)
y_pred_dnn_best = (y_probs_dnn_best > optimal_threshold_dnn).astype("int32")

# Vyhodnocení výsledného DNN modelu pomocí confusion matrix
conf_matrix_dnn_best = confusion_matrix(y_test, y_pred_dnn_best)
plt.figure(figsize=(10, 7))
sns.heatmap(conf_matrix_dnn_best, annot=True, fmt='d', cmap='Blues')
plt.title(f'Konfuzní Matice (DNN) - Testovací data, Optimalizovaný Threshold: {optimal_threshold_dnn:.2f}')
plt.ylabel('Skutečný Štítek')
plt.xlabel('Predikovaný Štítek')
plt.show()

# Vytvoření nejlepšího modelu s optimalizovanými aktivačními funkcemi pro CNN
best_cnn_model = create_model(best_activation_fns_cnn, model_type='cnn')
best_cnn_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
best_cnn_model.fit(X_train.reshape(-1, 2, 1), y_train, epochs=200, batch_size=10, verbose=0, class_weight=class_weight_dict)

# Predikce s nejlepšími aktivačními funkcemi a optimálním threshold pro CNN
y_probs_cnn_best = best_cnn_model.predict(X_test.reshape(-1, 2, 1))
y_pred_cnn_best = (y_probs_cnn_best > optimal_threshold_cnn).astype("int32")

# Vyhodnocení výsledného CNN modelu pomocí confusion matrix
conf_matrix_cnn_best = confusion_matrix(y_test, y_pred_cnn_best)
plt.figure(figsize=(10, 7))
sns.heatmap(conf_matrix_cnn_best, annot=True, fmt='d', cmap='Blues')
plt.title(f'Konfuzní Matice (CNN) - Testovací data, Optimalizovaný Threshold: {optimal_threshold_cnn:.2f}')
plt.ylabel('Skutečný Štítek')
plt.xlabel('Predikovaný Štítek')
plt.show()