ニューラルネットワークのアーキテクチャをどう考えるか ～ドロップアウト率や層数の調整、さらに考慮すべきポイントまで～

import numpy as np
import pandas as pd
import itertools
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.metrics import roc_auc_score

import tensorflow as tf
from tensorflow.keras import layers, regularizers

# 層構成（2層と3層の例）
layer_configs = [
    [128, 64],
    [128, 64, 32],
]

# 2層用、3層用でドロップアウト率を網羅的に生成
dropout_rates_2_layers = list(itertools.product([0.1, 0.2, 0.3], repeat=2))  # 3^2=9通り
dropout_rates_3_layers = list(itertools.product([0.1, 0.2, 0.3], repeat=3))  # 3^3=27通り

def build_dnn_model_per_layer(input_dim, layer_sizes, dropout_rates, l2_reg=1e-4):
    if len(layer_sizes) != len(dropout_rates):
        raise ValueError("Mismatch in length of layer_sizes and dropout_rates.")
    
    model = tf.keras.Sequential()
    model.add(layers.Input(shape=(input_dim,)))
    
    for size, d_rate in zip(layer_sizes, dropout_rates):
        model.add(layers.Dense(size, activation='relu', kernel_regularizer=regularizers.l2(l2_reg)))
        model.add(layers.Dropout(d_rate))
    
    model.add(layers.Dense(1, activation='sigmoid'))
    model.compile(
        optimizer='adam',
        loss='binary_crossentropy',
        metrics=[tf.keras.metrics.AUC(name='auc')]
    )
    return model

def run_cv_for_config(X, y, layer_sizes, dropout_rates, folds=10, repeats=3, l2_reg=1e-4):
    rkf = RepeatedStratifiedKFold(n_splits=folds, n_repeats=repeats, random_state=42)
    oof_preds = np.zeros(len(X))
    
    for train_idx, val_idx in rkf.split(X, y):
        X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]
        y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]
        
        model = build_dnn_model_per_layer(
            input_dim=X_train.shape[1],
            layer_sizes=layer_sizes,
            dropout_rates=dropout_rates,
            l2_reg=l2_reg
        )
        
        early_stopping = tf.keras.callbacks.EarlyStopping(
            monitor='val_auc',
            patience=5,
            restore_best_weights=True,
            mode='max'
        )
        
        model.fit(
            X_train, y_train,
            validation_data=(X_val, y_val),
            epochs=100,
            batch_size=32,
            callbacks=[early_stopping],
            verbose=0
        )
        
        val_preds = model.predict(X_val).ravel()
        oof_preds[val_idx] = val_preds
    
    return roc_auc_score(y, oof_preds)

# 結果を格納
results = []
for config in layer_configs:
    if len(config) == 2:
        # 2層用
        for d_rates in dropout_rates_2_layers:
            auc_score = run_cv_for_config(X, y, config, d_rates)
            results.append({'layer_config': config, 'dropout_rates': d_rates, 'auc_score': auc_score})
    elif len(config) == 3:
        # 3層用
        for d_rates in dropout_rates_3_layers:
            auc_score = run_cv_for_config(X, y, config, d_rates)
            results.append({'layer_config': config, 'dropout_rates': d_rates, 'auc_score': auc_score})

df_results = pd.DataFrame(results)
print(df_results.sort_values(by='auc_score', ascending=False).head(10))

# 例: ユニット数を増やして表現力を高める
model = keras.Sequential([
    layers.Input(shape=(input_dim,)),
    layers.Dense(256, activation='relu'),  # 256ユニット
    layers.Dense(128, activation='relu'),  # 128ユニット
    layers.Dense(1, activation='sigmoid')
])

# 例: Swishを使ってみる
# (TensorFlow 2.2 以降なら "tf.nn.swish" も使えます)
model = keras.Sequential([
    layers.Input(shape=(input_dim,)),
    layers.Dense(128, activation='relu'),
    layers.Dense(64, activation='selu'),  # SELUを試す例
    layers.Dense(32, activation='gelu'),  # GELUを試す例
    layers.Dense(1, activation='sigmoid')
])

from tensorflow.keras import regularizers

model = keras.Sequential([
    layers.Input(shape=(input_dim,)),
    layers.Dense(128, activation='relu',
                 kernel_regularizer=regularizers.l2(1e-4)),  # L2正則化
    layers.Dense(64, activation='relu',
                 kernel_regularizer=regularizers.l2(1e-4)),
    layers.Dense(1, activation='sigmoid')
])

model = keras.Sequential([
    layers.Input(shape=(input_dim,)),
    layers.Dense(128, use_bias=False),  # BNでバイアス不要になることも多い
    layers.BatchNormalization(),
    layers.Activation('relu'),
    layers.Dense(64, use_bias=False),
    layers.BatchNormalization(),
    layers.Activation('relu'),
    layers.Dense(1, activation='sigmoid')
])

model.compile(
    optimizer=tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9),
    loss='binary_crossentropy',
    metrics=[tf.keras.metrics.AUC(name='auc')]
)

lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
    initial_learning_rate=0.01,
    decay_steps=1000,
    decay_rate=0.96,
    staircase=True
)

optimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule)

model.compile(
    optimizer=optimizer,
    loss='binary_crossentropy',
    metrics=[tf.keras.metrics.AUC(name='auc')]
)

from tensorflow.keras.preprocessing.image import ImageDataGenerator

datagen = ImageDataGenerator(
    rotation_range=20,
    width_shift_range=0.2,
    height_shift_range=0.2,
    horizontal_flip=True
)

# fitやflowを利用して拡張された画像を学習に使う
datagen.fit(X_train)  # 画像が入った4D配列を想定

model.fit(
    X_train, y_train,
    validation_data=(X_val, y_val),
    epochs=50,
    batch_size=64,  # バッチサイズを大きく/小さくする
    callbacks=[early_stopping],
    verbose=1
)