0
$\begingroup$

I'm trying to predict a class of some data, and am struggling.

So, to debug I created a simple test dataset, yet I am having the same issues. I've tried adding weighting, and lastly adding a column that directly matches the output. Still, I'm only ever getting prediction of one class.

I think I'm missing something obvious in my code, and am just too close to it.

Code to make the data:

import polars as pl
from datetime import datetime, timedelta

start_date = datetime(2020, 1, 1)
num_samples = 100000
dates = [start_date + timedelta(days=i) for i in range(num_samples)]
incrementing_numbers = list(range(num_samples))

df = pl.DataFrame({
    "timestamp": dates,
    "i": incrementing_numbers
})

# make a "fizz_buzz" like column
df = df.with_columns(
    pl.when((pl.col("i") % 10 == 0))
    .then(-1)
    .when(pl.col("i") % 3 == 0)
    .then(1)
    .otherwise(0)
    .alias("fizz_buzz")
)

# dup last col, now let the the that be part of the inputs...
df = df.with_columns(
    pl.when((pl.col("i") % 10 == 0))
    .then(-1)
    .when(pl.col("i") % 3 == 0)
    .then(1)
    .otherwise(0)
    .alias("fizz_buzz_helper")
)

Code for the weights:

# pick where we're gonna run this
device = torch.device("cuda")


class_counts = df.group_by("fizz_buzz").agg([
    pl.col("fizz_buzz").count().alias("count")
])
class_counts = class_counts.sort("fizz_buzz")
class_sample_counts = class_counts["count"].to_list()

weights = 1. / torch.tensor(class_sample_counts, dtype=torch.float)
weights = weights / weights.sum()  # Normalize weights

print(weights)
criterion = nn.CrossEntropyLoss(weight=weights.to(device), label_smoothing=0.1)

the main model et al

import time
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import numpy as np
from torch.utils.data import Dataset, DataLoader
import polars as pl
from sklearn.metrics import precision_score, recall_score, f1_score
from sklearn.preprocessing import StandardScaler
from collections import Counter

class SimpleDataset(Dataset):
    def __init__(self, dataframe):
        # Drop 'fizz_buzz' column for features and select 'fizz_buzz' column for labels
        self.features = dataframe.drop('fizz_buzz').to_numpy()
        self.labels = dataframe.select('fizz_buzz').to_numpy().squeeze()

    def __len__(self):
        return len(self.labels)

    def __getitem__(self, idx):
        features = torch.tensor(self.features[idx], dtype=torch.float)
        labels = torch.tensor(self.labels[idx], dtype=torch.long) + 1

        return features, labels

class Net(nn.Module):
    def __init__(self, input_size, layer_sizes, output_classes, dropout_rate=0.25):
        super(Net, self).__init__()
        self.layers = nn.ModuleList()

        # Create layers based on the provided configuration
        for i, layer_size in enumerate(layer_sizes):
            if i == 0:
                self.layers.append(nn.Linear(input_size, layer_size))
            else:
                self.layers.append(nn.Linear(layer_sizes[i-1], layer_size))
            if i < len(layer_sizes) - 1:  # No dropout after the last layer
                self.layers.append(nn.Dropout(dropout_rate))

        self.output = nn.Linear(layer_sizes[-1], output_classes)

    def forward(self, x):
        for layer in self.layers:
            if isinstance(layer, nn.Linear):
                x = torch.relu(layer(x))
            else:
                x = layer(x)  # Apply dropout
        x = self.output(x)
        return x



def train(model, device, train_loader, test_loader, num_epochs):
    model = model.to(device)
    optimizer = optim.Adam(net.parameters(), lr=0.001)
    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=5)

    # Training loop
    for epoch in range(num_epochs):
        start_time = time.time()

        # Training phase
        net.train()
        for i, (inputs, labels) in enumerate(train_loader):
            inputs, labels = inputs.to(device), labels.to(device)

            optimizer.zero_grad()
            outputs = net(inputs)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()



        net.eval()
        test_losses = []
        actuals = []
        predictions = []
        with torch.no_grad():
            for i, (inputs, targets) in enumerate(test_loader):
                inputs, targets = inputs.to(device), targets.to(device)
                inputs = inputs.view(inputs.shape[0], -1)
                outputs = net(inputs).to(device)
                loss = criterion(outputs, targets)
                test_losses.append(loss.item())

                _, predicted_classes = torch.max(outputs, 1)

                actuals.extend(targets.cpu().numpy())
                predictions.extend(predicted_classes.cpu().numpy())


        # Calculate the average test loss for the epoch
        avg_test_loss = np.mean(test_losses)
        elapsed_time = time.time() - start_time

        precision = precision_score(actuals, predictions, average='macro', zero_division=0)
        recall = recall_score(actuals, predictions, average='macro', zero_division=0)
        f1 = f1_score(actuals, predictions, average='macro', zero_division=0)

        current_lr = optimizer.param_groups[0]['lr']
        prediction_counts = Counter(predictions)
        print(f'Epoch {epoch+1}, LR: {current_lr:5.9f}, Avg Test Loss: {avg_test_loss:.3f}, Precision: {precision:.3f}, Recall: {recall:.3f}, F1: {f1:.3f}, Time: {elapsed_time:.2f}s, Pred: {prediction_counts}')

        scheduler.step(avg_test_loss)











# save 20% for testing...... and at the end of the dataset (.....as timeseries?)
split_point = int(len(df) * 0.8)
train_data = df.slice(0, split_point)
test_data = df.slice(split_point, len(df) - split_point)


params = {'batch_size': 256, 'shuffle': True, 'num_workers': 12, 'pin_memory': True}
train_loader = DataLoader(SimpleDataset(train_data), **params)
test_loader = DataLoader(SimpleDataset(test_data), **params)



net_output = 3
net_features = len(train_data.columns) - 1
layer_sizes = [64, 128, 64]
dropout_rate = 0.25
net = Net(net_features, layer_sizes, 3, dropout_rate).to(device)


print(f"Training on {device} 🀞")
train(net, device, train_loader, test_loader, 25)
print('Finished Training')

This results in:

Total data points: 100000
Matches: 10000
Mismatches: 90000
Accuracy: 10.00%

Distribution of 'fizz_buzz' values:
shape: (3, 2)
β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”
β”‚ fizz_buzz ┆ len   β”‚
β”‚ ---       ┆ ---   β”‚
β”‚ i32       ┆ u32   β”‚
β•žβ•β•β•β•β•β•β•β•β•β•β•β•ͺ═══════║
β”‚ -1        ┆ 10000 β”‚
β”‚ 0         ┆ 60000 β”‚
β”‚ 1         ┆ 30000 β”‚
β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”˜

Distribution of 'pred' (prediction) values:
shape: (1, 2)
β”Œβ”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”
β”‚ pred ┆ len    β”‚
β”‚ ---  ┆ ---    β”‚
β”‚ i64  ┆ u32    β”‚
β•žβ•β•β•β•β•β•β•ͺ════════║
β”‚ -1   ┆ 100000 β”‚
β””β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”˜
$\endgroup$

0

Your Answer

By clicking β€œPost Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.