pytorch tabular

1class TabularModel(nn.Module):
2
3    def __init__(self, emb_szs, n_cont, out_sz, layers, p=0.5):
4        # Call the parent __init__
5        super().__init__()
6        
7        # Set up the embedding, dropout, and batch normalization layer attributes
8        self.embeds = nn.ModuleList([nn.Embedding(ni, nf) for ni,nf in emb_szs])
9        self.emb_drop = nn.Dropout(p)
10        self.bn_cont = nn.BatchNorm1d(n_cont)
11        
12        # Assign a variable to hold a list of layers
13        layerlist = []
14        
15        # Assign a variable to store the number of embedding and continuous layers
16        n_emb = sum((nf for ni,nf in emb_szs))
17        n_in = n_emb + n_cont
18        
19        # Iterate through the passed-in "layers" parameter (ie, [200,100]) to build a list of layers
20        for i in layers:
21            layerlist.append(nn.Linear(n_in,i)) 
22            layerlist.append(nn.ReLU(inplace=True))
23            layerlist.append(nn.BatchNorm1d(i))
24            layerlist.append(nn.Dropout(p))
25            n_in = i
26        layerlist.append(nn.Linear(layers[-1],out_sz))
27        
28        # Convert the list of layers into an attribute
29        self.layers = nn.Sequential(*layerlist)
30    
31    def forward(self, x_cat, x_cont):
32        # Extract embedding values from the incoming categorical data
33        embeddings = []
34        for i,e in enumerate(self.embeds):
35            embeddings.append(e(x_cat[:,i]))
36        x = torch.cat(embeddings, 1)
37        # Perform an initial dropout on the embeddings
38        x = self.emb_drop(x)
39        
40        # Normalize the incoming continuous data
41        x_cont = self.bn_cont(x_cont)
42        x = torch.cat([x, x_cont], 1)
43        
44        # Set up model layers
45        x = self.layers(x)
46        return x
47
48# one hidden layer containing 50 neurons
49model = TabularModel(emb_szs, conts.shape[1], 2, [50], p=0.4)
50
51#train
52criterion = nn.CrossEntropyLoss() # classification
53#criterion = nn.MSELoss() for regression
54optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
55epochs = 300
56losses = []
57
58for i in range(epochs):
59    i+=1
60    y_pred = model(cat_train, con_train)
61    loss = criterion(y_pred, y_train)
62    losses.append(loss)
63    
64    # a neat trick to save screen space:
65    if i%25 == 1:
66        print(f'epoch: {i:3}  loss: {loss.item():10.8f}')
67
68    optimizer.zero_grad()
69    loss.backward()
70    optimizer.step()