nn_layers.py

import torch
from torch import nn
from librosa.filters import mel as librosa_mel_fn
from stft import STFT

torch.manual_seed(1234)

clip_val = 1e-5
C = 1


class convolutional_module(nn.Module):
    """This class defines a 1d convolutional layer and its initialization for the system we are
    replicating"""
    def __init__(self, in_ch, out_ch, kernel_size=1, stride=1, padding=None, dilation=1, bias=True,
                 w_init_gain='linear'):
        # in PyTorch you define your Models as subclasses of torch.nn.Module
        super(convolutional_module, self).__init__()
        if padding is None:
            assert(kernel_size % 2 == 1)
            padding = int(dilation * (kernel_size - 1) / 2)

        # initialize the convolutional layer which is an instance of Conv1d
        # torch.nn.Conv1d calls internally the method torch.nn.functional.conv1d, which accepts the
        # input with the shape (minibatch x in_channels x input_w), and a weight of shape
        # (out_channels x (in_channels/groups) x kernel_w). In our case, we do not split into groups.
        # Then, our input shape will be (48 x 512 x 189) and the weights are set up as
        # (512 x 512 x 5)
        self.conv_layer = torch.nn.Conv1d(in_ch, out_ch, kernel_size=kernel_size, stride=stride,
                                          padding=padding, dilation=dilation, bias=bias)

        """Useful information of Xavier initialization in:
        https://prateekvjoshi.com/2016/03/29/understanding-xavier-initialization-in-deep-neural-networks/"""
        torch.nn.init.xavier_uniform_(self.conv_layer.weight, gain=torch.nn.init.calculate_gain(w_init_gain))

    def forward(self, x):
        conv_output = self.conv_layer(x)
        return conv_output


class linear_module(torch.nn.Module):
    """This class defines a linear layer and its initialization method for the system we are
    replicating. This implements a linear transformation: y = xA^t + b"""
    def __init__(self, in_dim, out_dim, bias=True, w_init_gain='linear'):
        super(linear_module, self).__init__()
        self.linear_layer = torch.nn.Linear(in_dim, out_dim, bias=bias)

        torch.nn.init.xavier_uniform_(self.linear_layer.weight, gain=torch.nn.init.calculate_gain(w_init_gain))

    def forward(self, x):
        return self.linear_layer(x)


class location_layer(nn.Module):
    def __init__(self, attention_n_filters, attention_kernel_size, attention_dim):
        super(location_layer, self).__init__()
        padding = int((attention_kernel_size - 1) / 2)
        """We are being very restricting without training a bias"""
        """I think in_channels = 2 is k (number of vectors for every encoded stage position from prev.
        alignment)."""
        self.location_conv = convolutional_module(2, attention_n_filters, kernel_size=attention_kernel_size,
                                                  padding=padding, bias=False, stride=1, dilation=1)
        self.location_dense = linear_module(attention_n_filters, attention_dim, bias=False,
                                            w_init_gain='tanh')

    def forward(self, attention_weights_cat):
        processed_attention = self.location_conv(attention_weights_cat)
        processed_attention = processed_attention.transpose(1, 2)
        processed_attention = self.location_dense(processed_attention)
        return processed_attention


class TacotronSTFT(nn.Module):
    def __init__(self, filter_length=1024, hop_length=256, win_length=1024,
                 n_mel_channels=80, sampling_rate=22050, mel_fmin=0.0,
                 mel_fmax=8000.0):
        super(TacotronSTFT, self).__init__()
        self.n_mel_channels = n_mel_channels
        self.sampling_rate = sampling_rate
        self.stft_fn = STFT(filter_length, hop_length, win_length)
        mel_basis = librosa_mel_fn(sr=sampling_rate, n_fft=filter_length, n_mels=n_mel_channels, 
                                   fmin=mel_fmin, fmax=mel_fmax)
        mel_basis = torch.from_numpy(mel_basis).float()
        self.register_buffer('mel_basis', mel_basis)

    def spectral_de_normalize(self, magnitudes):
        output = torch.exp(magnitudes) / C
        return output

    def mel_spectrogram(self, y):
        """Computes mel-spectrograms from a batch of waves
        PARAMS
        ------
        y: Variable(torch.FloatTensor) with shape (B, T) in range [-1, 1]

        RETURNS
        -------
        mel_output: torch.FloatTensor of shape (B, n_mel_channels, T)
        """
        assert(torch.min(y.data) >= -1)
        assert(torch.max(y.data) <= 1)

        magnitudes, phases = self.stft_fn.transform(y)
        magnitudes = magnitudes.data
        mel_output = torch.matmul(self.mel_basis, magnitudes)
        mel_output = torch.log(torch.clamp(mel_output, min=clip_val) * C)
        return mel_output