0
$\begingroup$

I am trying to reproduce Audio super-resolution. At the bottom is architecture in PyTorch from this paper (https://github.com/dsgiitr/Audio-Super-Resolution).

It is supposed to accept downsamples/smaller frequency rate audio and return a higher frequency rate.

However, I am struggling with up-sampling layers, they seem to return the wrong shape of values and can't proceed further, could anyone provide any tips?

I feed one channel audio:

Input [1, 1, 8192]

Downsampling:

d1 - torch.Size([1, 128, 4096])
d2 - torch.Size([1, 256, 2048])
d3 - torch.Size([1, 512, 1024])
d4 - torch.Size([1, 512, 512])

Bottlenck:

b - torch.Size([1, 512, 256])

Upsampling:

up1 - torch.Size([1, 128, 512])

After first upsampling there are problems: torch.cat([up1, d4]) - doesn't work, unless you cat on on channels, torch.cat([up1, d4],1) but this has effects that later Subpixel1d is not working anymore. So I guess something is wrong in the process?

UNet Architecture

#SubPixel1d
def SubPixel1d(tensor, r): #(b,r,w)
    ps = nn.PixelShuffle(r)
    tensor = torch.unsqueeze(tensor, -1) #(b,r,w,1)
    tensor = ps(tensor)
    #print(tensor.shape) #(b,1,w*r,r)
    tensor = torch.mean(tensor, -1)
    #print(tensor.shape) #(b,1,w*r)
    return tensor

n_filters = [128, 384, 512, 512, 512, 512, 512, 512]
n_filtersizes = [65, 33, 17,  9,  9,  9,  9, 9, 9]
num_layers = 4



class Down1D(nn.Module):
    """doc string for Down1D"""
    def __init__(self, in_channel, n_filters, n_filtersizes):
        super(Down1D, self).__init__()

        self.c1 = nn.Conv1d(in_channel, n_filters, kernel_size=n_filtersizes, stride=2, padding=math.floor(n_filtersizes/2)) #, padding=kernel/2 ) 
        nn.init.orthogonal_(self.c1.weight)

    def forward(self, x):
        x1 = self.c1(x)
        x1 = F.leaky_relu(x1, negative_slope=0.2)
        return x1



class Up1D(nn.Module):
    """doc string for Up1D"""
    def __init__(self, in_channel, n_filters, n_filtersizes):
        super(Up1D, self).__init__()

        self.c1 = nn.ConvTranspose1d(in_channel, n_filters, kernel_size=n_filtersizes, stride=2,padding=math.floor(n_filtersizes/2))
        nn.init.orthogonal_(self.c1.weight)
        self.drop = nn.Dropout(p=0.5)


    def forward(self, x):
        x1 = self.c1(x)
        x1 = self.drop(x1)
        x1 = F.relu(x1)
        x1 = SubPixel1d(x, r=2)
        return x1



class Bottleneck(nn.Module):
    """doc string for Bottleneck"""
    def __init__(self, in_channel, n_filters, n_filtersizes):
        super(Bottleneck, self).__init__()

        self.c1 = nn.Conv1d(in_channel, n_filters, kernel_size=n_filtersizes, stride=2, padding=math.floor(n_filtersizes/2))
        nn.init.orthogonal_(self.c1.weight)
        self.drop = nn.Dropout(p=0.5)


    def forward(self, x):
        x1 = self.c1(x)
        x1 = self.drop(x1)
        x1 = F.leaky_relu(x1, negative_slope=0.2)
        return x1

class AudioUnet(nn.Module):
    def __init__(self, num_layers):
        super(AudioUnet, self).__init__()
        self.downsample = nn.ModuleList([])
        in_channels = 1
        for l, nf, fs in zip(range(num_layers), n_filters, n_filtersizes):
            self.downsample.append(Down1D(in_channels, nf, fs))
            in_channels = nf

        self.bottleneck = Bottleneck(in_channels, n_filters[-1], n_filtersizes[-1])
        
        self.upsample = nn.ModuleList([])
        for l, nf, fs in reversed(list(zip(range(num_layers), n_filters, n_filtersizes))):
            self.upsample.append(Up1D(in_channels, nf, fs))
            in_channels = nf

        self.final = nn.Conv1d(in_channels, 2, 9, stride=2, padding=5)
        nn.init.normal_(self.final.weight)

    def forward(self, x):
        down_outs = [x]

        for i in range(num_layers):
            down_outs.append(self.downsample[i](down_outs[i]))
    

        x1 = self.bottleneck(down_outs[-1])
        for i, d in zip(range(num_layers), reversed(down_outs[1:])):
            x1 = self.upsample[i](x1)
            x1 = torch.cat([x1, down_outs[i]]) #concat axis =-1 for tf
        x1 = self.final(x1)
        x1 = SubPixel1D(x1, r=2)
        x1 = x1 + x

        return x1
```
$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.