0
$\begingroup$

I am using ViT for image classification, I scaled images in range of [-1,1], and I also padded images. Then, I used the following code to see the original image and generated patches, but the output of generated patches is not what I expected. I added images too. I have a couple of questions:

  1. Can I use pretrained vit for non-square images?
  2. Does padding have negative effect on performance when the original size is (128,431) and I padded the data into (432,432) because I did not want to lose any information from my image.
  3. How can I have a correct generated patches for vit ?

I appreciate your help.

enter image description here

enter image description here

import numpy as np
import tensorflow as tf
import tensorflow.keras.layers as L
import matplotlib.pyplot as plt
import librosa.display


output_train = np.load('C:/..../ViT/output_train.npy')

class Patches(L.Layer):
    def __init__(self, patch_size):
        super(Patches, self).__init__()
        self.patch_size = patch_size

    def call(self, images):
        batch_size = tf.shape(images)[0]
        patches = tf.image.extract_patches(
            images = images,
            sizes = [1, self.patch_size, self.patch_size, 1],
            strides = [1, self.patch_size, self.patch_size, 1],
            rates = [1, 1, 1, 1],
            padding = 'VALID',
        )
        patch_dims = patches.shape[-1]
        patches = tf.reshape(patches, [batch_size, -1, patch_dims])
        return patches

def visualize_patches(image_size, patch_size):
    
    plt.figure(figsize=(4, 4))
    image = output_train[0][0]

    single_channel_spectrogram = np.mean(image, axis=-1)
    single_channel_spectrogram = np.squeeze(single_channel_spectrogram)
    librosa.display.specshow(single_channel_spectrogram,sr=44100,
                             x_axis='time',y_axis='mel',
                             fmax=5000)
    plt.axis('off')

    resized_image = tf.image.resize(
        tf.convert_to_tensor([image]), size = (image_size, image_size)
    )

    patches = Patches(patch_size)(resized_image)
    print(f'Image size: {image_size} X {image_size}')
    print(f'Patch size: {patch_size} X {patch_size}')
    print(f'Patches per image: {patches.shape[1]}')
    print(f'Elements per patch: {patches.shape[-1]}')
    
    n = int(np.sqrt(patches.shape[1]))
    plt.figure(figsize=(4, 4))
    
    for i, patch in enumerate(patches[0]):
        ax = plt.subplot(n, n, i + 1)
        patch_img = tf.reshape(patch, (patch_size, patch_size, 3))
        single_channel_patch_img = np.mean(patch_img, axis=-1)
        single_channel_patch_img = np.squeeze(single_channel_patch_img)
        librosa.display.specshow(single_channel_patch_img,sr=44100,
                                 x_axis='time',y_axis='mel',
                                 fmax=5000)
        
        plt.axis('off')
    plt.show()

if __name__ == '__main__':
    visualize_patches(224, 32)


$\endgroup$

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.