i am trying to use a pretrained model facebook/wav2vec2-xls-r-300m and fine tune it for audio classification and more specifically emotion recognition. i am using an audio labeled dataset ( 12 labels )
model conf:
MAX_DURATION = 5
SAMPLING_RATE = 16000
BATCH_SIZE = 32
NUM_CLASSES = 12
HIDDEN_DIM = 768
MAX_SEQ_LENGTH = MAX_DURATION * SAMPLING_RATE # Maximum length of the input audio file.
MAX_FRAMES = 49 MAX_EPOCHS = 2
MODEL_CHECKPOINT = "facebook/wav2vec2-xls-r-300m" # Name of pretrained model from Hugging Face Model
from transformers import Wav2Vec2FeatureExtractor
config = {
"do_normalize": True,
"feature_extractor_type": "Wav2Vec2FeatureExtractor",
"feature_size": 1,
"padding_side": "right",
"padding_value": 0,
"return_attention_mask": True,
"sampling_rate": 16000
}
# Initialize the feature extractor
feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(MODEL_CHECKPOINT, **config)
# Define your preprocess function
def preprocess_function(examples):
audio_arrays = [x["array"] for x in examples["audio"]]
inputs = feature_extractor(
audio_arrays,
max_length=MAX_SEQ_LENGTH,
truncation=True,
padding=True,
)
return inputs
classification head :
from transformers import TFWav2Vec2Model
def mean_pool(hidden_states, feature_lengths):
attenion_mask = tf.sequence_mask(
feature_lengths, maxlen=MAX_FRAMES, dtype=tf.dtypes.int64
)
padding_mask = tf.cast(
tf.reverse(tf.cumsum(tf.reverse(attenion_mask, [-1]), -1), [-1]),
dtype=tf.dtypes.bool,
)
hidden_states = tf.where(
tf.broadcast_to(
tf.expand_dims(~padding_mask, -1), (BATCH_SIZE, MAX_FRAMES, HIDDEN_DIM)
),
0.0,
hidden_states,
)
pooled_state = tf.math.reduce_sum(hidden_states, axis=1) / tf.reshape(
tf.math.reduce_sum(tf.cast(padding_mask, dtype=tf.dtypes.float32), axis=1),
[-1, 1],
)
return pooled_state
class TFWav2Vec2ForAudioClassification(layers.Layer):
"""Combines the encoder and decoder into an end-to-end model for training."""
def __init__(self, model_checkpoint, num_classes):
super().__init__()
# Instantiate the Wav2Vec 2.0 model without the Classification-Head
self.wav2vec2 = TFWav2Vec2Model.from_pretrained(
model_checkpoint, apply_spec_augment=False, from_pt=True
)
self.pooling = layers.GlobalAveragePooling1D()
# Drop-out layer before the final Classification-Head
self.intermediate_layer_dropout = layers.Dropout(0.5)
# Classification-Head
self.final_layer = layers.Dense(num_classes, activation="softmax")
def call(self, inputs):
# We take only the first output in the returned dictionary corresponding to the
# output of the last layer of Wav2vec 2.0
hidden_states = self.wav2vec2(inputs["input_values"])[0]
# If attention mask does exist then mean-pool only un-masked output frames
if tf.is_tensor(inputs["attention_mask"]):
# Get the length of each audio input by summing up the attention_mask
# (attention_mask = (BATCH_SIZE x MAX_SEQ_LENGTH) ∈ {1,0})
audio_lengths = tf.cumsum(inputs["attention_mask"], -1)[:, -1]
# Get the number of Wav2Vec 2.0 output frames for each corresponding audio input
# length
feature_lengths = self.wav2vec2.wav2vec2._get_feat_extract_output_lengths(
audio_lengths
)
pooled_state = mean_pool(hidden_states, feature_lengths)
# If attention mask does not exist then mean-pool only all output frames
else:
pooled_state = self.pooling(hidden_states)
intermediate_state = self.intermediate_layer_dropout(pooled_state)
final_state = self.final_layer(intermediate_state)
return final_state
**model**
def build_model():
# Model's input
inputs = {
"input_values": tf.keras.Input(shape=(MAX_SEQ_LENGTH,), dtype="float32"),
"attention_mask": tf.keras.Input(shape=(MAX_SEQ_LENGTH,), dtype="int32"),
}
# Instantiate the Wav2Vec 2.0 model with Classification-Head using the desired
# pre-trained checkpoint
wav2vec2_model = TFWav2Vec2ForAudioClassification(MODEL_CHECKPOINT, NUM_CLASSES)(
inputs
)
# Model
model = tf.keras.Model(inputs, wav2vec2_model)
# Loss
loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False)
# Optimizer
optimizer = keras.optimizers.Adam(learning_rate=1e-5)
# Compile and return
model.compile(loss=loss, optimizer=optimizer, metrics=["accuracy"])
return model
model = build_model()
i got this error
ValueError Traceback (most recent call last)
<ipython-input-32-b7f377675faf> in <cell line: 23>()
21
22
---> 23 model = build_model()
4 frames
/tmp/__autograph_generated_filejjrdimiv.py in tf__mean_pool(hidden_states, feature_lengths)
10 attenion_mask = ag__.converted_call(ag__.ld(tf).sequence_mask, (ag__.ld(feature_lengths),), dict(maxlen=ag__.ld(MAX_FRAMES), dtype=ag__.ld(tf).dtypes.int64), fscope)
11 padding_mask = ag__.converted_call(ag__.ld(tf).cast, (ag__.converted_call(ag__.ld(tf).reverse, (ag__.converted_call(ag__.ld(tf).cumsum, (ag__.converted_call(ag__.ld(tf).reverse, (ag__.ld(attenion_mask), [-1]), None, fscope), -1), None, fscope), [-1]), None, fscope),), dict(dtype=ag__.ld(tf).dtypes.bool), fscope...
---> 12 hidden_states = ag__.converted_call(ag__.ld(tf).where, (ag__.converted_call(ag__.ld(tf).broadcast_to, (ag__.converted_call(ag__.ld(tf).expand_dims, (~ag__.ld(padding_mask), -1), None, fscope), (ag__.ld(BATCH_SIZE), ag__.ld(MAX_FRAMES), ag__.ld(HIDDEN_DIM))), None, fscope), 0.0, ag__.ld(hidden_states)), None, fscope)
13 pooled_state = ag__.converted_call(ag__.ld(tf).math.reduce_sum, (ag__.ld(hidden_states),), dict(axis=1), fscope) / ag__.converted_call(ag__.ld(tf).reshape, (ag__.converted_call(ag__.ld(tf).math.reduce_sum, (ag__.converted_call(ag__.ld(tf).cast, (ag__.ld(padding_mask),), dict(dtype=ag__.ld(tf).dtypes.float32), fscope),), dict(axis=0...
14 try:
ValueError: Exception encountered when calling layer "tf_wav2_vec2_for_audio_classification" (type TFWav2Vec2ForAudioClassification).
in user code:
File "<ipython-input-31-474bb7233250>", line 56, in call *
pooled_state = mean_pool(hidden_states, feature_lengths)
File "<ipython-input-31-474bb7233250>", line 12, in mean_pool *
hidden_states = tf.where(
ValueError: Dimensions must be equal, but are 49 and 249 for '{{node tf_wav2_vec2_for_audio_classification/SelectV2}} = SelectV2[T=DT_FLOAT](tf_wav2_vec2_for_audio_classification/BroadcastTo, tf_wav2_vec2_for_audio_classification/SelectV2/t, tf_wav2_vec2_for_audio_classification/tf_wav2_vec2_model/wav2vec2/encoder/layer_norm/batchnorm/add_1)' with input shapes: [32,49,768], [], [?,249,1024].
Call arguments received by layer "tf_wav2_vec2_for_audio_classification" (type TFWav2Vec2ForAudioClassification):
• inputs={'input_values': 'tf.Tensor(shape=(None, 80000), dtype=float32)', 'attention_mask': 'tf.Tensor(shape=(None, 80000), dtype=int32)'}
i am using the EdwardLin2023/ASVP_ESD from hugging face , where i found this exact tutorial on how to finetune a facebook/wav2vec2-base on audio classification and since the model i am using facebook/wav2vec2-xls-r-300m is based on wav2vec2 i didnt change any parameters
heres the config of the model i am using as instructed in huggingface
{
"do_normalize": true,
"feature_extractor_type": "Wav2Vec2FeatureExtractor",
"feature_size": 1,
"padding_side": "right",
"padding_value": 0,
"return_attention_mask": true,
"sampling_rate": 16000
}
{
"activation_dropout": 0.0,
"apply_spec_augment": true,
"architectures": [
"Wav2Vec2ForPreTraining"
],
"attention_dropout": 0.1,
"bos_token_id": 1,
"codevector_dim": 768,
"contrastive_logits_temperature": 0.1,
"conv_bias": true,
"conv_dim": [
512,
512,
512,
512,
512,
512,
512
],
"conv_kernel": [
10,
3,
3,
3,
3,
2,
2
],
"conv_stride": [
5,
2,
2,
2,
2,
2,
2
],
"ctc_loss_reduction": "sum",
"ctc_zero_infinity": false,
"diversity_loss_weight": 0.1,
"do_stable_layer_norm": true,
"eos_token_id": 2,
"feat_extract_activation": "gelu",
"feat_extract_dropout": 0.0,
"feat_extract_norm": "layer",
"feat_proj_dropout": 0.1,
"feat_quantizer_dropout": 0.0,
"final_dropout": 0.0,
"gradient_checkpointing": false,
"hidden_act": "gelu",
"hidden_dropout": 0.1,
"hidden_size": 1024,
"initializer_range": 0.02,
"intermediate_size": 4096,
"layer_norm_eps": 1e-05,
"layerdrop": 0.1,
"mask_feature_length": 10,
"mask_feature_prob": 0.0,
"mask_time_length": 10,
"mask_time_prob": 0.075,
"model_type": "wav2vec2",
"num_attention_heads": 16,
"num_codevector_groups": 2,
"num_codevectors_per_group": 320,
"num_conv_pos_embedding_groups": 16,
"num_conv_pos_embeddings": 128,
"num_feat_extract_layers": 7,
"num_hidden_layers": 24,
"num_negatives": 100,
"pad_token_id": 0,
"proj_codevector_dim": 768,
"torch_dtype": "float32",
"transformers_version": "4.12.0.dev0",
"use_weighted_layer_sum": false
}
```
