Im using BERT in tensorflow, but when I try to turn it deterministic I got the error: "When determinism is enabled, random ops must have a seed specified. [[{{node dropout/dropout/random_uniform/RandomUniform}}]] [Op:__inference_train_function_536707]"
I alredy know that the problem is that Im training BERT with my data
hub.KerasLayer(encoder_url, trainable=True, name='BERT_encoder')
But I need to fine-tune BERT and I need reproducibility too. Any thoughts how to solve this?
A short version of my code:
import os
import pandas as pd
import numpy as np
import random
import tensorflow as tf
SEED = 12
os.environ['PYTHONHASHSEED'] = str(SEED)
random.seed(SEED)
np.random.seed(SEED)
tf.random.set_seed(SEED)
tf.keras.utils.set_random_seed(SEED)
tf.config.experimental.enable_op_determinism()
import tensorflow_hub as hub
import tensorflow_text as text
from tensorflow import keras
from tensorflow.keras.metrics import BinaryAccuracy, Precision, Recall
preprocess_url = "https://tfhub.dev/tensorflow/bert_multi_cased_preprocess/3"
encoder_url = "https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/4"
input_layer = keras.layers.Input(shape=(), dtype=tf.string, name='text')
preprocessing_layer = hub.KerasLayer(preprocess_url, name='preprocessing')
encoder_inputs = preprocessing_layer(input_layer)
encoder = hub.KerasLayer(encoder_url, trainable=True, name='BERT_encoder') # The problem is trainable=True, when trainable=False it works fine
outputs = encoder(encoder_inputs)
net = outputs['pooled_output']
net = keras.layers.Dropout(0.1)(net)
net = keras.layers.Dense(1, activation='sigmoid', name='classifier')(net)
model = keras.Model(input_layer, net)
loss = keras.losses.BinaryCrossentropy(from_logits=False)
metrics=[Precision(name='precision'), Recall(name='recall')]
model.compile(optimizer=keras.optimizers.Adam(), loss=loss, metrics=metrics)
model.fit(train_ds, epochs=2, shuffle=False)
