SymbolicException: Inputs to eager execution function cannot be Keras symbolic tensors

Question

I am writing Encoder-Decoder architecture with Bahdanau Attention using tf.keras with TensorFlow 2.0. Below is my code This is working with TensorFlow 1.15 but getting the error in 2.0. you can check the code in colab notebook here. can you please tell me what is wrong in the code.

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tensorflow.keras.layers import Input, Dense, Conv2D, BatchNormalization, Activation, Dropout, GRU, Embedding
from tensorflow.keras.models import Model
from tensorflow.keras import activations
from tensorflow.keras.layers import Layer
from tensorflow.keras import layers
import tensorflow as tf
from tensorflow.keras.layers import GRU, concatenate, Lambda

ENCODER_SEQ_LEN = 30
DECODER_SEQ_LEN = 20
VOCAB_SIZE = 500
units = 16

tf.keras.backend.clear_session()
class Encoder(Model):
    def __init__(self, vocab_size, embedding_dim, input_length, units):
        super(Encoder, self).__init__()
        self.vocab_size = vocab_size
        self.embedding_dim = embedding_dim
        self.input_length = input_length
        self.units = units
        self.embedding = Embedding(input_dim=VOCAB_SIZE, output_dim=50, input_length=self.input_length,
                           mask_zero=False, name="embedding_layer_encoder")
        self.gru = GRU(self.units, return_state=True, return_sequences=True, name="Encoder_GRU")
    @tf.function
    def call(self, inputs, training=True):
        x_embedd = self.embedding(inputs)
        gru_output, gru_state = self.gru(x_embedd)
        return gru_output, gru_state

class BahdanauAttention(tf.keras.layers.Layer):
    def __init__(self, units):
        super(BahdanauAttention, self).__init__()
        self.W1 = tf.keras.layers.Dense(units)
        self.W2 = tf.keras.layers.Dense(units)
        self.V = tf.keras.layers.Dense(1)

    def call(self, query, values):
        # hidden shape == (batch_size, hidden size)
        # # hidden_with_time_axis shape == (batch_size, 1, hidden size)
        # # we are doing this to perform addition to calculate the score
        hidden_with_time_axis = tf.expand_dims(query, 1)
        # score shape == (batch_size, max_length, 1)
        # # we get 1 at the last axis because we are applying score to self.V
        # # the shape of the tensor before applying self.V is (batch_size, max_length, units)
        score = self.V(tf.nn.tanh(self.W1(values) + self.W2(hidden_with_time_axis)))
        # attention_weights shape == (batch_size, max_length, 1)
        attention_weights = tf.nn.softmax(score, axis=1)
        # context_vector shape after sum == (batch_size, hidden_size)
        context_vector = attention_weights * values
        context_vector = tf.reduce_sum(context_vector, axis=1)
        return context_vector

class onestepDecoder(Model):
    def __init__(self, vocab_size, embedding_dim, dec_units, att_units):
        super(onestepDecoder, self).__init__()
        self.vocab_size = vocab_size
        self.embedding_dim = embedding_dim
        self.dec_units = dec_units
        self.att_units = att_units
        self.embedd = Embedding(input_dim=self.vocab_size, output_dim=self.embedding_dim,
                      input_length=1, mask_zero=False, name="Decoder_Embedding_layer")
        self.att_layer = BahdanauAttention(units=self.att_units) #name='Attention')
        self.dense = Dense(self.vocab_size, activation="softmax", name="DenseOut")
        self.gru = GRU(units=self.dec_units, return_state=True, name="DecGRU")
    @tf.function
    def call(self, input_decoder, input_state, encoder_outputs, training=True):
        x_embedd = self.embedd(input_decoder)
        context_vector = self.att_layer(input_state, encoder_outputs )
        concat = tf.concat([tf.expand_dims(context_vector, 1), x_embedd], axis=-1)
        decoder_output, Decoder_state = self.gru(concat, initial_state=input_state)
        output = self.dense(decoder_output)
        return (output, Decoder_state)
class Decoder(Model):
    def __init__(self, vocab_size, embedding_dim, dec_units, att_units):
        super(Decoder, self).__init__()
        self.vocab_size = vocab_size
        self.embedding_dim = embedding_dim
        self.dec_units = dec_units
        self.att_units = att_units
        self.stepdec = onestepDecoder(self.vocab_size, self.embedding_dim, self.dec_units, self.att_units)
    @tf.function
    def call(self, input_decoder, input_state, encoder_outputs):
        all_outputs= tf.TensorArray(tf.float32, size=input_decoder.shape[1], name="output_arrays")
        for timestep in range(input_decoder.shape[1]):
            output, input_state = self.stepdec(input_decoder[:,timestep:timestep+1], input_state, encoder_outputs)
            all_outputs = all_outputs.write(timestep, output)
        all_outputs = tf.transpose(all_outputs.stack(), [1, 0, 2])
        return all_outputs

encoder_input = Input(shape=(ENCODER_SEQ_LEN,), name='encoder_input_final')
decoder_input = Input(shape=(DECODER_SEQ_LEN,), name="Decoder_inout_final")
encoder = Encoder(vocab_size=VOCAB_SIZE, embedding_dim=50, input_length=ENCODER_SEQ_LEN, units=16)
x_gru_out, x_gru_state = encoder(encoder_input)
decoder = Decoder(vocab_size=VOCAB_SIZE, embedding_dim=50, dec_units=16, att_units=20)
all_outputs = decoder(decoder_input, x_gru_state, x_gru_out)
encoder_decoder = Model([encoder_input, decoder_input], outputs=all_outputs)
encoder_decoder.compile(optimizer='adam',loss='sparse_categorical_crossentropy')

x = np.random.randint(0, 499, size=(2000, ENCODER_SEQ_LEN))
y = np.random.randint(0, 499, size=(2000, DECODER_SEQ_LEN))

encoder_decoder.fit(x=[x,y], y=y, epochs=1,verbose=1,batch_size=32)

Error: TypeError Traceback (most recent call last) /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name) 60 op_name, inputs, attrs, ---> 61 num_outputs) 62 except core._NotOkStatusException as e:

TypeError: An op outside of the function building code is being passed a "Graph" tensor. It is possible to have Graph tensors leak out of the function building context by including a tf.init_scope in your function building code. For example, the following function will fail: @tf.function def has_init_scope(): my_constant = tf.constant(1.) with tf.init_scope(): added = my_constant * 2 The graph tensor has name: keras_learning_phase:0

During handling of the above exception, another exception occurred:

_SymbolicException Traceback (most recent call last) 11 frames /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name) 73 raise core._SymbolicException( 74 "Inputs to eager execution function cannot be Keras symbolic " ---> 75 "tensors, but found {}".format(keras_symbolic_tensors)) 76 raise e 77 # pylint: enable=protected-access

_SymbolicException: Inputs to eager execution function cannot be Keras symbolic tensors, but found []

Answer 1

It is, as you noticed, about the new version that changed the backend, now using something called eager execution.

The discussion on GitHub has some solutions though. Some suggested it is about incompatibility between the layers. But as the title of the issue implies it is a pain in the neck.

What may solve the problem is disabling it, which works in TF 2.1, but not in 2.3.

model.compile(..., experimental_run_tf_function=False)

The following may work in newer versions:

tf.config.experimental_run_functions_eagerly(True)

or you can try this one:

import tensorflow as tf
tf.compat.v1.disable_eager_execution()

None worked for me, so I changed the backend to theano.