0
$\begingroup$

I have been trying to write a generator for DistillBertFast model

## Generator
def _generator(text=train_texts, label=Y_oh_train, batch_size=1):
# label = tf.ragged.constant(label)
while True:
    for i in range(0,len(text),batch_size):
        yield dict(tokenizer(text[i:i+batch_size], truncation=True, padding=True, return_tensors='tf')), label[i:i+batch_size]

## tf Dataset
train_dataset = tf.data.Dataset.from_generator(_generator, output_types=({'input_ids':tf.int32,
                                                                      'attention_mask':tf.int32}, tf.float32))


## model compile

    loss_fn=tf.keras.losses.CategoricalCrossentropy(from_logits=True)
model.compile(
    optimizer=tf.keras.optimizers.Adam(learning_rate=0.1),
    loss=loss_fn,
    metrics=[tf.keras.metrics.categorical_accuracy])

## sample data
train_texts = ['This gif kills me Death is literally gushing towards you and you really gon do a whole 3point turn', 'LOVE TEST Raw Real JaDine', 'We would like to wish everyone a very Happy New Year and all the best in 2018']

Y_oh_train=array([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.,
    0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
    0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
    0.],
   [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
    0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,
    0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
    0.],
   [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
    1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
    0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
    0.]])

But when I try to fit the model it gives error:

    ValueError                                Traceback (most recent call last)
<ipython-input-195-05df82e86e2e> in <module>()
      4     loss=loss_fn,
      5     metrics=[tf.keras.metrics.categorical_accuracy])
----> 6 model.fit(t)

9 frames
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
   1181                 _r=1):
   1182               callbacks.on_train_batch_begin(step)
-> 1183               tmp_logs = self.train_function(iterator)
   1184               if data_handler.should_sync:
   1185                 context.async_wait()

/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py in __call__(self, *args, **kwds)
    887 
    888       with OptionalXlaContext(self._jit_compile):
--> 889         result = self._call(*args, **kwds)
    890 
    891       new_tracing_count = self.experimental_get_tracing_count()

/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py in _call(self, *args, **kwds)
    931       # This is the first call of __call__, so we have to initialize.
    932       initializers = []
--> 933       self._initialize(args, kwds, add_initializers_to=initializers)
    934     finally:
    935       # At this point we know that the initialization is complete (or less

/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py in _initialize(self, args, kwds, add_initializers_to)
    762     self._concrete_stateful_fn = (
    763         self._stateful_fn._get_concrete_function_internal_garbage_collected(  # pylint: disable=protected-access
--> 764             *args, **kwds))
    765 
    766     def invalid_creator_scope(*unused_args, **unused_kwds):

/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py in _get_concrete_function_internal_garbage_collected(self, *args, **kwargs)
   3048       args, kwargs = None, None
   3049     with self._lock:
-> 3050       graph_function, _ = self._maybe_define_function(args, kwargs)
   3051     return graph_function
   3052 

/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py in _maybe_define_function(self, args, kwargs)
   3442 
   3443           self._function_cache.missed.add(call_context_key)
-> 3444           graph_function = self._create_graph_function(args, kwargs)
   3445           self._function_cache.primary[cache_key] = graph_function
   3446 

/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py in _create_graph_function(self, args, kwargs, override_flat_arg_shapes)
   3287             arg_names=arg_names,
   3288             override_flat_arg_shapes=override_flat_arg_shapes,
-> 3289             capture_by_value=self._capture_by_value),
   3290         self._function_attributes,
   3291         function_spec=self.function_spec,

/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/func_graph.py in func_graph_from_py_func(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes)
    997         _, original_func = tf_decorator.unwrap(python_func)
    998 
--> 999       func_outputs = python_func(*func_args, **func_kwargs)
   1000 
   1001       # invariant: `func_outputs` contains only Tensors, CompositeTensors,

/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py in wrapped_fn(*args, **kwds)
    670         # the function a weak reference to itself to avoid a reference cycle.
    671         with OptionalXlaContext(compile_with_xla):
--> 672           out = weak_wrapped_fn().__wrapped__(*args, **kwds)
    673         return out
    674 

/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)
    984           except Exception as e:  # pylint:disable=broad-except
    985             if hasattr(e, "ag_error_metadata"):
--> 986               raise e.ag_error_metadata.to_exception(e)
    987             else:
    988               raise

ValueError: in user code:

    /usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:855 train_function  *
        return step_function(self, iterator)
    /usr/local/lib/python3.7/dist-packages/transformers/models/distilbert/modeling_tf_distilbert.py:800 call  *
        distilbert_output = self.distilbert(
    /usr/local/lib/python3.7/dist-packages/transformers/models/distilbert/modeling_tf_distilbert.py:415 call  *
        embedding_output = self.embeddings(
    /usr/local/lib/python3.7/dist-packages/transformers/models/distilbert/modeling_tf_distilbert.py:122 call  *
        final_embeddings = self.LayerNorm(inputs=final_embeddings)
    /usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/base_layer.py:1030 __call__  **
        outputs = call_fn(inputs, *args, **kwargs)
    /usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/layers/normalization.py:1218 call
        ndims = len(input_shape)
    /usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py:855 __len__
        raise ValueError("Cannot take the length of shape with unknown rank.")

ValueError: Cannot take the length of shape with unknown rank.

I have been trying to find a work around, I can't put in a fixed tensor shape in generator because I can't control the shape of output from generator, it'd be based on the max length on each call, I can't load all the data at once, since the data is too huge to be loaded in memory

$\endgroup$

1 Answer 1

0
$\begingroup$

I was able to write a generator but had to mention the output shape

def padded_tokenizer(text, max_len=80):
    result = dict(tokenizer(text, truncation=True, padding=True, return_tensors='tf', pad_to_multiple_of=2))
    for key in result.keys():
        result[key] = tf.keras.preprocessing.sequence.pad_sequences(result[key], padding="post", maxlen=max_len)
    return result

def _generator(arg=0, batch_size=1):
    if arg==0:
        text=train_texts
        label=Y_oh_train
    if arg==1:
        text=val_texts
        label=Y_oh_val
    else:
        text=test_texts
        label=Y_oh_test
    # batch_size=1
    # label = tf.ragged.constant(label)
    while True:
        for i in range(0,len(text),batch_size):
            yield padded_tokenizer(text[i:i+batch_size]), label[i:i+batch_size]

train_dataset = tf.data.Dataset.from_generator(_generator, output_signature=({'input_ids':tf.TensorSpec(shape=(None,80), dtype=tf.int32),
                                                                          'attention_mask':tf.TensorSpec(shape=(None,80), dtype=tf.int32)},
                                                                          tf.TensorSpec(shape=(None,49), dtype=tf.int32)), 
                                               args=([0,16])
                                               )

It seems the signature has to be mentioned if output is generated from a generator, but the speed is too slow, compared to loading everything in memory(for obv reasons)

Still would appreciate a faster solution

$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.