1
$\begingroup$

I know this question has answers, but they are not working.

why this fails ? (and why tensorflow has troubles in reading numpy!!)

def data_generator(df):
    for index, row in df.iterrows():
        audio = row['audio']  # Replace 'audio' with the appropriate column name from your DataFrame
        audio = np.asarray(audio).astype(np.float32)
        audio_tensor = tf.convert_to_tensor(audio, dtype=tf.float32)
        yield audio_tensor  # this is a tensor for sure

and

list(data_generator(sample_data));  # it is fine! the list is a list of TF tensors

but the generator function, when passed on the from_generator, will fail:

dataset = tf.data.Dataset.from_generator(data_generator, args=([sample_data]), output_signature=tf.TensorSpec(shape=(), dtype=tf.float32)) # sample_data is my dataframe

Error:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[219], line 3
      1 list(data_generator(sample_data));
      2 # Create a tf.data.Dataset from the data generator
----> 3 dataset = tf.data.Dataset.from_generator(data_generator, args=([sample_data]), output_signature=tf.TensorSpec(shape=(), dtype=tf.float32))

File ~/miniconda3/lib/python3.10/site-packages/tensorflow/python/util/deprecation.py:576, in deprecated_args.<locals>.deprecated_wrapper.<locals>.new_func(*args, **kwargs)
    568         _PRINTED_WARNING[(func, arg_name)] = True
    569       logging.warning(
    570           'From %s: calling %s (from %s) with %s is deprecated and will '
    571           'be removed %s.\nInstructions for updating:\n%s',
   (...)
    574           'in a future version' if date is None else ('after %s' % date),
    575           instructions)
--> 576 return func(*args, **kwargs)

File ~/miniconda3/lib/python3.10/site-packages/tensorflow/python/data/ops/dataset_ops.py:970, in DatasetV2.from_generator(generator, output_types, output_shapes, args, output_signature, name)
    966 # Loaded lazily due to a circular dependency (dataset_ops ->
    967 # from_generator_op -> dataset_ops).
    968 # pylint: disable=g-import-not-at-top,protected-access
    969 from tensorflow.python.data.ops import from_generator_op
--> 970 return from_generator_op._from_generator(generator, output_types,
    971                                          output_shapes, args,
    972                                          output_signature, name)

File ~/miniconda3/lib/python3.10/site-packages/tensorflow/python/data/ops/from_generator_op.py:155, in _from_generator(generator, output_types, output_shapes, args, output_signature, name)
    153   args = ()
    154 else:
--> 155   args = tuple(ops.convert_n_to_tensor(args, name="args"))
    157 generator_state = dataset_ops.DatasetV2._GeneratorState(generator)  # pylint: disable=protected-access
    159 def get_iterator_id_fn(unused_dummy):

File ~/miniconda3/lib/python3.10/site-packages/tensorflow/python/framework/ops.py:1742, in convert_n_to_tensor(values, dtype, name, preferred_dtype)
   1719 def convert_n_to_tensor(values, dtype=None, name=None, preferred_dtype=None):
   1720   """Converts `values` to a list of `Tensor` objects.
   1721 
   1722   Args:
   (...)
   1740       value.
   1741   """
-> 1742   return internal_convert_n_to_tensor(
   1743       values=values,
   1744       dtype=dtype,
   1745       name=name,
   1746       preferred_dtype=preferred_dtype,
   1747       as_ref=False)

File ~/miniconda3/lib/python3.10/site-packages/tensorflow/python/framework/ops.py:1709, in internal_convert_n_to_tensor(values, dtype, name, as_ref, preferred_dtype, ctx)
   1706 for i, value in enumerate(values):
   1707   n = None if name is None else "%s_%d" % (name, i)
   1708   ret.append(
-> 1709       convert_to_tensor(
   1710           value,
   1711           dtype=dtype,
   1712           name=n,
   1713           as_ref=as_ref,
   1714           preferred_dtype=preferred_dtype,
   1715           ctx=ctx))
   1716 return ret

File ~/miniconda3/lib/python3.10/site-packages/tensorflow/python/profiler/trace.py:183, in trace_wrapper.<locals>.inner_wrapper.<locals>.wrapped(*args, **kwargs)
    181   with Trace(trace_name, **trace_kwargs):
    182     return func(*args, **kwargs)
--> 183 return func(*args, **kwargs)

File ~/miniconda3/lib/python3.10/site-packages/tensorflow/python/framework/ops.py:1642, in convert_to_tensor(value, dtype, name, as_ref, preferred_dtype, dtype_hint, ctx, accepted_result_types)
   1633       raise RuntimeError(
   1634           _add_error_prefix(
   1635               f"Conversion function {conversion_func!r} for type "
   (...)
   1638               f"actual = {ret.dtype.base_dtype.name}",
   1639               name=name))
   1641 if ret is None:
-> 1642   ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
   1644 if ret is NotImplemented:
   1645   continue

File ~/miniconda3/lib/python3.10/site-packages/tensorflow/python/framework/constant_op.py:344, in _constant_tensor_conversion_function(v, dtype, name, as_ref)
    341 def _constant_tensor_conversion_function(v, dtype=None, name=None,
    342                                          as_ref=False):
    343   _ = as_ref
--> 344   return constant(v, dtype=dtype, name=name)

File ~/miniconda3/lib/python3.10/site-packages/tensorflow/python/framework/constant_op.py:268, in constant(value, dtype, shape, name)
    171 @tf_export("constant", v1=[])
    172 def constant(value, dtype=None, shape=None, name="Const"):
    173   """Creates a constant tensor from a tensor-like object.
    174 
    175   Note: All eager `tf.Tensor` values are immutable (in contrast to
   (...)
    266     ValueError: if called on a symbolic tensor.
    267   """
--> 268   return _constant_impl(value, dtype, shape, name, verify_shape=False,
    269                         allow_broadcast=True)

File ~/miniconda3/lib/python3.10/site-packages/tensorflow/python/framework/constant_op.py:280, in _constant_impl(value, dtype, shape, name, verify_shape, allow_broadcast)
    278     with trace.Trace("tf.constant"):
    279       return _constant_eager_impl(ctx, value, dtype, shape, verify_shape)
--> 280   return _constant_eager_impl(ctx, value, dtype, shape, verify_shape)
    282 g = ops.get_default_graph()
    283 tensor_value = attr_value_pb2.AttrValue()

File ~/miniconda3/lib/python3.10/site-packages/tensorflow/python/framework/constant_op.py:305, in _constant_eager_impl(ctx, value, dtype, shape, verify_shape)
    303 def _constant_eager_impl(ctx, value, dtype, shape, verify_shape):
    304   """Creates a constant on the current device."""
--> 305   t = convert_to_eager_tensor(value, ctx, dtype)
    306   if shape is None:
    307     return t

File ~/miniconda3/lib/python3.10/site-packages/tensorflow/python/framework/constant_op.py:103, in convert_to_eager_tensor(value, ctx, dtype)
    101     dtype = dtypes.as_dtype(dtype).as_datatype_enum
    102 ctx.ensure_initialized()
--> 103 return ops.EagerTensor(value, ctx.device_name, dtype)

ValueError: Failed to convert a NumPy array to a Tensor (Unsupported object type float).

Where does that error come from ?

It seems on the conversion function, but it worked on the generator. So I m lost. Is it a TF bug ?

NB:

this would work:

tensor_list = [tf.convert_to_tensor(row['audio'], dtype=tf.float32) for _, row in sample_data.iterrows()]
dataset = tf.data.Dataset.from_tensor_slices(tensor_list)

But I want to use a generator to reduce memory pressure.

$\endgroup$
3
  • $\begingroup$ Did you ever find a solution? I'm having the exact same problem. $\endgroup$
    – khm
    Nov 22, 2023 at 4:42
  • 1
    $\begingroup$ hi @khm, I did but unfortunately i don't recall what i did and do not have the code with me to check. however, i found great benefit in using chatgpt or alike to debug the error. if i recall well, you may want to specify the shape in the generator output_signature=tf.TensorSpec(shape=(), so that he knows the dimension of the tensor he has to transform the numpy vector from. it might be that it read a numpy value as a float, and so it fails. Try to tell it which is the dimension of the vector output. $\endgroup$
    – user305883
    Nov 30, 2023 at 9:49
  • $\begingroup$ @user305883 The 'use chatGPT' answer you provided worked perfectly for me. There was a typo on my code that generated the error, and chatGPT pointed this out for me. $\endgroup$ Jan 26 at 13:58

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Browse other questions tagged or ask your own question.