2
$\begingroup$

I'm new in tensorflow and machine learning. Could you explain me how implement deconvulation on this CNN model(http://www.wildml.com/2015/12/implementing-a-cnn-for-text-classification-in-tensorflow/) for text classification?

CNN model:

class TextCNN(object):
"""
A CNN for text classification.
Uses an embedding layer, followed by a convolutional, max-pooling and softmax layer.
"""
def __init__(
  self, sequence_length, num_classes, vocab_size,
  embedding_size, filter_sizes, num_filters, l2_reg_lambda=0.0):

    # Placeholders for input, output and dropout
    self.input_x = tf.placeholder(tf.int32, [None, sequence_length], name="input_x")
    self.input_y = tf.placeholder(tf.float32, [None, num_classes], name="input_y")
    self.dropout_keep_prob = tf.placeholder(tf.float32, name="dropout_keep_prob")

    # Keeping track of l2 regularization loss (optional)
    l2_loss = tf.constant(0.0)

    # Embedding layer
    with tf.device('/cpu:0'), tf.name_scope("embedding"):
        self.W = tf.Variable(
            tf.random_uniform([vocab_size, embedding_size], -1.0, 1.0),
            name="W")
        self.embedded_chars = tf.nn.embedding_lookup(self.W, self.input_x)
        self.embedded_chars_expanded = tf.expand_dims(self.embedded_chars, -1)

    # Create a convolution + maxpool layer for each filter size
    pooled_outputs = []
    for i, filter_size in enumerate(filter_sizes):
        with tf.name_scope("conv-maxpool-%s" % filter_size):
            # Convolution Layer
            filter_shape = [filter_size, embedding_size, 1, num_filters]
            W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W")
            b = tf.Variable(tf.constant(0.1, shape=[num_filters]), name="b")
            conv = tf.nn.conv2d(
                self.embedded_chars_expanded,
                W,
                strides=[1, 1, 1, 1],
                padding="VALID",
                name="conv")
            # Apply nonlinearity
            h = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu")
            # Maxpooling over the outputs
            pooled = tf.nn.max_pool(
                h,
                ksize=[1, sequence_length - filter_size + 1, 1, 1],
                strides=[1, 1, 1, 1],
                padding='VALID',
                name="pool")
            pooled_outputs.append(pooled)

    # Combine all the pooled features
    num_filters_total = num_filters * len(filter_sizes)
    self.h_pool = tf.concat(pooled_outputs, 3)
    self.h_pool_flat = tf.reshape(self.h_pool, [-1, num_filters_total])

    # Add dropout
    with tf.name_scope("dropout"):
        self.h_drop = tf.nn.dropout(self.h_pool_flat, self.dropout_keep_prob)

    # Final (unnormalized) scores and predictions
    with tf.name_scope("output"):
        W = tf.get_variable(
            "W",
            shape=[num_filters_total, num_classes],
            initializer=tf.contrib.layers.xavier_initializer())
        b = tf.Variable(tf.constant(0.1, shape=[num_classes]), name="b")
        l2_loss += tf.nn.l2_loss(W)
        l2_loss += tf.nn.l2_loss(b)
        self.scores = tf.nn.xw_plus_b(self.h_drop, W, b, name="scores")
        self.predictions = tf.argmax(self.scores, 1, name="predictions")

    # CalculateMean cross-entropy loss
    with tf.name_scope("loss"):
        losses = tf.nn.softmax_cross_entropy_with_logits(logits=self.scores, labels=self.input_y)
        self.loss = tf.reduce_mean(losses) + l2_reg_lambda * l2_loss

    # Accuracy
    with tf.name_scope("accuracy"):
        correct_predictions = tf.equal(self.predictions, tf.argmax(self.input_y, 1))
        self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy")

I know about tf.nn.conv2d_transpose, but I don't understand how can I get original data (not embedded_chars)

$\endgroup$
4
  • $\begingroup$ The conv2d_transpose is nothing but the gradient of the conv2d. So if u want to get original data, calculate gradient of the embedded_chars w.r.t. original input. $\endgroup$ – Bhagyesh Vikani Mar 22 '17 at 3:22
  • $\begingroup$ @BhagyeshVikani , thank you for you comment. I'm confused, how gradient of the embedded_chars w.r.t. original input ((tf.gradients(embedded_chars, input_x)) can help me get original data? $\endgroup$ – Mihail Salnikov Mar 22 '17 at 16:36
  • $\begingroup$ Yes, tf.gradients(embedded_chars, input_x) can help get original data. You can also do this: tf.gradients(layer_output_tensor, input_x). This will give you direct original data from any given layer output tensor. $\endgroup$ – Bhagyesh Vikani Mar 22 '17 at 16:46
  • $\begingroup$ is P in unpooling step , the output of pooled layer ? can you explain how unpooling is happening ? $\endgroup$ – rakeshKM Mar 12 '18 at 11:16
1
$\begingroup$

Deconvolution have very simple structure: unpooling → deconv like this:

# Unpooling
Ps = (tf.gradients(pooled, h))[0]
unpooled = tf.multiply(Ps, P)

# Deconv
batch_size = tf.shape(self.input_x)[0]
ds = [batch_size]
ds.append(self.embedded_chars_expanded.get_shape()[1])
ds.append(self.embedded_chars_expanded.get_shape()[2])
ds.append(self.embedded_chars_expanded.get_shape()[3])
deconv_shape = tf.stack(ds)
deconv = tf.nn.conv2d_transpose(
    unpooled,
    W,
    deconv_shape,
    strides=[1, 1, 1, 1],
    padding='VALID',
    name="Deconv"
    )
$\endgroup$
1
  • $\begingroup$ What is P in your solution here ? $\endgroup$ – Yasmin May 1 '19 at 17:39

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.