Hi everyone, I found a interesting thing but I don't know the reason.
When I restore a CNN network with BatchNormLayer from npz file, the accuracy is slightly different, my code as attached. Hope someone can help me, thanks in advance.

• Here is my result:
• Result of training is_test_only = False: (note: I set n_epoch=1 and use only a small part of trainig data for fast debugging.)

Epoch 1 of 1 took 28.871696s
   train loss: 0.823357
   train acc: 0.757412
   val loss: 0.804865
   val acc: 0.768029
Evaluation
   test loss: 0.829894
   test acc: 0.760116

• Result of restoring the network from npz file, is_test_only = True:

Evaluation
   test loss: 0.832265
   test acc: 0.760917

• @sczhengyabin I saw you set variables = tf.GraphKeys.GLOBAL_VARIABLES in BatchNormLayer (layers.py line 1825), but I found it will get 8 parameters ... are you sure it is correct? I tried the following setting, but the accuracy still has slightly difference ...

if variables = tf.GraphKeys.GLOBAL_VARIABLES, it has 8 variables <- @sczhengyabin using this.
if variables = tf.GraphKeys.TRAINABLE_VARIABLES, it has 2 variables (gamma, beta)   
or variables = [beta, gamma, moving_mean, moving_variance] ? 

• @boscotsang as I discuss with you in pull/42, the testing and training cost are all drop normally, but I really don't understand why the accuracies are different after restoring and what variables should be included in the BatchNormLayer.

• My code
  environment: TensorFlow 0.12 and TensorLayer 1.3.0

#! /usr/bin/python
# -*- coding: utf8 -*-

import numpy as np
import tensorflow as tf
import tensorlayer as tl
from tensorlayer.layers import set_keep
import time

is_test_only = False # if True, restore and test without training

X_train, y_train, X_val, y_val, X_test, y_test = \
                tl.files.load_mnist_dataset(shape=(-1, 28, 28, 1))

X_train = np.asarray(X_train, dtype=np.float32)[0:10000]#<-- small training set for fast debugging
y_train = np.asarray(y_train, dtype=np.int64)[0:10000]
X_val = np.asarray(X_val, dtype=np.float32)
y_val = np.asarray(y_val, dtype=np.int64)
X_test = np.asarray(X_test, dtype=np.float32)
y_test = np.asarray(y_test, dtype=np.int64)

print('X_train.shape', X_train.shape)
print('y_train.shape', y_train.shape)
print('X_val.shape', X_val.shape)
print('y_val.shape', y_val.shape)
print('X_test.shape', X_test.shape)
print('y_test.shape', y_test.shape)
print('X %s   y %s' % (X_test.dtype, y_test.dtype))

sess = tf.InteractiveSession()

batch_size = 128
x = tf.placeholder(tf.float32, shape=[batch_size, 28, 28, 1])
y_ = tf.placeholder(tf.int64, shape=[batch_size,])

def inference(x, is_train, reuse=None):
    gamma_init = tf.random_normal_initializer(1., 0.02)
    with tf.variable_scope("CNN", reuse=reuse):
        tl.layers.set_name_reuse(reuse)
        network = tl.layers.InputLayer(x, name='input_layer')
        network = tl.layers.Conv2d(network, n_filter=32, filter_size=(5, 5), strides=(1, 1),
                act=None, b_init=None, padding='SAME', name='cnn_layer1')
        network = tl.layers.BatchNormLayer(network, act=tf.nn.relu,
                gamma_init=gamma_init, is_train=is_train, name='batch1')

        network = tl.layers.MaxPool2d(network, filter_size=(2, 2), strides=(2, 2),
                padding='SAME', name='pool_layer1')
        network = tl.layers.Conv2d(network, n_filter=64, filter_size=(5, 5), strides=(1, 1),
                act=None, b_init=None, padding='SAME', name='cnn_layer2')
        network = tl.layers.BatchNormLayer(network, act=tf.nn.relu,
                gamma_init=gamma_init, is_train=is_train, name='batch2')

        network = tl.layers.MaxPool2d(network, filter_size=(2, 2), strides=(2, 2),
                padding='SAME', name='pool_layer2')
        ## end of conv
        network = tl.layers.FlattenLayer(network, name='flatten_layer')
        if is_train:
            network = tl.layers.DropoutLayer(network, keep=0.5, is_fix=True, name='drop1')
        network = tl.layers.DenseLayer(network, n_units=256,
                                        act = tf.nn.relu, name='relu1')
        if is_train:
            network = tl.layers.DropoutLayer(network, keep=0.5, is_fix=True, name='drop2')
        network = tl.layers.DenseLayer(network, n_units=10,
                                        act = tf.identity, name='output_layer')   
    return network


# train phase
network = inference(x, is_train=True, reuse=False)
y = network.outputs
cost = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(y, y_))
correct_prediction = tf.equal(tf.argmax(y, 1), y_)
acc = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

# test phase
network_test = inference(x, is_train=False, reuse=True)
y_t = network_test.outputs
cost_t = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(y_t, y_))
correct_prediction = tf.equal(tf.argmax(y_t, 1), y_)
acc_t = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))


# train
if is_test_only:
    n_epoch = 1
else:
    n_epoch = 1
learning_rate = 0.0001
print_freq = 1

train_params = network.all_params
train_op = tf.train.AdamOptimizer(learning_rate, beta1=0.9, beta2=0.999,
    epsilon=1e-08, use_locking=False).minimize(cost, var_list=train_params)

tl.layers.initialize_global_variables(sess)

if is_test_only:
    load_params = tl.files.load_npz(name='_model_test.npz')
    tl.files.assign_params(sess, load_params, network)

network.print_params(True)
network.print_layers()

print('   learning_rate: %f' % learning_rate)
print('   batch_size: %d' % batch_size)

if not is_test_only:
    for epoch in range(n_epoch):
        start_time = time.time()
        for X_train_a, y_train_a in tl.iterate.minibatches(
                                    X_train, y_train, batch_size, shuffle=True):
            sess.run(train_op, feed_dict={x: X_train_a, y_: y_train_a})

        if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
            print("Epoch %d of %d took %fs" % (epoch + 1, n_epoch, time.time() - start_time))
            train_loss, train_acc, n_batch = 0, 0, 0
            for X_train_a, y_train_a in tl.iterate.minibatches(
                                    X_train, y_train, batch_size, shuffle=True):
                err, ac = sess.run([cost_t, acc_t], feed_dict={x: X_train_a, y_: y_train_a})
                train_loss += err; train_acc += ac; n_batch += 1
            print("   train loss: %f" % (train_loss/ n_batch))
            print("   train acc: %f" % (train_acc/ n_batch))
            val_loss, val_acc, n_batch = 0, 0, 0
            for X_val_a, y_val_a in tl.iterate.minibatches(
                                        X_val, y_val, batch_size, shuffle=True):
                err, ac = sess.run([cost_t, acc_t], feed_dict={x: X_val_a, y_: y_val_a})
                val_loss += err; val_acc += ac; n_batch += 1
            print("   val loss: %f" % (val_loss/ n_batch))
            print("   val acc: %f" % (val_acc/ n_batch))

print('Evaluation')
test_loss, test_acc, n_batch = 0, 0, 0
for X_test_a, y_test_a in tl.iterate.minibatches(
                            X_test, y_test, batch_size, shuffle=False):
    err, ac = sess.run([cost_t, acc_t], feed_dict={x: X_test_a, y_: y_test_a})
    test_loss += err; test_acc += ac; n_batch += 1
print("   test loss: %f" % (test_loss/n_batch))
print("   test acc: %f" % (test_acc/n_batch))

network.print_params(True)

tl.files.save_npz(network.all_params, name='_model_test.npz', sess=sess)

when I use tf.gradient to calculate gradient grads = tf.gradients(loss, var_list), var_list=tf.trainable_variables(), the error LookupError: No gradient defined for operation 'gradients_1/continous_shared/continous_pool1_grad/MaxPoolGrad' (op type: MaxPoolGrad) shows.

When state_is_tuple, we need to feed the cell and hidden state seperately, the code will look like below. Is there any simple way to simplify the code?

# reset all states at the begining of every epoch
state1 = tl.layers.initialize_rnn_state(lstm1.initial_state)
state2 = tl.layers.initialize_rnn_state(lstm2.initial_state)
for step, (x, y) in enumerate(tl.iterate.ptb_iterator(train_data,
                                                    batch_size, num_steps)):
            feed_dict = {input_data: x, targets: y,
                        lstm1.initial_state.c: state1[0],
                        lstm1.initial_state.h: state1[1],
                        lstm2.initial_state.c: state2[0],
                        lstm2.initial_state.h: state2[1],
                        }
            # For training, enable dropout
            feed_dict.update( network.all_drop )
            _cost, state1_c, state1_h, state2_c, state2_h, _ = \
                                    sess.run([cost,
                                            lstm1.final_state.c,
                                            lstm1.final_state.h,
                                            lstm2.final_state.c,
                                            lstm2.final_state.h,
                                            train_op],
                                            feed_dict=feed_dict
                                            )
            state1 = (state1_c, state1_h)
            state2 = (state2_c, state2_h)

Dropout and Batch Normalization are two major structures that should behave differently in both training and test phases.

According to the API reference (http://tensorlayer.readthedocs.io/en/latest/modules/layers.html#dropout-layer) , switching train/test for the DropoutLayer is handled by set different data to feed_dict.
e.g.

# for training
feed_dict = {x: X_train_a, y_: y_train_a}
feed_dict.update( network.all_drop )     # enable noise layers

# for testing
dp_dict = tl.utils.dict_to_one( network.all_drop ) # disable noise layers
feed_dict = {x: X_val_a, y_: y_val_a}
feed_dict.update(dp_dict)

I couldn't find how to switch for BatchNormLayer in the tutorial or API reference but according to the DCGAN example (https://github.com/zsdonghao/dcgan), it creates two different networks.
For training phase, is_train=True is passed to BatchNormLayer and is_train=False for test phase.

I think this is confusing that switching method is not unified. Or, is there any standard way for batch norm?
For example, TFLearn switches training/test by tflearn.is_training method.

The input of each layer (e.g., Dense layer and Conv layer) except "Input layer" must be a Layer instance.
I think this behavior is not so transparent to TensorFlow.

In the case that I want to use an intermediate tf.Tensor as input for a Layer instance,
is there any existing method of TensorLayer to implement this function?

Thank you for providing such a powerful but user-friendly wrapper. Is there any example for resnet in which the network cannot be constructed by simply stacking layer by layer, by using tl.layer?

Seems some documentation is missing in the readthedocs site?

For example documentation for ConvolutionalLayer2D

The ReconLayer.pretrain() function has hard coded visualize.W() shape to be 28*28, so if we change the training data shape it will crash.

This shape should be drawn from the input data.

There seems to be a bug in the examples/tutorials_cifar10_tfrecords.py

Using tensorflow 0.12.1 I get the following error:
File "tutorial_cifar10_tfrecord.py", line 116, in read_and_decode
img = tf.image.per_image_whitening(img)
AttributeError: 'module' object has no attribute 'per_image_whitening'
[deepmind@deepmind tensorlayer]$

As you mention we can read the docs on docs/_build/html/index.html, but I can't find the index.html, how should I do?

In latest tensorflow api, tf.cond is available while there is no tf.python.control_flow_ops.cond. I'm not sure that if the tf.cond is available and tf.python.control_flow_ops.cond behave well in tensorflow 0.12. I'm wondering if there is a need to change the tf.python.control_flow_ops.cond to tf.cond. What's your opinion?

Training model distributely with replicated models is critical for large scale deep learning training. I'm wondering whether tensorlayer support it and whether there is any tutorials for it?

I want to use PReluLayer in my network, but an error happened as follows:
ValueError: Variable g_/alphas already exists, disallowed. Did you mean to set reuse=True in VarScope?

alphas is the parameter in PReluLayer.

Anyone has this problem?

Tensorflow supports multi-gpu training, has tensorlayer supported multi-gpu training already?

Actually, this error make a lot of sense, since the variable W should be filters.

                filters = tf.get_variable(name='filter', shape=shape, initializer=W_init, **W_init_args )
            if b_init:
                b = tf.get_variable(name='b', shape=(n_filter), initializer=b_init, **b_init_args )
                self.outputs = act(tf.nn.atrous_conv2d(self.inputs, filters, rate, padding) + b)
            else:
                self.outputs = act(tf.nn.atrous_conv2d(self.inputs, filters, rate, padding))
        self.all_layers = list(layer.all_layers)
        self.all_params = list(layer.all_params)
        self.all_drop = dict(layer.all_drop)
        self.all_layers.extend( [self.outputs] )
        if b_init:
            self.all_params.extend( [W, b] )
        else:
            self.all_params.extend( [W] )

Some deep learning task, such as SSD and neural-style may need to extract previous layers. While it seems that we have to explicitly store those certain layers in v1.3.0, instead of getting it by name like network[conv5_3]. But I found that we have a self.all_layers Attribute in Layer, update in a new layer like:

self.all_layers = list(layer.all_layers)
...
self.all_layers.extend( [self.outputs] )

should we simply turn this list into dict: self.all_layers['conv5_3'] = self.output ? Then we can get layer output by name network.all_layers[conv5_3].
Or we already have another way to access every output?

Start to implement different kinds of Seq2Seq layers.

see http://tensorlayer.readthedocs.io/en/latest/modules/layers.html#sequence-to-sequence

Is there some simple way of initializing a Deconv2D layer with Bilinear weights?

I could write my own initalizer, but seems like a common thing to do.
Would it make sense to add it as a library feature?

PS
Great work with TensorLayer! :)

In Keras, I just use

callbacks=[Tensorboard(log_dir)='log']

in model.fit(), and it can generate training loss in tensorboard automatically. I wonder how can I visualize the training loss via tensorboard in tensorlayer?

Many to one, like this, but i don't know how to set 'max_grad_norm' ? what's the meaning of 'max_grad_norm'? how to set 'max_grad_norm'?

network = tl.layers.DynamicRNNLayer(network,
                        cell_fn=tf.nn.rnn_cell.BasicLSTMCell,
                        cell_init_args={'forget_bias': 0.0, 'state_is_tuple': True},
                        n_hidden=hidden_size,
                        n_layer=2,
                        dropout=0.5,
                        initializer=tf.random_uniform_initializer(-init_scale, init_scale),
                        sequence_length = tl.layers.retrieve_seq_length_op2(inputs),
                        return_last=True,
                        name='dynamic_lstm_layer')
lstm = network

Dear all:

I find the tensor layer is resemble the caffe structure, both are based model block.
I just wonder if there is a way to load caffe model and initialize tensor layer model?

comments and suggestions are welcome

best regards

When I import a slim model, e.g. vgg16, network.all_drop shows {}, but there should be some dropout layers in the model. Is this a bug?

Thanks a lot!

Tensorboard is really powerful in tracking the training process. Currently, does tensorlayer support summaries?
E.g. with the netwrok stacked by several tensorlayers , I want to use histogram_summary to view the last layer's activation and use scalar_summary to record the sparsity rate of last layer's activation by tf.nn.zero_fraction.
I tried the following code in a wish to achieve the above purpose, but failed.

tf.histogram_summary(tensor_name + '_activations', network.output) tf.scalar_summary(tensor_name + '_sparsity', tf.nn.zero_fraction(network.output))

If there is any way to track the summaries with tensorlayer, please notify me. And if there is none, could you add this support in tensorlayer?

Can the tensorlayer support multi-input and multi-output models ?
Is there any multi-input demo or example?

tensorlayer Supports BIRNN with batch normal????

Hi Guys, I have trained the translation model using python tuaorial_translate.py, then i want using my own words to test the model,so i switch the main_train() to main_decode(), but the python shell raise one exception; more information as follow:
first, the exception occurs at 391 lines of tutorial_translate.py if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path)；and the ckpt object is not null, ckpt.model_checkpoint_path value is 'wmt/model_translate_enfr.ckpt-10'; in fact, the directory does not have this file , i wonder why the pre-sentence ckpt = tf.train.get_checkpoint_state(train_dir) return the error data, hope your help, please! thanks a lot

I am a new learner for python and Tensorflow. So I choose TensorLayer as my tool. After install TensorLayer, I run the examples for mnist dataset. The CNN model (main_test_cnn_layer) runs very well and very fast. But an error occurs when I run the 'main_test_stacked_denoise_AE '. The traceback lists as follows.

File "", line 1, in
runfile('/home/feng/DeepLearning/MultiLabel/try_tensorlayer_mnist_SDAE.py', wdir='/home/feng/DeepLearning/MultiLabel')

File "/home/feng/anaconda2/lib/python2.7/site-packages/spyderlib/widgets/externalshell/sitecustomize.py", line 714, in runfile
execfile(filename, namespace)

File "/home/feng/anaconda2/lib/python2.7/site-packages/spyderlib/widgets/externalshell/sitecustomize.py", line 81, in execfile
builtins.execfile(filename, *where)

File "/home/feng/DeepLearning/MultiLabel/try_tensorlayer_mnist_SDAE.py", line 185, in
main_test_stacked_denoise_AE(model='relu') # model = relu, sigmoid

File "/home/feng/DeepLearning/MultiLabel/try_tensorlayer_mnist_SDAE.py", line 55, in main_test_stacked_denoise_AE
act = act_recon, name='recon_layer1')

File "/home/feng/anaconda2/lib/python2.7/site-packages/tensorlayer/layers.py", line 661, in init
ce = cost.cross_entropy(y, x_recon)

File "/home/feng/anaconda2/lib/python2.7/site-packages/tensorlayer/cost.py", line 38, in cross_entropy
return tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(output, target))

File "/home/feng/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/nn_ops.py", line 560, in sparse_softmax_cross_entropy_with_logits
labels_static_shape.ndims, logits.get_shape().ndims)

ValueError: ('Rank mismatch: Labels rank (received %s) should equal logits rank (received %s) - 1.', 2, 2)

The code is wrong here:
54 line recon_layer1 = tl.layers.ReconLayer(network, x_recon=x, n_units=784,
55 line act = act_recon, name='recon_layer1')

Can anyone guide me to fix this error? Thanks!

I'm trying to concat a cnn model and a nn model together. After concat them, I want to add a full-connected nn network.It always says error when I concat them. I don't know why.
TypeError: cannot convert dictionary update sequence element #0 to a sequence

For tensorflow 0.12 version, it seems that tensorlayer has an error on BatchNormLayer function. The error information as follow:

network = tl.layers.BatchNormLayer(network, is_train=True, name='batch_norm1')

File "D:\Program_Files\Anaconda3\lib\tensorlayer\layers.py", line 1725, in init
initializer=gamma_init)

File "D:\Program_Files\Anaconda3\lib\tensorlayer\layers.py", line 1707, in _get_variable
trainable=trainable)

File "D:\Program_Files\Anaconda3\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 1024, in get_variable
custom_getter=custom_getter)

File "D:\Program_Files\Anaconda3\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 850, in get_variable
custom_getter=custom_getter)

File "D:\Program_Files\Anaconda3\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 346, in get_variable
validate_shape=validate_shape)

File "D:\Program_Files\Anaconda3\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 331, in _true_getter
caching_device=caching_device, validate_shape=validate_shape)

File "D:\Program_Files\Anaconda3\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 677, in _get_single_variable
expected_shape=shape)

File "D:\Program_Files\Anaconda3\lib\site-packages\tensorflow\python\ops\variables.py", line 224, in init
expected_shape=expected_shape)

File "D:\Program_Files\Anaconda3\lib\site-packages\tensorflow\python\ops\variables.py", line 327, in _init_from_args
initial_value(), name="initial_value", dtype=dtype)

File "D:\Program_Files\Anaconda3\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 665, in
shape.as_list(), dtype=dtype, partition_info=partition_info)

TypeError: ones_initializer() got multiple values for argument 'dtype'

utils.py line 154 should be test_loss? val_loss is just copy paste error?

Now the load datasets functions in tl.files store the data files in current work directory and have no way to customize the path. I wonder if I can add path parameter to specify the data path. If no specification path, it is store in ~/.tensorlayer/datasets/. If you think so, I may do this work.

I am looking to implement "Unpooling" layers that upsamples an image and "undoes" a corresponding max-pooling layer. For example, used in SegNet type architecures.

Is there some good way of implementing this in TensorLayer?

when i run "tutorial_mnist_simple.py" in eclipse ,i have get a error message:

X_train, y_train, X_val, y_val, X_test, y_test =tl.files.load_mnist_dataset(shape=(-1,784))
AttributeError: module 'tensorlayer' has no attribute 'files'

when i run "tutorial_mnist_simple.py" in Pycharm, it will be ok.
but, my eclipse run tensorflow simples ,has no errors!

Is there an equivalent of a Lambda layer (from Keras) for TensorLayer?

when import tensorlayer:

Traceback (most recent call last):
File "", line 1, in
File "/usr/local/lib/python2.7/dist-packages/tensorlayer/init.py", line 19, in
from . import layers
File "/usr/local/lib/python2.7/dist-packages/tensorlayer/layers.py", line 1939, in
class RNNLayer(Layer):
File "/usr/local/lib/python2.7/dist-packages/tensorlayer/layers.py", line 2093, in RNNLayer
cell_fn = tf.nn.rnn_cell.BasicRNNCell,
AttributeError: 'module' object has no attribute 'rnn_cell'

python2.7, CUDA8.0, cudnn5.1, tensorflow_GPU0.12.0, ubuntu16.04
My tensorflow runs successful, how to solve this problem?

On windows pip install tensorflow puts imagenet_classes.py into site-packages/data.

This seems like a pretty generic package name to write into... Was this intended?

hi, @zsdonghao
in tensorlayer/example/tutorial_cifar10_tfrecord.py, there is
''
network.outputs = tf.nn.lrn(network.outputs, 4, bias=1.0, alpha=0.001 / 9.0,
beta=0.75, name='norm1')
''
the input is 'network.outputs', but other layer's input is 'network', for example
''
network = tl.layers.PoolLayer(network,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='SAME',
pool = tf.nn.max_pool,
name ='pool_layer2')
''
my question is that, what is the function of above sentence(network.outputs), and what is the difference between the two kinds inputs?
thank you !

I want to load the pretrained AlexNet model bvlc_alexnet.npy, and I know using np.load('bvlc_alexnet.npy') can get some numpy array of weights and bias. (Like net = np.load('bvlc_alexnet.npy'), net["conv1"][0] represent the weights of conv1) But there are some problems loading them into Conv2d layer.

Then while running, it gives error:
Variable cannot be callable.

Could someone give an example to load weights of pretrained model?By the way, I tried to install TensorLayer from git, while inputting command "pip install . -e", it said -e need an argument.. so how to install it properly if I want to modify some codes of it? Hope someone can help me. Thanks very much.

I am passing a batch of 1D vector as input and need a convolution 1D layer in the middle. However, when I do

tl.layers.Conv1dLayer(in_net, act=tf.nn.relu, shape=[5, 1, 50], strides=[1, 1, 1], padding='SAME', name='cnn_0')

where in_net is the inputlayer.

But I got an error message of

TypeError: Expected int for argument 'strides' not [1, 1, 1].

If I change to stride=1, I got error

ValueError: Shape must be rank 4 but is rank 3

Can someone help me out? Thanks.

looks like tensorlayer.visualize.W() always throws an error "total size of new array must be unchanged".

perhaps it is because of the line " feature = W[:,count-1] / np.sqrt( (W[:,count-1]**2).sum()) "
where if count==1, then it will be an empty list.

Hi, I am using TensorLayer with Tensorflow to accomplish some classification task.
Basically I use cross validation technique to gain some trustworthy results. The outline of my code is: first get different training and testing sets, then feed them into the model.
A very interesting question is that in TensorLayer, every layer requires a name, which is necessary or it will be an error. For me, I compacted the model thing into a whole function called model_training(), which takes inputs from every training/test sets.
Here comes the erros: in the function, I basically give every layer a name, say, "input_layer". For a single round of model training, this works fine. But for multiple rounds, it just says Use different name for different 'Layer'. It seems a little bizzar. I did initialized session every time. Here is the code:

And I get the following error:
Traceback (most recent call last): File "H:/projects/python/parkinson/main.py", line 115, in <module> train_model(x_train,x_test,y_train,y_test) File "H:/projects/python/parkinson/main.py", line 69, in train_model network = tl.layers.InputLayer(inputs=x,name='input_layer') File "C:\Users\jindongwang\Anaconda3\lib\site-packages\tensorlayer\layers.py", line 271, in __init__ Layer.__init__(self, inputs=inputs, name=name) File "C:\Users\jindongwang\Anaconda3\lib\site-packages\tensorlayer\layers.py", line 201, in __init__ \nHint : Use different name for different 'Layer' (The name is used to control parameter sharing)" % name) Exception: Layer 'input_layer' already exists, please choice other 'name'. Hint : Use different name for different 'Layer' (The name is used to control parameter sharing)

It seems after the first round of training, the errors occurs.
Is there any way to fix this?

I am working with conda environment. After installing tensorlayer, I tried to import it: import tensorlayer as tl, but got such an error:

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/yobichi/miniconda3/envs/dl/lib/python3.5/site-packages/tensorlayer/__init__.py", line 16, in <module>
    from . import files
  File "/Users/yobichi/miniconda3/envs/dl/lib/python3.5/site-packages/tensorlayer/files.py", line 13, in <module>
    from . import visualize
  File "/Users/yobichi/miniconda3/envs/dl/lib/python3.5/site-packages/tensorlayer/visualize.py", line 6, in <module>
    import matplotlib.pyplot as plt
  File "/Users/yobichi/miniconda3/envs/dl/lib/python3.5/site-packages/matplotlib/pyplot.py", line 115, in <module>
    _backend_mod, new_figure_manager, draw_if_interactive, _show = pylab_setup()
  File "/Users/yobichi/miniconda3/envs/dl/lib/python3.5/site-packages/matplotlib/backends/__init__.py", line 32, in pylab_setup
    globals(),locals(),[backend_name],0)
  File "/Users/yobichi/miniconda3/envs/dl/lib/python3.5/site-packages/matplotlib/backends/backend_qt5agg.py", line 16, in <module>
    from .backend_qt5 import QtCore
  File "/Users/yobichi/miniconda3/envs/dl/lib/python3.5/site-packages/matplotlib/backends/backend_qt5.py", line 26, in <module>
    import matplotlib.backends.qt_editor.figureoptions as figureoptions
  File "/Users/yobichi/miniconda3/envs/dl/lib/python3.5/site-packages/matplotlib/backends/qt_editor/figureoptions.py", line 20, in <module>
    import matplotlib.backends.qt_editor.formlayout as formlayout
  File "/Users/yobichi/miniconda3/envs/dl/lib/python3.5/site-packages/matplotlib/backends/qt_editor/formlayout.py", line 56, in <module>
    from matplotlib.backends.qt_compat import QtGui, QtWidgets, QtCore
  File "/Users/yobichi/miniconda3/envs/dl/lib/python3.5/site-packages/matplotlib/backends/qt_compat.py", line 175, in <module>
    "Matplotlib qt-based backends require an external PyQt4, PyQt5,\n"
ImportError: Matplotlib qt-based backends require an external PyQt4, PyQt5,
or PySide package to be installed, but it was not found.

This issue is not supposed to arise with conda. When I import matplotlib individually, it works well. I wonder how can I deal with this issue. Thanks in advance!

tensorlayer support deepbirnn layer????

how to use it??

multi birnn layer support???how to split input??

Code:

1. import tensorflow as tf
2. import tensorlayer as tl
3. import numpy as np
4. from PIL import Image
5. import os
8. def _int64_feature(value):

9. return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
   

12. def _bytes_feature(value):

13. return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
    

16. path_save = r'W:/PlantClassification/'
17. path_train = r"W:/PlantClassification/train/"
18. classes = [str(i) for i in list(range(7))]
19. writer = tf.python_io.TFRecordWriter(path_save + 'train.tfrecords')
20. for index, name in enumerate(classes):

21. class_path = path_train + name + '/'
    

22. for img_name in os.listdir(class_path):
    

23.     img_path = class_path + img_name
    

24.     img = Image.open(img_path)
    

25.     img = img.resize((227, 227))
    

26.     img_raw = img.tobytes()
    

27.     example = tf.train.Example(features=tf.train.Features(feature={
    

28.         'label': _int64_feature(index),
    

29.         'img_raw': _bytes_feature(img_raw)}))
    

30.     writer.write(example.SerializeToString())
    

31. writer.close()
34. def read_and_decode(filename):

35. filename_queue = tf.train.string_input_producer([filename])
    

36. reader = tf.TFRecordReader()
    

37. _, serialized_example = reader.read(filename_queue)
    

38. features = tf.parse_single_example(serialized_example,
    

39.                                    features={
    

40.                                        'label': tf.FixedLenFeature([], tf.int64),
    

41.                                        'img_raw': tf.FixedLenFeature([], tf.string),
    

42.                                    })
    

43. # You can do more image distortion here for training data
    

44. img = tf.decode_raw(features['img_raw'], tf.float32)
    

45. img = tf.reshape(img, [227, 227, 3])
    

46. label = tf.cast(features['label'], tf.int32)
    

47. return img, label
    

50. img, label = read_and_decode(r"W:\PlantClassification\train.tfrecords")
51. img_batch, label_batch = tf.train.shuffle_batch([img, label],

52.                                             batch_size=4,
    

53.                                             capacity=150000,
    

54.                                             min_after_dequeue=2000,
    

55.                                             num_threads=4)
    

58. init = tf.global_variables_initializer()
59. with tf.Session() as sess:

60. sess.run(init)
    

61. coord = tf.train.Coordinator()
    

62. threads = tf.train.start_queue_runners(sess=sess, coord=coord)
    

64. for i in range(3):
    

65.     print("Step %d" % i)
    

66.     val, l = sess.run([img_batch, label_batch])
    

67.     print(val.shape, l)
    

68.     tl.visualize.images2d(val, second=1, saveable=False, name='batch' + str(i), dtype=np.uint8, fig_idx=2020121)
    

70. coord.request_stop()
    

71. coord.join(threads)
    

72. sess.close()
    

Error:
S:\Python35\python.exe W:/PlantClassification/visualize_data.py
I c:\tf_jenkins\home\workspace\release-win\device\gpu\os\windows\tensorflow\stream_executor\dso_loader.cc:128] successfully opened CUDA library cublas64_80.dll locally
I c:\tf_jenkins\home\workspace\release-win\device\gpu\os\windows\tensorflow\stream_executor\dso_loader.cc:128] successfully opened CUDA library cudnn64_5.dll locally
I c:\tf_jenkins\home\workspace\release-win\device\gpu\os\windows\tensorflow\stream_executor\dso_loader.cc:128] successfully opened CUDA library cufft64_80.dll locally
I c:\tf_jenkins\home\workspace\release-win\device\gpu\os\windows\tensorflow\stream_executor\dso_loader.cc:128] successfully opened CUDA library nvcuda.dll locally
I c:\tf_jenkins\home\workspace\release-win\device\gpu\os\windows\tensorflow\stream_executor\dso_loader.cc:128] successfully opened CUDA library curand64_80.dll locally
img_batch : (4, 227, 227, 3)
label_batch : (4,)
I c:\tf_jenkins\home\workspace\release-win\device\gpu\os\windows\tensorflow\core\common_runtime\gpu\gpu_device.cc:885] Found device 0 with properties:
name: GeForce GTX 960
major: 5 minor: 2 memoryClockRate (GHz) 1.1775
pciBusID 0000:01:00.0
Total memory: 2.00GiB
Free memory: 1.64GiB
I c:\tf_jenkins\home\workspace\release-win\device\gpu\os\windows\tensorflow\core\common_runtime\gpu\gpu_device.cc:906] DMA: 0
I c:\tf_jenkins\home\workspace\release-win\device\gpu\os\windows\tensorflow\core\common_runtime\gpu\gpu_device.cc:916] 0: Y
I c:\tf_jenkins\home\workspace\release-win\device\gpu\os\windows\tensorflow\core\common_runtime\gpu\gpu_device.cc:975] Creating TensorFlow device (/gpu:0) -> (device: 0, name: GeForce GTX 960, pci bus id: 0000:01:00.0)
Step 0
W c:\tf_jenkins\home\workspace\release-win\device\gpu\os\windows\tensorflow\core\framework\op_kernel.cc:975] Invalid argument: Input to DecodeRaw has length 154587 that is not a multiple of 4, the size of float
[[Node: DecodeRaw = DecodeRawlittle_endian=true, out_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]]
W c:\tf_jenkins\home\workspace\release-win\device\gpu\os\windows\tensorflow\core\framework\op_kernel.cc:975] Invalid argument: Input to DecodeRaw has length 154587 that is not a multiple of 4, the size of float
[[Node: DecodeRaw = DecodeRawlittle_endian=true, out_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]]
W c:\tf_jenkins\home\workspace\release-win\device\gpu\os\windows\tensorflow\core\framework\op_kernel.cc:975] Invalid argument: Input to DecodeRaw has length 154587 that is not a multiple of 4, the size of float
[[Node: DecodeRaw = DecodeRawlittle_endian=true, out_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]]
W c:\tf_jenkins\home\workspace\release-win\device\gpu\os\windows\tensorflow\core\framework\op_kernel.cc:975] Invalid argument: Input to DecodeRaw has length 154587 that is not a multiple of 4, the size of float
[[Node: DecodeRaw = DecodeRawlittle_endian=true, out_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]]
Traceback (most recent call last):
File "S:\Python35\lib\site-packages\tensorflow\python\client\session.py", line 1021, in _do_call
return fn(*args)
File "S:\Python35\lib\site-packages\tensorflow\python\client\session.py", line 1003, in _run_fn
status, run_metadata)
File "S:\Python35\lib\contextlib.py", line 66, in exit
next(self.gen)
File "S:\Python35\lib\site-packages\tensorflow\python\framework\errors_impl.py", line 469, in raise_exception_on_not_ok_status
pywrap_tensorflow.TF_GetCode(status))
tensorflow.python.framework.errors_impl.OutOfRangeError: RandomShuffleQueue '_1_shuffle_batch/random_shuffle_queue' is closed and has insufficient elements (requested 4, current size 0)
[[Node: shuffle_batch = QueueDequeueMany[_class=["loc:@shuffle_batch/random_shuffle_queue"], component_types=[DT_FLOAT, DT_INT32], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](shuffle_batch/random_shuffle_queue, shuffle_batch/n)]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "W:/PlantClassification/visualize_data.py", line 39, in
val, l = sess.run([img_batch, label_batch])
File "S:\Python35\lib\site-packages\tensorflow\python\client\session.py", line 766, in run
run_metadata_ptr)
File "S:\Python35\lib\site-packages\tensorflow\python\client\session.py", line 964, in _run
feed_dict_string, options, run_metadata)
File "S:\Python35\lib\site-packages\tensorflow\python\client\session.py", line 1014, in _do_run
target_list, options, run_metadata)
File "S:\Python35\lib\site-packages\tensorflow\python\client\session.py", line 1034, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.OutOfRangeError: RandomShuffleQueue '_1_shuffle_batch/random_shuffle_queue' is closed and has insufficient elements (requested 4, current size 0)
[[Node: shuffle_batch = QueueDequeueMany[_class=["loc:@shuffle_batch/random_shuffle_queue"], component_types=[DT_FLOAT, DT_INT32], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](shuffle_batch/random_shuffle_queue, shuffle_batch/n)]]

Caused by op 'shuffle_batch', defined at:
File "W:/PlantClassification/visualize_data.py", line 27, in
num_threads=4)
File "S:\Python35\lib\site-packages\tensorflow\python\training\input.py", line 917, in shuffle_batch
dequeued = queue.dequeue_many(batch_size, name=name)
File "S:\Python35\lib\site-packages\tensorflow\python\ops\data_flow_ops.py", line 458, in dequeue_many
self._queue_ref, n=n, component_types=self._dtypes, name=name)
File "S:\Python35\lib\site-packages\tensorflow\python\ops\gen_data_flow_ops.py", line 1099, in _queue_dequeue_many
timeout_ms=timeout_ms, name=name)
File "S:\Python35\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 759, in apply_op
op_def=op_def)
File "S:\Python35\lib\site-packages\tensorflow\python\framework\ops.py", line 2240, in create_op
original_op=self._default_original_op, op_def=op_def)
File "S:\Python35\lib\site-packages\tensorflow\python\framework\ops.py", line 1128, in init
self._traceback = _extract_stack()

OutOfRangeError (see above for traceback): RandomShuffleQueue '_1_shuffle_batch/random_shuffle_queue' is closed and has insufficient elements (requested 4, current size 0)
[[Node: shuffle_batch = QueueDequeueMany[_class=["loc:@shuffle_batch/random_shuffle_queue"], component_types=[DT_FLOAT, DT_INT32], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](shuffle_batch/random_shuffle_queue, shuffle_batch/n)]]

Process finished with exit code 1

Can we add an example in our tutorial for the end-2-end Memory Network (like this one in pure tensorflow: https://github.com/domluna/memn2n )
I think an example to demo Memory Network and neural attention mechanism is important for our framework, thanks.

Hi all.

import tensorflow as tf
import tensorlayer as tl

sess = tf.InteractiveSession()

# prepare data
X_train, y_train, X_val, y_val, X_test, y_test = \
                                tl.files.load_mnist_dataset(shape=(-1,784))
# define placeholder
x = tf.placeholder(tf.float32, shape=[None, 784], name='x')
y_ = tf.placeholder(tf.int64, shape=[None, ], name='y_')

# define the network
network = tl.layers.InputLayer(x, name='input_layer')
network = tl.layers.DropoutLayer(network, keep=0.8, name='drop1')
network = tl.layers.DenseLayer(network, n_units=800,
                                act = tf.nn.relu, name='relu1')
network = tl.layers.DropoutLayer(network, keep=0.5, name='drop2')
network = tl.layers.DenseLayer(network, n_units=800,
                                act = tf.nn.relu, name='relu2')
network = tl.layers.DropoutLayer(network, keep=0.5, name='drop3')
# the softmax is implemented internally in tl.cost.cross_entropy(y, y_) to
# speed up computation, so we use identity here.
# see tf.nn.sparse_softmax_cross_entropy_with_logits()
network = tl.layers.DenseLayer(network, n_units=10,
                                act = tf.identity,
                                name='output_layer')

# define cost function and metric.
y = network.outputs
cost = tl.cost.cross_entropy(y, y_)
correct_prediction = tf.equal(tf.argmax(y, 1), y_)
acc = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
y_op = tf.argmax(tf.nn.softmax(y), 1)

# define the optimizer
train_params = network.all_params
train_op = tf.train.AdamOptimizer(learning_rate=0.0001, beta1=0.9, beta2=0.999,
                            epsilon=1e-08, use_locking=False).minimize(cost, var_list=train_params)

# initialize all variables
sess.run(tf.initialize_all_variables())

# print network information
network.print_params()
network.print_layers()

# train the network
tl.utils.fit(sess, network, train_op, cost, X_train, y_train, x, y_,
            acc=acc, batch_size=500, n_epoch=500, print_freq=5,
            X_val=X_val, y_val=y_val, eval_train=False)

# evaluation
tl.utils.test(sess, network, acc, X_test, y_test, x, y_, batch_size=None, cost=cost)

# save the network to .npz file
tl.files.save_npz(network.all_params , name='model.npz')
sess.close()

I am a little confused with the follow line in tutorial_mnist_simple.py

39: y_op = tf.argmax(tf.nn.softmax(y), 1)

According to the documentation, """ y_op is the integer output represents the class index. """
It seems that we will compare "y_op"(output of network) and "y_"(ground_truth label).

But on line 36, we have defined cost function as follows:

36: cost = tl.cost.cross_entropy(y, y_)

So, we compare the variable "y" (not "y_op") and “y_”.

However, I found that the variable "y_op" never been used in tutorial_mnist_simple.py. (from line 40 to last line, it never appear again)

I don't know how the line work in tutorial_mnist_simple.py? Does it means that the line "y_op = tf.argmax(tf.nn.softmax(y), 1)" doesn't work in tutorial_mnist_simple.py?

it's really strange!

Hi all.

I want to apply activation function to the output of RNNLayer.
However the RNNLayer has no act argument as DenseLayer or Convolution layers have.

class tensorlayer.layers.RNNLayer(layer=None, cell_fn=<class 'tensorflow.python.ops.rnn_cell.BasicRNNCell'>, cell_init_args={}, n_hidden=100, initializer=<function _initializer>, n_steps=5, initial_state=None, return_last=False, return_seq_2d=False, name='rnn_layer')

I currently use the LambdaLayer to apply activation as follows.

net = tl.layers.RNNLayer(net, ...)
net = tl.layers.LambdaLayer(net, lambda x: act_fn(x))

where act_fn is a function such as tf.nn.relu.

Is it correct usage or is there any other way?

Thank you.

Thanks for your debug, but...

1. "max_length = tf.shape(self.outputs)[1]
   self.outputs = tf.reshape(tf.concat(1, outputs), [-1, max_length, n_hidden])", this is your new code
   but also wrong, "max_length = tf.shape(self.outputs)[1]" no self.outputs, maybe like this
   "max_length = tf.shape(outputs)[1]"

2. i can not find some DynamicRNNLayer example, the examples:

   input_seqs = tf.placeholder(dtype=tf.int64, shape=[batch_size, None], name="input_seqs")
   network = tl.layers.EmbeddingInputlayer(
                inputs = input_seqs,
                vocabulary_size = vocab_size,
                embedding_size = embedding_size,
                name = 'seq_embedding')
    network = tl.layers.DynamicRNNLayer(network,
                cell_fn = tf.nn.rnn_cell.BasicLSTMCell,
                n_hidden = embedding_size,
                dropout = 0.7,
                sequence_length = tl.layers.retrieve_seq_length_op2(input_seqs),
                return_seq_2d = True,     # stack denselayer or compute cost after it
                name = 'dynamic_rnn',)
    network = tl.layers.DenseLayer(network, n_units=vocab_size,
                act=tf.identity, name="output")
   

   i can not understand "shape=[batch_size, None]" i just write None? or n_step(max)
   if my data like this: (Sentiment Analysis)

   x = [[2, 1, 8, 9, 2]
        [2, 4]
        [1, 1, 3, 5] 
        [6, 3, 2]]
   y = [1, 0, 1, 1]
   

   i padding with zero

    x = [[2, 1, 8, 9, 2]
         [2, 4, 0, 0, 0]
         [1, 1, 3, 5, 0] 
         [6, 3, 2, 0, 0]]
   

   how to use DynamicRNNLayer?

To make it easier to contribute code it would be great if there was a set of existing tests collected in one place that could be run on committed code/Pull Requests.
Maybe the absolute best would be some form of TravisCI as for TFLearn?

See e.g. Keras tests
or TFLearn tests

How can I exclude some layers from training when I want to fine-tune a model?In keras I can simply set the trainable=False.Is there a method for tensorlayer to do this?