Fully-Connected Winner-Take-All Autoencoders Trained on MNIST

In [1]:
import sparsenet.nn_utils as nn
import sparsenet.convnet as convnet
import sparsenet.dataset as dataset

#We use Gnumpy as the matrix backend.
nn.set_backend("gnumpy",board=0,conv_backend="cudaconvnet")
gnumpy is running on GPU board: 0   convolution backend: cudaconvnet   seed: 859722

In [16]:
X,T,X_test,T_test,X_labels,T_labels=dataset.MNIST.load("numpy")
dp = nn.dp_ram(X=X,T=X,X_test=X_test,T_test=T_test,T_train_labels=None,T_labels=T_labels,
               train_range = [0,6],
               test_range = [0,1])
nn.show_images(X[:25,:,:,:],(5,5))
In [4]:
cfg = convnet.cn.NeuralNetCfg(want_lifetime_sparsity=True,want_dropout = False,want_tied= False)

cfg.input_dense(shape=784,dropout = None)
#Lifetime sparsity of 5%
cfg.dense(num_filters=1000,activation=nn.relu,lifetime_sparsity=5,dropout = None)
cfg.output_dense(num_filters=784,activation=nn.linear)
cfg.cost("euclidean")

#We are training a dense autoencoder on MNIST.
cfg.params(arch='dense',learning='auto',dataset='mnist')

cfg.info()
cfg.save_location('fc-wta')
AE=convnet.NeuralNet(cfg)
layer shape    :    [784, 1000, 784]
filter shape    :    [784, 1000]

In []:
AE.train(dp,
         mini_batch=100,
         num_epochs = 3000,
         initial_weights=.01,
         momentum=.9,
         learning_rate=0.01,learn_params={'epoch_1':[],'epoch_2':[]},
         want_visual=True,visual_params={'interval':10,'save':False},
         want_test=False,test_params={'interval':10},
         want_log=True,log_params={'interval':1},
         want_weights=True,weights_params={'interval':100,'want_last':True})

1% Sparsity

In [6]:
AE.load("/media/gpu3/work/save/mnist_auto_1/mnist_auto_1_last")
AE.show_filters()

2% Sparsity

In [13]:
AE.load("/media/gpu3/work/save/mnist_auto_2/mnist_auto_2_last")
AE.show_filters()

3% Sparsity

In [7]:
AE.load("/media/gpu3/work/save/mnist_auto_3/mnist_auto_3_last")
AE.show_filters()

5% Sparsity (Best Classification Rate)

In [5]:
AE.load("/media/gpu3/work/save/mnist_auto_5/mnist_auto_5_last")
AE.show_filters()