Fully-Connected Winner-Take-All Autoencoders Trained on MNIST

In [1]: 
import sparsenet.nn_utils as nn
import sparsenet.convnet as convnet
import sparsenet.dataset as dataset

#We use Gnumpy as the matrix backend.
nn.set_backend("gnumpy",board=0,conv_backend="cudaconvnet")

gnumpy is running on GPU board: 0   convolution backend: cudaconvnet   seed: 859722
In [16]: 
X,T,X_test,T_test,X_labels,T_labels=dataset.MNIST.load("numpy")
dp = nn.dp_ram(X=X,T=X,X_test=X_test,T_test=T_test,T_train_labels=None,T_labels=T_labels,
               train_range = [0,6],
               test_range = [0,1])
nn.show_images(X[:25,:,:,:],(5,5))
In [4]: 
cfg = convnet.cn.NeuralNetCfg(want_lifetime_sparsity=True,want_dropout = False,want_tied= False)

cfg.input_dense(shape=784,dropout = None)
#Lifetime sparsity of 5%
cfg.dense(num_filters=1000,activation=nn.relu,lifetime_sparsity=5,dropout = None)
cfg.output_dense(num_filters=784,activation=nn.linear)
cfg.cost("euclidean")

#We are training a dense autoencoder on MNIST.
cfg.params(arch='dense',learning='auto',dataset='mnist')

cfg.info()
cfg.save_location('fc-wta')
AE=convnet.NeuralNet(cfg)

layer shape    :    [784, 1000, 784]
filter shape    :    [784, 1000]
In []: 
AE.train(dp,
         mini_batch=100,
         num_epochs = 3000,
         initial_weights=.01,
         momentum=.9,
         learning_rate=0.01,learn_params={'epoch_1':[],'epoch_2':[]},
         want_visual=True,visual_params={'interval':10,'save':False},
         want_test=False,test_params={'interval':10},
         want_log=True,log_params={'interval':1},
         want_weights=True,weights_params={'interval':100,'want_last':True})

1% Sparsity

In [6]: 
AE.load("/media/gpu3/work/save/mnist_auto_1/mnist_auto_1_last")
AE.show_filters()

2% Sparsity

In [13]: 
AE.load("/media/gpu3/work/save/mnist_auto_2/mnist_auto_2_last")
AE.show_filters()

3% Sparsity

In [7]: 
AE.load("/media/gpu3/work/save/mnist_auto_3/mnist_auto_3_last")
AE.show_filters()

5% Sparsity (Best Classification Rate)

In [5]: 
AE.load("/media/gpu3/work/save/mnist_auto_5/mnist_auto_5_last")
AE.show_filters()

10% Sparsity

In [8]: 
AE.load("/media/gpu3/work/save/mnist_auto_10/mnist_auto_10_last")
AE.show_filters()

Classification Result (Logistic Regression on top of unsupervised features)

~120 Errors on MNIST

In [18]: 
#Tying weights improve the classification performance
cfg = convnet.cn.NeuralNetCfg(want_lifetime_sparsity=True,want_dropout = False,want_tied= True)

cfg.input_dense(shape=784,dropout = None)
#lifetime_sparsity of 5%
cfg.dense(num_filters=2000,activation=nn.relu,lifetime_sparsity=5,dropout = None)
cfg.output_dense(num_filters=784,activation=nn.linear)
cfg.cost("euclidean")

cfg.params(arch='dense',learning='auto',dataset='mnist')
cfg.info()
AE=convnet.NeuralNet(cfg)

layer shape    :    [784, 2000, 784]
filter shape    :    [784, 2000]
In [20]: 
AE.load("/media/gpu3/work/save/mnist_auto_2000_5/mnist_auto_2000_5")
In [25]: 
#At test time, we turn off the sparsity constraint
AE.test_mode = True

#We create a new dataset by passing all the training points through the network and obtaining their representations:
H_train = np.zeros((60000,2000))
for i in range(300):
    AE.feedforward(nn.garray(dp.X[i*200:(i+1)*200]))
    H_train[i*200:(i+1)*200] = AE.H[1].as_numpy_array()

H_test = np.zeros((10000,2000))
for i in range(50):
    AE.feedforward(nn.garray(dp.X_test[i*200:(i+1)*200]))
    H_test[i*200:(i+1)*200] = AE.H[1].as_numpy_array()
    
AE.test_mode = False

Logistic Regression

In [27]: 
#We train a logitic regression on top of the unsupervised features.
LR_cfg = convnet.cn.NeuralNetCfg()
LR_cfg.input_dense(shape=2000)
LR_cfg.output_dense(num_filters=10,activation=nn.softmax)
LR_cfg.cost("cross-entropy")
LR_cfg.params(arch='dense',learning='disc',dataset='mnist')
LR=convnet.NeuralNet(LR_cfg)
In [28]: 
#Data Provider for Logistic Regression
dp = nn.dp_ram(X=H_train,T=T,X_test=H_test,T_test=T_test,T_train_labels=None,T_labels=None,
               train_range = [0,6],
               test_range = [0,1])
In [30]: 
LR.train(dp,
         mini_batch=100,
         num_epochs = 20,
         initial_weights=.01,
         momentum=.9,
         learning_rate=.1,learn_params={'epoch_1':[15,.01],'epoch_2':[18,.001]},
         want_visual=False,visual_params={'interval':3,'save':False},
         want_test=True,test_params={'interval':10},
         want_log=False,log_params={'interval':1},
         want_weights=False,weights_params={'interval':100})

Epoch:1.4   Train Error:0.644022950363   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0480680465698
Epoch:1.5   Train Error:0.183785036922   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0442991256714
Epoch:1.0   Train Error:0.159657656574   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0443818569183
Epoch:1.1   Train Error:0.140641869116   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0441770553589
Epoch:1.2   Train Error:0.12455858798   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0442090034485
Epoch:1.3   Train Error:0.116922957063   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0440580844879
Epoch:2.4   Train Error:0.107135946941   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0442070960999
Epoch:2.5   Train Error:0.0843695004106   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0441598892212
Epoch:2.0   Train Error:0.0899140512943   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0452280044556
Epoch:2.1   Train Error:0.0847666001797   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0441870689392
Epoch:2.2   Train Error:0.0825759699106   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0443730354309
Epoch:2.3   Train Error:0.0821630342245   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.044303894043
---------------
Test Error:194/10000=0.0194   Logprob Test:0.0767152475864   ---Best:194/10000=0.0194   ---Name:No-Name
---------------
Epoch:3.4   Train Error:0.0799640577078   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0535969734192
Epoch:3.5   Train Error:0.0640831068635   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0441479682922
Epoch:3.0   Train Error:0.0699244976163   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0440530776978
Epoch:3.1   Train Error:0.0656170445204   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0441558361053
Epoch:3.2   Train Error:0.0658681884527   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0439920425415
Epoch:3.3   Train Error:0.0670310666561   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.044065952301
Epoch:4.4   Train Error:0.0665817611217   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0444831848145
Epoch:4.5   Train Error:0.0536468141615   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0443198680878
Epoch:4.0   Train Error:0.0591606158972   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0441009998322
Epoch:4.1   Train Error:0.054914758563   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.044184923172
---------------
Test Error:162/10000=0.0162   Logprob Test:0.0620755337395   ---Best:162/10000=0.0162   ---Name:No-Name
---------------
Epoch:4.2   Train Error:0.0561828789473   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0442490577698
Epoch:4.3   Train Error:0.057905471158   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0439801216125
Epoch:5.4   Train Error:0.0580567039728   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0444300174713
Epoch:5.5   Train Error:0.0468908232719   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0439848899841
---------------
Test Error:156/10000=0.0156   Logprob Test:0.0586059596434   ---Best:156/10000=0.0156   ---Name:No-Name
---------------
Epoch:5.0   Train Error:0.0520821051717   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0445420742035
Epoch:5.1   Train Error:0.0477957065344   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0443370342255
Epoch:5.2   Train Error:0.049609202683   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.04447889328
Epoch:5.3   Train Error:0.0515608594418   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.04430103302
Epoch:6.4   Train Error:0.0519415459752   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0444498062134
Epoch:6.5   Train Error:0.0420144724935   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0443959236145
Epoch:6.0   Train Error:0.0469344470561   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0445349216461
Epoch:6.1   Train Error:0.0426112050653   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0442879199982
Epoch:6.2   Train Error:0.0447325434029   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0445280075073
Epoch:6.3   Train Error:0.0467835618377   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0443480014801
Epoch:7.4   Train Error:0.0472473412514   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0445330142975
Epoch:7.5   Train Error:0.0382596018106   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0445249080658
Epoch:7.0   Train Error:0.0429498749852   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0444331169128
Epoch:7.1   Train Error:0.0386158610404   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0442950725555
Epoch:7.2   Train Error:0.040902970916   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0444321632385
Epoch:7.3   Train Error:0.0429981943846   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0443949699402
---------------
Test Error:140/10000=0.014   Logprob Test:0.050424724035   ---Best:140/10000=0.014   ---Name:No-Name
---------------
Epoch:8.4   Train Error:0.0434819799542   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0441558361053
Epoch:8.5   Train Error:0.035241899246   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0444560050964
Epoch:8.0   Train Error:0.0397306047797   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0451939105988
Epoch:8.1   Train Error:0.035415337944   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0444159507751
Epoch:8.2   Train Error:0.0377764209688   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0443050861359
Epoch:8.3   Train Error:0.0398911900401   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0443959236145
Epoch:9.4   Train Error:0.0403667852342   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0442039966583
Epoch:9.5   Train Error:0.0327424694106   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0443420410156
Epoch:9.0   Train Error:0.0370478274167   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0441508293152
Epoch:9.1   Train Error:0.0327781748831   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0443818569183
---------------
Test Error:131/10000=0.0131   Logprob Test:0.048126297421   ---Best:131/10000=0.0131   ---Name:No-Name
---------------
Epoch:9.2   Train Error:0.0351524500966   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0444579124451
Epoch:9.3   Train Error:0.0372745751023   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0440039634705
Epoch:10.4   Train Error:0.0377297370732   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0443580150604
Epoch:10.5   Train Error:0.0306258578733   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0440421104431
---------------
Test Error:129/10000=0.0129   Logprob Test:0.047102564073   ---Best:129/10000=0.0129   ---Name:No-Name
---------------
Epoch:10.0   Train Error:0.0347597220898   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0448958873749
Epoch:10.1   Train Error:0.0305577512622   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0444610118866
Epoch:10.2   Train Error:0.032905429101   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0444719791412
Epoch:10.3   Train Error:0.0350273777664   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0443389415741
Epoch:11.4   Train Error:0.0354577270627   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0444378852844
Epoch:11.5   Train Error:0.0288028159261   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.044625043869
Epoch:11.0   Train Error:0.0327732208848   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0443689823151
Epoch:11.1   Train Error:0.0286556722581   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0443589687347
Epoch:11.2   Train Error:0.030951769197   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0448169708252
Epoch:11.3   Train Error:0.0330673587859   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0444638729095
Epoch:12.4   Train Error:0.0334726904154   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0445218086243
Epoch:12.5   Train Error:0.0272115493551   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0444679260254
Epoch:12.0   Train Error:0.0310243914634   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.044450044632
Epoch:12.1   Train Error:0.0270029659986   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0443120002747
Epoch:12.2   Train Error:0.0292329514861   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0444259643555
Epoch:12.3   Train Error:0.031336178261   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0445010662079
---------------
Test Error:121/10000=0.0121   Logprob Test:0.0443921170203   ---Best:121/10000=0.0121   ---Name:No-Name
---------------
Epoch:13.4   Train Error:0.0317185541689   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.044203042984
Epoch:13.5   Train Error:0.0258074928552   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0445230007172
Epoch:13.0   Train Error:0.0294676045239   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0441520214081
Epoch:13.1   Train Error:0.0255497071445   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0442981719971
Epoch:13.2   Train Error:0.0277062618792   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0440299510956
Epoch:13.3   Train Error:0.0297910563946   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0450668334961
Epoch:14.4   Train Error:0.0301537469208   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0442810058594
Epoch:14.5   Train Error:0.0245574392825   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0443730354309
Epoch:14.0   Train Error:0.0280692314446   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0441861152649
Epoch:14.1   Train Error:0.0242588754177   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.044319152832
---------------
Test Error:120/10000=0.012   Logprob Test:0.0436528496584   ---Best:120/10000=0.012   ---Name:No-Name
---------------
Epoch:14.2   Train Error:0.0263394501776   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0445039272308
Epoch:14.3   Train Error:0.028399807322   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.0441288948059
Epoch:15.4   Train Error:0.0287183415174   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.0445659160614
Epoch:15.5   Train Error:0.0232384380504   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.044182062149
---------------
Test Error:121/10000=0.0121   Logprob Test:0.0431733927241   ---Best:120/10000=0.012   ---Name:No-Name
---------------
Epoch:15.0   Train Error:0.0266777597427   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.0444121360779
Epoch:15.1   Train Error:0.022686493364   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.0452508926392
Epoch:15.2   Train Error:0.0244148959816   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.0444719791412
Epoch:15.3   Train Error:0.0263036305904   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.0443849563599
Epoch:16.4   Train Error:0.0285003356695   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.044508934021
Epoch:16.5   Train Error:0.0231241753578   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.044429063797
Epoch:16.0   Train Error:0.0264755713105   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.0444638729095
Epoch:16.1   Train Error:0.0225726972044   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.0443019866943
Epoch:16.2   Train Error:0.0243120174766   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.0444087982178
Epoch:16.3   Train Error:0.026234936434   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.0445139408112
Epoch:17.4   Train Error:0.0283162332833   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.0443968772888
Epoch:17.5   Train Error:0.0230114364401   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.0443661212921
Epoch:17.0   Train Error:0.0263084679455   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.0441401004791
Epoch:17.1   Train Error:0.022466381371   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.0440988540649
Epoch:17.2   Train Error:0.0242141721517   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.0443339347839
Epoch:17.3   Train Error:0.0261574331403   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.0441110134125
---------------
Test Error:120/10000=0.012   Logprob Test:0.0430298213465   ---Best:120/10000=0.012   ---Name:No-Name
---------------
Epoch:18.4   Train Error:0.0281443959832   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0444250106812
Epoch:18.5   Train Error:0.0228765123099   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0443069934845
Epoch:18.0   Train Error:0.0261472959936   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0438580513
Epoch:18.1   Train Error:0.0223290716946   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0442218780518
Epoch:18.2   Train Error:0.0240406672597   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0441510677338
Epoch:18.3   Train Error:0.025965487361   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0441610813141
Epoch:19.4   Train Error:0.0281281063974   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0441401004791
Epoch:19.5   Train Error:0.0228653363094   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0441000461578
Epoch:19.0   Train Error:0.0261319668204   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0439188480377
Epoch:19.1   Train Error:0.0223186538339   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0440649986267
---------------
Test Error:120/10000=0.012   Logprob Test:0.0430302093094   ---Best:120/10000=0.012   ---Name:No-Name
---------------
Epoch:19.2   Train Error:0.0240310953856   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0446720123291
Epoch:19.3   Train Error:0.0259578445673   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0437040328979
Epoch:20.4   Train Error:0.0281115069866   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0442960262299
Epoch:20.5   Train Error:0.0228541681677   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0438799858093
---------------
Test Error:120/10000=0.012   Logprob Test:0.0430256764438   ---Best:120/10000=0.012   ---Name:No-Name
---------------
Epoch:20.0   Train Error:0.0261168103456   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0441911220551
Epoch:20.1   Train Error:0.0223082630754   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0444860458374
Epoch:20.2   Train Error:0.0240215409458   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0445220470428
Epoch:20.3   Train Error:0.0259501305282   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.0442309379578
Out[30]:
(120, 0.04302150782458483)
Convolutional Winner-Take-All Autoencoders

Convolutional Winner-Take-All Autoencoders

In [11]: 
import sparsenet.nn_utils as nn
import sparsenet.convnet as convnet
import sparsenet.dataset as dataset

#We use Gnumpy for matrix operations and CUDNN for convolution backend
nn.set_backend("gnumpy",board=0,conv_backend="cudnn")

gnumpy is running on GPU board: 0   convolution backend: cudnn   seed: 344261
In [3]: 
X,T,X_test,T_test,X_labels,T_labels=dataset.MNIST.load("numpy")
dp = nn.dp_ram(X=X,T=X,X_test=X_test,T_test=T_test,T_train_labels=None,T_labels=T_labels,
               train_range = [0,6],
               test_range = [0,1])
nn.show_images(X[:25,:,:,:],(5,5))

Architecture: 64conv5-64conv5-64conv5-64deconv11

In [22]: 
cfg = convnet.cn.NeuralNetCfg(want_spatial_sparsity=True,want_dropout = False,want_tied=False)

cfg.input_conv(shape=[1,28,28],dropout=False)

cfg.convolution(num_filters=64,activation=nn.relu,filter_width=5,padding=2,stride=1,partial_sum=4,dropout = None,spatial_sparsity=None,lifetime_sparsity=None)
cfg.convolution(num_filters=64,activation=nn.relu,filter_width=5,padding=2,stride=1,partial_sum=4,dropout = None,spatial_sparsity=None,lifetime_sparsity=None)
#This is the last layer of encoder with Spatial and Lifetime sparsity constarints.
cfg.convolution(num_filters=64,activation=nn.relu,filter_width=5,padding=2,stride=1,partial_sum=4,dropout = None,
                spatial_sparsity=1,lifetime_sparsity=20)
#The decoder is a linear deconvolutional layer.
cfg.output_deconv(num_filters=1,activation=nn.linear,filter_width=11,padding=5,stride=1,partial_sum=4,imSizeX=28)
cfg.cost("euclidean")

#This is for visualization purposes. We are training a 'conv' autencoder on MNIST.
cfg.params(arch='conv',learning='auto',dataset='mnist')

cfg.info()
cfg.save_location('conv-wta')
AE=convnet.NeuralNet(cfg)

layer shape    :    [[1, 28, 28], [64, 28, 28], [64, 28, 28], [64, 28, 28], [1, 28, 28]]
filter shape    :    [[64, 1, 5, 5], [64, 64, 5, 5], [64, 64, 5, 5], [64, 1, 11, 11]]
In []: 
AE.train(dp,
         mini_batch=100,
         num_epochs = 10000,
         initial_weights=.001,
         momentum=.9,
         #learning rate starts with 1e-3 and after 200 epochs will change to 1e-4
         learning_rate=1e-3,learn_params={'epoch_1':[200,1e-4],'epoch_2':[]},
         want_visual=True,visual_params={'interval':20,'save':True},
         want_test=False,test_params={'interval':10},
         want_log=False,log_params={'interval':1},
         want_weights=True,weights_params={'interval':20})

Spatial Sparsity only

In [42]: 
#We are loading the weights from the previously trained netwroks.
AE.load("/media/gpu3/work/save/mnist_auto_conv_64_55511/mnist_auto_conv_64_55511_last")
AE.show_filters()

Spatial Sparsity + Lifetime Sparsity of 20%

In [43]: 
AE.load("/media/gpu3/work/save/mnist_auto_conv_64_55511_lifetime20/mnist_auto_conv_64_55511_lifetime20_last")
AE.show_filters()

Spatial Sparsity + Lifetime Sparsity of 5%

In [44]: 
AE.load("/media/gpu3/work/save/mnist_auto_conv_64_55511_lifetime5/mnist_auto_conv_64_55511_lifetime5_last")
AE.show_filters()

Classification Result (Logistic Regression on top of unsupervised features)

~100 Errors on MNIST with only 16 Maps

In [7]: 
import sparsenet.nn_utils as nn
import sparsenet.convnet as convnet
import sparsenet.dataset as dataset

nn.set_backend("gnumpy",board=0,conv_backend="cudaconvnet")

gnumpy is running on GPU board: 0   convolution backend: cudaconvnet   seed: 859722
In [8]: 
X,T,X_test,T_test,X_labels,T_labels=dataset.MNIST.load("numpy")
dp = nn.dp_ram(X=X,T=X,X_test=X_test,T_test=T_test,T_train_labels=None,T_labels=T_labels,
                     train_range = [0,6],
                     test_range = [0,1])
nn.show_images(X[:25,:,:,:],(5,5))
In [11]: 
cfg = convnet.cn.NeuralNetCfg(want_spatial_sparsity=True,want_dropout = False,want_tied=False)
cfg.input_conv(shape=[1,28,28],dropout=False)
cfg.convolution(num_filters=16,activation=nn.relu,filter_width=7,padding=3,stride=1,partial_sum=4,
                dropout = None,spatial_sparsity=1,lifetime_sparsity=None)
cfg.output_deconv(num_filters=1,activation=nn.linear,filter_width=7,padding=3,stride=1,partial_sum=4,imSizeX=28)
cfg.cost("euclidean")
cfg.params(arch='conv',learning='auto',dataset='mnist')
cfg.info()
AE=convnet.NeuralNet(cfg)

layer shape    :    [[1, 28, 28], [16, 28, 28], [1, 28, 28]]
filter shape    :    [[16, 1, 7, 7], [16, 1, 7, 7]]
In [10]: 
#We are training the autoencoder, and after every two epochs, we train a logistic regression on top of 
#the unsupervised features to find the classification rate on the test data.
AE.train(dp,
         mini_batch=100,
         num_epochs = 5,
         initial_weights=.001,
         momentum=.9,
         learning_rate=1e-3,learn_params={'epoch_1':[4,1e-4],'epoch_2':[]},
         want_visual=False,visual_params={'interval':1,'save':False},
         want_test=True,test_params={'interval':10},
         want_log=False,log_params={'interval':1},
         want_weights=False,weights_params={'interval':100})

Epoch:1.0   Train Error:35.3927266884   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0399060249329
Epoch:1.1   Train Error:22.1174298859   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0397160053253
Epoch:1.2   Train Error:18.263107996   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0306839942932
Epoch:1.3   Train Error:15.9310403442   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0309500694275
Epoch:1.4   Train Error:14.8963329887   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0306708812714
Epoch:1.5   Train Error:13.895044241   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0307970046997
Epoch:2.0   Train Error:13.8057173824   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0306899547577
Epoch:2.1   Train Error:13.3864077187   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0308029651642
Epoch:2.2   Train Error:13.2882362461   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0305969715118
Epoch:2.3   Train Error:12.6034511757   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0307278633118
---------------
Test Error:130/10000=0.013   Logprob Test:0.0522884076645   ---Best:130/10000=0.013   ---Name:No-Name
---------------
Epoch:2.4   Train Error:12.524947176   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0308170318604
Epoch:2.5   Train Error:12.0195212936   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0305421352386
Epoch:3.0   Train Error:12.1439297104   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0307281017303
Epoch:3.1   Train Error:11.8947193146   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0306360721588
Epoch:3.2   Train Error:11.8795312119   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0306549072266
Epoch:3.3   Train Error:11.3494461918   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0303659439087
Epoch:3.4   Train Error:11.350540514   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0305399894714
Epoch:3.5   Train Error:10.965830946   Time:0.81   Learning Rate:0.001   Databatch Waiting:0.0307738780975
Epoch:4.0   Train Error:11.1731748009   Time:0.81   Learning Rate:0.0001   Databatch Waiting:0.0308310985565
Epoch:4.1   Train Error:11.0117273903   Time:0.81   Learning Rate:0.0001   Databatch Waiting:0.0306169986725
---------------
Test Error:103/10000=0.0103   Logprob Test:0.032054324194   ---Best:103/10000=0.0103   ---Name:No-Name
---------------
Epoch:4.2   Train Error:11.0532069588   Time:0.81   Learning Rate:0.0001   Databatch Waiting:0.0305500030518
Epoch:4.3   Train Error:10.8307056332   Time:0.81   Learning Rate:0.0001   Databatch Waiting:0.0306520462036
Epoch:4.4   Train Error:10.9267267227   Time:0.81   Learning Rate:0.0001   Databatch Waiting:0.0304820537567
Epoch:4.5   Train Error:10.7131963158   Time:0.81   Learning Rate:0.0001   Databatch Waiting:0.0305819511414
Epoch:5.0   Train Error:10.9158666325   Time:0.81   Learning Rate:0.0001   Databatch Waiting:0.0306940078735
Epoch:5.1   Train Error:10.9496178532   Time:0.81   Learning Rate:0.0001   Databatch Waiting:0.0300278663635
Epoch:5.2   Train Error:10.9887722683   Time:0.81   Learning Rate:0.0001   Databatch Waiting:0.029853105545
Epoch:5.3   Train Error:10.7748430729   Time:0.81   Learning Rate:0.0001   Databatch Waiting:0.0300738811493
Epoch:5.4   Train Error:10.8793296719   Time:0.81   Learning Rate:0.0001   Databatch Waiting:0.0298931598663
Epoch:5.5   Train Error:10.6610181332   Time:0.81   Learning Rate:0.0001   Databatch Waiting:0.0299699306488
---------------
Test Error:98/10000=0.0098   Logprob Test:0.0311423671881   ---Best:98/10000=0.0098   ---Name:No-Name
---------------
Out[10]:
(99, 0.031232589889992973)

~65 Errors on MNIST with 128 Maps

In [12]: 
cfg = convnet.cn.NeuralNetCfg(want_spatial_sparsity=True,want_dropout = False,want_tied=False)
cfg.input_conv(shape=[1,28,28],dropout = None)
cfg.convolution(num_filters=128,activation=nn.relu,filter_width=7,padding=3,stride=1,partial_sum = 4,dropout = None,
                spatial_sparsity = 1)
cfg.output_deconv(num_filters=1,activation=nn.linear,filter_width=7,padding=3,stride=1,partial_sum = 4,imSizeX=28)
cfg.cost("euclidean")
cfg.params(arch='conv',learning='auto',dataset='mnist')
cfg.info()
AE=convnet.NeuralNet(cfg)

layer shape    :    [[1, 28, 28], [128, 28, 28], [1, 28, 28]]
filter shape    :    [[128, 1, 7, 7], [128, 1, 7, 7]]
In [13]: 
#Loading weights 
AE.load("/media/gpu4/work/save/mnist_auto_conv_128_7/mnist_auto_conv_128_7")

We create a new dataset by passing all the training examples through the network and obtaining their representations:

In [17]: 
#Turning off spatial sparsity
AE.test_mode = True            

H_train = np.zeros((60000,128, 10, 10))
for i in range(300):
    AE.feedforward(nn.garray(dp.X[i*200:(i+1)*200,:,:,:]))
    #Applying local Max-Pooling to feature maps 
    H_train[i*200:(i+1)*200,:,:,:] = nn.MaxPool(AE.H[1],subsX=5,startX=0,strideX=3,outputsX=10).as_numpy_array()
H_test = np.zeros((10000,128, 10, 10))
for i in range(50):    
    AE.feedforward(nn.garray(dp.X_test[i*200:(i+1)*200,:,:,:]))
    #Applying local Max-Pooling to feature maps     
    H_test[i*200:(i+1)*200,:,:,:] = nn.MaxPool(AE.H[1],subsX=5,startX=0,strideX=3,outputsX=10).as_numpy_array()

Logistic Regression

In [18]: 
#Defining a logistic regression that will be trained top of the pooled feature maps.
LR_cfg = convnet.cn.NeuralNetCfg()
LR_cfg.input_conv(shape=[128,10,10])
LR_cfg.output_dense(num_filters=10,activation=nn.softmax)
LR_cfg.cost("cross-entropy")
LR_cfg.params(arch='conv',learning='disc',dataset='mnist')
LR=convnet.NeuralNet(LR_cfg)
In [19]: 
#Loading labels
_,T,_,T_test,_,T_labels=dataset.MNIST.load()
#Data provider for the neural net.
dp = nn.dp_ram(X=H_train,T=T,X_test=H_test,T_test=T_test,T_train_labels=None,T_labels=T_labels,
               train_range = [0,6],
               test_range = [0,1])
In [20]: 
#Trainig a logistic regression and reporting the test error.
LR.train(dp,
         silent_mode = False,
         mini_batch=100,
         num_epochs = 20,
         initial_weights=.01,
         momentum=.9,
         learning_rate=.1,learn_params={'epoch_1':[10,.01],'epoch_2':[20,.001]},
         want_visual=False,visual_params={'interval':3,'save':''},
         want_test=True,test_params={'interval':10},
         want_log=False,log_params={'interval':1},
         want_weights=False,weights_params={'interval':100})   

Epoch:1.0   Train Error:0.284026336145   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.332408189774
Epoch:1.1   Train Error:0.108634877992   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.329989910126
Epoch:1.2   Train Error:0.0752703015566   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.275141000748
Epoch:1.3   Train Error:0.0709591593742   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.272775888443
Epoch:1.4   Train Error:0.0701843433738   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.273972988129
Epoch:1.5   Train Error:0.0544888845637   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.272833108902
Epoch:2.0   Train Error:0.0514113571107   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.275990962982
Epoch:2.1   Train Error:0.0436212804735   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.273031949997
Epoch:2.2   Train Error:0.0426612812698   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.275010108948
Epoch:2.3   Train Error:0.0453374712586   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.271399021149
---------------
Test Error:129/10000=0.0129   Logprob Test:0.0443956168473   ---Best:129/10000=0.0129   ---Name:No-Name
---------------
Epoch:2.4   Train Error:0.0466553240895   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.330680847168
Epoch:2.5   Train Error:0.0377294398971   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.274796009064
Epoch:3.0   Train Error:0.0379667551756   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.270910024643
Epoch:3.1   Train Error:0.030505122593   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.275496006012
Epoch:3.2   Train Error:0.0338391330302   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.270956039429
Epoch:3.3   Train Error:0.0363248024136   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.274075984955
Epoch:3.4   Train Error:0.0373740191162   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.273199081421
Epoch:3.5   Train Error:0.0305616247989   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.274172067642
Epoch:4.0   Train Error:0.0311186860323   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.271520853043
Epoch:4.1   Train Error:0.0243843811005   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.273855924606
---------------
Test Error:113/10000=0.0113   Logprob Test:0.0336383603839   ---Best:113/10000=0.0113   ---Name:No-Name
---------------
Epoch:4.2   Train Error:0.028804285571   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.271671056747
Epoch:4.3   Train Error:0.0309687415302   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.271884918213
Epoch:4.4   Train Error:0.0318109278202   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.27242398262
Epoch:4.5   Train Error:0.0259696418729   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.271989107132
Epoch:5.0   Train Error:0.0266154930443   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.272705078125
Epoch:5.1   Train Error:0.0206567114547   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.272003173828
Epoch:5.2   Train Error:0.0252626900554   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.272742986679
Epoch:5.3   Train Error:0.0272505120158   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.270750999451
Epoch:5.4   Train Error:0.0278952620298   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.27312207222
Epoch:5.5   Train Error:0.0227355662029   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.271359920502
---------------
Test Error:96/10000=0.0096   Logprob Test:0.0291962042421   ---Best:96/10000=0.0096   ---Name:No-Name
---------------
Epoch:6.0   Train Error:0.0233041291147   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.275066137314
Epoch:6.1   Train Error:0.0180433257148   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.272437095642
Epoch:6.2   Train Error:0.0225755017549   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.274875879288
Epoch:6.3   Train Error:0.0244397978365   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.272458076477
Epoch:6.4   Train Error:0.0249129104048   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.276134967804
Epoch:6.5   Train Error:0.0203064524926   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.272984027863
Epoch:7.0   Train Error:0.0207437165856   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.274546146393
Epoch:7.1   Train Error:0.0160791188359   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.273824930191
Epoch:7.2   Train Error:0.0204460196823   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.2739610672
Epoch:7.3   Train Error:0.0221833769515   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.273180007935
---------------
Test Error:89/10000=0.0089   Logprob Test:0.0300451236826   ---Best:89/10000=0.0089   ---Name:No-Name
---------------
Epoch:7.4   Train Error:0.02253125287   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.272524118423
Epoch:7.5   Train Error:0.0183854380254   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.276216983795
Epoch:8.0   Train Error:0.0187024926335   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.272177934647
Epoch:8.1   Train Error:0.0145472281218   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.274477005005
Epoch:8.2   Train Error:0.0186981875762   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.272365093231
Epoch:8.3   Train Error:0.0203065953627   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.274996042252
Epoch:8.4   Train Error:0.0205635370165   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.271797180176
Epoch:8.5   Train Error:0.0168113235261   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.274996042252
Epoch:9.0   Train Error:0.017036350666   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.271562814713
Epoch:9.1   Train Error:0.0133212132052   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.27654004097
---------------
Test Error:81/10000=0.0081   Logprob Test:0.0262204909001   ---Best:81/10000=0.0081   ---Name:No-Name
---------------
Epoch:9.2   Train Error:0.017220775266   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.273588895798
Epoch:9.3   Train Error:0.0187142451048   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.27113199234
Epoch:9.4   Train Error:0.0188960041493   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.2735850811
Epoch:9.5   Train Error:0.0154922116345   Time:0.11   Learning Rate:0.1   Databatch Waiting:0.272118091583
Epoch:10.0   Train Error:0.0159761180952   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272716999054
Epoch:10.1   Train Error:0.011868674469   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272994995117
Epoch:10.2   Train Error:0.0148939071715   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.27436709404
Epoch:10.3   Train Error:0.0158631484404   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.271317958832
Epoch:10.4   Train Error:0.0151879264191   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.273232936859
Epoch:10.5   Train Error:0.0126669569429   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272150993347
---------------
Test Error:78/10000=0.0078   Logprob Test:0.0241627276557   ---Best:78/10000=0.0078   ---Name:No-Name
---------------
Epoch:11.0   Train Error:0.0147142156452   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.274322986603
Epoch:11.1   Train Error:0.0115320343554   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.274160861969
Epoch:11.2   Train Error:0.0146521955267   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.274236917496
Epoch:11.3   Train Error:0.01563164161   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272225141525
Epoch:11.4   Train Error:0.0150391616866   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.276109933853
Epoch:11.5   Train Error:0.012664014551   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.273364067078
Epoch:12.0   Train Error:0.0144817577839   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.274981021881
Epoch:12.1   Train Error:0.0113721193194   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272464036942
Epoch:12.2   Train Error:0.0144565214515   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.275065898895
Epoch:12.3   Train Error:0.0154622474343   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272764205933
---------------
Test Error:70/10000=0.007   Logprob Test:0.0238840958367   ---Best:70/10000=0.007   ---Name:No-Name
---------------
Epoch:12.4   Train Error:0.0149282404616   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.27236199379
Epoch:12.5   Train Error:0.0126421303812   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.274827957153
Epoch:13.0   Train Error:0.0142965781018   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.274821996689
Epoch:13.1   Train Error:0.0112465966761   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.275616884232
Epoch:13.2   Train Error:0.0143006296411   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.274358034134
Epoch:13.3   Train Error:0.0153195154056   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.273938894272
Epoch:13.4   Train Error:0.014825860621   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272725820541
Epoch:13.5   Train Error:0.0126046465939   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.274909973145
Epoch:14.0   Train Error:0.0141368630052   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.271871089935
Epoch:14.1   Train Error:0.0111395007968   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.275788068771
---------------
Test Error:72/10000=0.0072   Logprob Test:0.023983591057   ---Best:70/10000=0.007   ---Name:No-Name
---------------
Epoch:14.2   Train Error:0.0141652935296   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272861003876
Epoch:14.3   Train Error:0.0151907491863   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.271625995636
Epoch:14.4   Train Error:0.0147260622397   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.27459192276
Epoch:14.5   Train Error:0.0125566468887   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.273568868637
Epoch:15.0   Train Error:0.0139930142358   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272516012192
Epoch:15.1   Train Error:0.0110433714569   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.273182153702
Epoch:15.2   Train Error:0.0140420389444   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272300958633
Epoch:15.3   Train Error:0.0150705196247   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272484064102
Epoch:15.4   Train Error:0.0146272389993   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.273044109344
Epoch:15.5   Train Error:0.0125017638959   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272047996521
---------------
Test Error:71/10000=0.0071   Logprob Test:0.0239695040727   ---Best:70/10000=0.007   ---Name:No-Name
---------------
Epoch:16.0   Train Error:0.0138600184679   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.275111913681
Epoch:16.1   Train Error:0.0109544049874   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272416114807
Epoch:16.2   Train Error:0.0139267298266   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.275237083435
Epoch:16.3   Train Error:0.0149561448067   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.273103952408
Epoch:16.4   Train Error:0.0145290699676   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.275171041489
Epoch:16.5   Train Error:0.0124423949752   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.274791002274
Epoch:17.0   Train Error:0.0137349646792   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.273802995682
Epoch:17.1   Train Error:0.010870479466   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.273255825043
Epoch:17.2   Train Error:0.0138171423644   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.274926900864
Epoch:17.3   Train Error:0.0148461535126   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272662162781
---------------
Test Error:72/10000=0.0072   Logprob Test:0.0237340855662   ---Best:70/10000=0.007   ---Name:No-Name
---------------
Epoch:17.4   Train Error:0.0144315945923   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272299051285
Epoch:17.5   Train Error:0.0123801234711   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.275557041168
Epoch:18.0   Train Error:0.0136160432816   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.273067951202
Epoch:18.1   Train Error:0.0107903344914   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.276652812958
Epoch:18.2   Train Error:0.0137119559586   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272254228592
Epoch:18.3   Train Error:0.0147396407977   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.274163007736
Epoch:18.4   Train Error:0.0143349383876   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272857904434
Epoch:18.5   Train Error:0.0123160033312   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.274688959122
Epoch:19.0   Train Error:0.0135020543486   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272243976593
Epoch:19.1   Train Error:0.0107131835118   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.27530503273
---------------
Test Error:72/10000=0.0072   Logprob Test:0.0238260519101   ---Best:70/10000=0.007   ---Name:No-Name
---------------
Epoch:19.2   Train Error:0.0136103315368   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272467851639
Epoch:19.3   Train Error:0.0146360319942   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272130012512
Epoch:19.4   Train Error:0.014239218767   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.273233175278
Epoch:19.5   Train Error:0.0122507452279   Time:0.11   Learning Rate:0.01   Databatch Waiting:0.272469997406
Epoch:20.0   Train Error:0.0135362301603   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.273046970367
Epoch:20.1   Train Error:0.0106876020312   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.273488044739
Epoch:20.2   Train Error:0.0132717124954   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.272972106934
Epoch:20.3   Train Error:0.0145221499234   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.271677970886
Epoch:20.4   Train Error:0.0139383209914   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.274121046066
Epoch:20.5   Train Error:0.0117869476827   Time:0.11   Learning Rate:0.001   Databatch Waiting:0.272468090057
---------------
Test Error:69/10000=0.0069   Logprob Test:0.0237230036227   ---Best:69/10000=0.0069   ---Name:No-Name
---------------
Out[20]:
(69, 0.023723003622703246)